Skip to content

M
milvus

milvus / milvus
9 个月 前同步成功

通知 260
Star 22476
Fork 2472
M
milvus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 4dee7dfa 编写于 作者: S shengjh 提交者： GitHub
浏览文件
操作

Improve codecov (#1095) 

* Optimize config test. Dir src/config 99% lines covered

* add unittest coverage

* optimize cache&config unittest

* code format

* format

* format code

* fix merge conflict

* cover src/utils unittest

*  '#831 fix exe_path judge error'

* #831 fix exe_path judge error

* add some unittest coverage

* add some unittest coverage

* improve coverage of src/wrapper

* improve src/wrapper coverage

* *test optimize db/meta unittest

* fix bug

* *test optimize mysqlMetaImpl unittest

* *style: format code

* import server& scheduler unittest coverage

* handover next work

* *test: add some test_meta test case

* *format code

* *fix: fix typo

* feat(codecov): improve code coverage for src/db(#872)

* feat(codecov): improve code coverage for src/db/engine(#872)

* feat(codecov): improve code coverage(#872)

* fix config unittest bug

* feat(codecov): improve code coverage core/db/engine(#872)

* feat(codecov): improve code coverage core/knowhere

* feat(codecov): improve code coverage core/knowhere

* feat(codecov): improve code coverage

* feat(codecov): fix cpu test some error

* feat(codecov): improve code coverage

* feat(codecov): rename some fiu

* fix(db/meta): fix switch/case default action

* feat(codecov): improve code coverage(#872)
* fix error caused by merge code
* format code

* feat(codecov): improve code coverage & format code(#872)

* feat(codecov): fix test error(#872)

* feat(codecov): fix unittest test_mem(#872)

* feat(codecov): fix unittest(#872)

* feat(codecov): fix unittest for resource manager(#872)

* feat(codecov): code format (#872)

* feat(codecov): trigger ci(#872)

* fix(RequestScheduler): remove a wrong sleep statement

* test(test_rpc): fix rpc test

* Fix format issue

* Remove unused comments

* Fix unit test error
Co-authored-by: NABNER-1 <ABNER-1@users.noreply.github.com>
Co-authored-by: NJin Hai <hai.jin@zilliz.com>
上级 cd800042
展开全部 隐藏空白更改
内联 并排
Showing with 4078 addition and 193 deletion
core/src/cache/CpuCacheMgr.cpp
浏览文件 @ 4dee7dfa
......@@ -19,6 +19,7 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <utility>
namespace milvus {
......
core/src/cache/GpuCacheMgr.cpp
浏览文件 @ 4dee7dfa
......@@ -19,6 +19,7 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <sstream>
#include <utility>
......
core/src/config/ConfigNode.cpp
浏览文件 @ 4dee7dfa
......@@ -19,6 +19,7 @@
#include "utils/Error.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <algorithm>
#include <sstream>
#include <string>
......
core/src/db/DBImpl.cpp
浏览文件 @ 4dee7dfa
......@@ -220,17 +220,20 @@ DBImpl::PreloadTable(const std::string& table_id) {
for (auto& file : files_array) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
fiu_do_on("DBImpl.PreloadTable.null_engine", engine = nullptr);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
size += engine->PhysicalSize();
fiu_do_on("DBImpl.PreloadTable.exceed_cache", size = available_size + 1);
if (size > available_size) {
ENGINE_LOG_DEBUG << "Pre-load canceled since cache almost full";
return Status(SERVER_CACHE_FULL, "Cache is full");
} else {
try {
fiu_do_on("DBImpl.PreloadTable.engine_throw_exception", throw std::exception());
std::string msg = "Pre-loaded file: " + file.file_id_ + " size: " + std::to_string(file.file_size_);
TimeRecorderAuto rc_1(msg);
engine->Load(true);
......@@ -492,6 +495,7 @@ DBImpl::QueryByFileID(const std::shared_ptr<server::Context>& context, const std
return status;
}
fiu_do_on("DBImpl.QueryByFileID.empty_files_array", files_array.clear());
if (files_array.empty()) {
return Status(DB_ERROR, "Invalid file id");
}
......@@ -601,11 +605,11 @@ DBImpl::StartMetricTask() {
return;
}
// ENGINE_LOG_TRACE << "Start metric task";
server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL);
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity();
fiu_do_on("DBImpl.StartMetricTask.InvalidTotalCache", cache_total = 0);
if (cache_total > 0) {
double cache_usage_double = cache_usage;
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage_double * 100 / cache_total);
......@@ -627,8 +631,6 @@ DBImpl::StartMetricTask() {
server::Metrics::GetInstance().GPUTemperature();
server::Metrics::GetInstance().CPUTemperature();
server::Metrics::GetInstance().PushToGateway();
// ENGINE_LOG_TRACE << "Metric task finished";
}
Status
......@@ -733,6 +735,8 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m
// step 3: serialize to disk
try {
status = index->Serialize();
fiu_do_on("DBImpl.MergeFiles.Serialize_ThrowException", throw std::exception());
fiu_do_on("DBImpl.MergeFiles.Serialize_ErrorStatus", status = Status(DB_ERROR, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
}
......@@ -987,6 +991,7 @@ DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& da
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = DropTableRecursively(schema.table_id_, dates);
fiu_do_on("DBImpl.DropTableRecursively.failed", status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -1000,6 +1005,8 @@ DBImpl::UpdateTableIndexRecursively(const std::string& table_id, const TableInde
DropIndex(table_id);
auto status = meta_ptr_->UpdateTableIndex(table_id, index);
fiu_do_on("DBImpl.UpdateTableIndexRecursively.fail_update_table_index",
status = Status(DB_META_TRANSACTION_FAILED, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
return status;
......@@ -1060,6 +1067,8 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = BuildTableIndexRecursively(schema.table_id_, index);
fiu_do_on("DBImpl.BuildTableIndexRecursively.fail_build_table_Index_for_partition",
status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -1068,6 +1077,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
// failed to build index for some files, return error
std::string err_msg;
index_failed_checker_.GetErrMsgForTable(table_id, err_msg);
fiu_do_on("DBImpl.BuildTableIndexRecursively.not_empty_err_msg", err_msg.append("fiu"));
if (!err_msg.empty()) {
return Status(DB_ERROR, err_msg);
}
......@@ -1089,6 +1099,8 @@ DBImpl::DropTableIndexRecursively(const std::string& table_id) {
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = DropTableIndexRecursively(schema.table_id_);
fiu_do_on("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition",
status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -1111,6 +1123,8 @@ DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_c
for (auto& schema : partition_array) {
uint64_t partition_row_count = 0;
status = GetTableRowCountRecursively(schema.table_id_, partition_row_count);
fiu_do_on("DBImpl.GetTableRowCountRecursively.fail_get_table_rowcount_for_partition",
status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/db/IDGenerator.cpp
浏览文件 @ 4dee7dfa
......@@ -18,6 +18,7 @@
#include "db/IDGenerator.h"
#include <assert.h>
#include <fiu-local.h>
#include <chrono>
#include <iostream>
......
core/src/db/Options.cpp
浏览文件 @ 4dee7dfa
......@@ -16,6 +16,8 @@
// under the License.
#include "db/Options.h"
#include <fiu-local.h>
#include <limits>
#include "utils/Exception.h"
#include "utils/Log.h"
......@@ -45,6 +47,7 @@ ArchiveConf::ParseCritirias(const std::string& criterias) {
boost::algorithm::split(tokens, criterias, boost::is_any_of(";"));
fiu_do_on("ArchiveConf.ParseCritirias.empty_tokens", tokens.clear());
if (tokens.size() == 0) {
return;
}
......@@ -65,6 +68,8 @@ ArchiveConf::ParseCritirias(const std::string& criterias) {
continue;
}
try {
fiu_do_on("ArchiveConf.ParseCritirias.OptionsParseCritiriasOutOfRange",
kv[1] = std::to_string(std::numeric_limits<int>::max() + 1UL));
auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value;
} catch (std::out_of_range&) {
......
core/src/db/Utils.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <boost/filesystem.hpp>
#include <chrono>
#include <mutex>
......@@ -94,6 +95,7 @@ CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
for (auto& path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
fiu_do_on("CreateTablePath.creat_slave_path", status = Status(DB_INVALID_PATH, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
......@@ -141,6 +143,7 @@ CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_f
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
fiu_do_on("CreateTableFilePath.fail_create", status = Status(DB_INVALID_PATH, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
......@@ -159,6 +162,7 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigS3Enable(s3_enable);
fiu_do_on("GetTableFilePath.enable_s3", s3_enable = true);
if (s3_enable) {
/* need not check file existence */
table_file.location_ = file_path;
......
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @ 4dee7dfa
......@@ -17,6 +17,7 @@
#include "db/engine/ExecutionEngineImpl.h"
#include <fiu-local.h>
#include <stdexcept>
#include <utility>
#include <vector>
......@@ -139,7 +140,10 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
server::Config& config = server::Config::GetInstance();
bool gpu_resource_enable = true;
config.GetGpuResourceConfigEnable(gpu_resource_enable);
fiu_do_on("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled", gpu_resource_enable = false);
#endif
fiu_do_on("ExecutionEngineImpl.CreatetVecIndex.invalid_type", type = EngineType::INVALID);
std::shared_ptr<VecIndex> index;
switch (type) {
case EngineType::FAISS_IDMAP: {
......
core/src/db/meta/MySQLConnectionPool.cpp
浏览文件 @ 4dee7dfa
......@@ -16,6 +16,7 @@
// under the License.
#include "db/meta/MySQLConnectionPool.h"
#include <fiu-local.h>
namespace milvus {
namespace engine {
......@@ -65,6 +66,8 @@ MySQLConnectionPool::getDB() {
mysqlpp::Connection*
MySQLConnectionPool::create() {
try {
fiu_do_on("MySQLConnectionPool.create.throw_exception", throw mysqlpp::ConnectionFailed());
// Create connection using the parameters we were passed upon
// creation.
auto conn = new mysqlpp::Connection();
......
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 4dee7dfa
此差异已折叠。
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ 4dee7dfa
......@@ -35,6 +35,7 @@
#include <memory>
#include <set>
#include <sstream>
#include <fiu-local.h>
namespace milvus {
namespace engine {
......@@ -120,7 +121,9 @@ SqliteMetaImpl::NextFileId(std::string& file_id) {
void
SqliteMetaImpl::ValidateMetaSchema() {
if (ConnectorPtr == nullptr) {
bool is_null_connector{ConnectorPtr == nullptr};
fiu_do_on("SqliteMetaImpl.ValidateMetaSchema.NullConnection", is_null_connector = true);
if (is_null_connector) {
return;
}
......@@ -140,6 +143,7 @@ Status
SqliteMetaImpl::Initialize() {
if (!boost::filesystem::is_directory(options_.path_)) {
auto ret = boost::filesystem::create_directory(options_.path_);
fiu_do_on("SqliteMetaImpl.Initialize.fail_create_directory", ret = false);
if (!ret) {
std::string msg = "Failed to create db directory " + options_.path_;
ENGINE_LOG_ERROR << msg;
......@@ -171,6 +175,7 @@ SqliteMetaImpl::CreateTable(TableSchema& table_schema) {
if (table_schema.table_id_ == "") {
NextTableId(table_schema.table_id_);
} else {
fiu_do_on("SqliteMetaImpl.CreateTable.throw_exception", throw std::exception());
auto table = ConnectorPtr->select(columns(&TableSchema::state_),
where(c(&TableSchema::table_id_) == table_schema.table_id_));
if (table.size() == 1) {
......@@ -187,6 +192,7 @@ SqliteMetaImpl::CreateTable(TableSchema& table_schema) {
table_schema.created_on_ = utils::GetMicroSecTimeStamp();
try {
fiu_do_on("SqliteMetaImpl.CreateTable.insert_throw_exception", throw std::exception());
auto id = ConnectorPtr->insert(table_schema);
table_schema.id_ = id;
} catch (std::exception& e) {
......@@ -206,6 +212,7 @@ SqliteMetaImpl::DescribeTable(TableSchema& table_schema) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.DescribeTable.throw_exception", throw std::exception());
auto groups = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::state_,
&TableSchema::dimension_,
......@@ -249,6 +256,7 @@ SqliteMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
has_or_not = false;
try {
fiu_do_on("SqliteMetaImpl.HasTable.throw_exception", throw std::exception());
server::MetricCollector metric;
auto tables = ConnectorPtr->select(columns(&TableSchema::id_),
where(c(&TableSchema::table_id_) == table_id
......@@ -268,8 +276,8 @@ SqliteMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
Status
SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
try {
fiu_do_on("SqliteMetaImpl.AllTables.throw_exception", throw std::exception());
server::MetricCollector metric;
auto selected = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::table_id_,
&TableSchema::dimension_,
......@@ -310,6 +318,8 @@ SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
Status
SqliteMetaImpl::DropTable(const std::string& table_id) {
try {
fiu_do_on("SqliteMetaImpl.DropTable.throw_exception", throw std::exception());
server::MetricCollector metric;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
......@@ -334,6 +344,8 @@ SqliteMetaImpl::DropTable(const std::string& table_id) {
Status
SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) {
try {
fiu_do_on("SqliteMetaImpl.DeleteTableFiles.throw_exception", throw std::exception());
server::MetricCollector metric;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
......@@ -369,6 +381,7 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) {
}
try {
fiu_do_on("SqliteMetaImpl.CreateTableFile.throw_exception", throw std::exception());
server::MetricCollector metric;
NextFileId(file_schema.file_id_);
......@@ -413,6 +426,8 @@ SqliteMetaImpl::DropDataByDate(const std::string& table_id,
}
try {
fiu_do_on("SqliteMetaImpl.DropDataByDate.throw_exception", throw std::exception());
// sqlite_orm has a bug, 'in' statement cannot handle too many elements
// so we split one query into multi-queries, this is a work-around!!
std::vector<DatesT> split_dates;
......@@ -453,6 +468,8 @@ SqliteMetaImpl::GetTableFiles(const std::string& table_id,
const std::vector<size_t>& ids,
TableFilesSchema& table_files) {
try {
fiu_do_on("SqliteMetaImpl.GetTableFiles.throw_exception", throw std::exception());
table_files.clear();
auto files = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::file_id_,
......@@ -505,6 +522,7 @@ Status
SqliteMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFlag.throw_exception", throw std::exception());
//set all backup file to raw
ConnectorPtr->update_all(
......@@ -526,6 +544,7 @@ SqliteMetaImpl::UpdateTableFile(TableFileSchema& file_schema) {
file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFile.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
......@@ -554,6 +573,7 @@ Status
SqliteMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFiles.throw_exception", throw std::exception());
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
......@@ -584,6 +604,7 @@ SqliteMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
}
return true;
});
fiu_do_on("SqliteMetaImpl.UpdateTableFiles.fail_commited", commited = false);
if (!commited) {
return HandleException("UpdateTableFiles error: sqlite transaction failed");
......@@ -600,6 +621,7 @@ Status
SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableIndex.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
......@@ -659,6 +681,7 @@ Status
SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFilesToIndex.throw_exception", throw std::exception());
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
......@@ -683,6 +706,7 @@ Status
SqliteMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.DescribeTableIndex.throw_exception", throw std::exception());
auto groups = ConnectorPtr->select(columns(&TableSchema::engine_type_,
&TableSchema::nlist_,
......@@ -708,6 +732,7 @@ Status
SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.DropTableIndex.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
......@@ -806,6 +831,7 @@ Status
SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.ShowPartitions.throw_exception", throw std::exception());
auto partitions = ConnectorPtr->select(columns(&TableSchema::table_id_),
where(c(&TableSchema::owner_table_) == table_id
......@@ -828,6 +854,7 @@ Status
SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.GetPartitionName.throw_exception", throw std::exception());
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
......@@ -859,6 +886,8 @@ SqliteMetaImpl::FilesToSearch(const std::string& table_id,
server::MetricCollector metric;
try {
fiu_do_on("SqliteMetaImpl.FilesToSearch.throw_exception", throw std::exception());
auto select_columns =
columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_,
&TableFileSchema::file_type_, &TableFileSchema::file_size_, &TableFileSchema::row_count_,
......@@ -972,6 +1001,8 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
files.clear();
try {
fiu_do_on("SqliteMetaImpl.FilesToMerge.throw_exception", throw std::exception());
server::MetricCollector metric;
// check table existence
......@@ -1040,6 +1071,8 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema& files) {
files.clear();
try {
fiu_do_on("SqliteMetaImpl.FilesToIndex.throw_exception", throw std::exception());
server::MetricCollector metric;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
......@@ -1078,6 +1111,8 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema& files) {
TableSchema table_schema;
table_schema.table_id_ = table_file.table_id_;
auto status = DescribeTable(table_schema);
fiu_do_on("SqliteMetaImpl_FilesToIndex_TableNotFound",
status = Status(DB_NOT_FOUND, "table not found"));
if (!status.ok()) {
return status;
}
......@@ -1108,6 +1143,8 @@ SqliteMetaImpl::FilesByType(const std::string& table_id,
}
try {
fiu_do_on("SqliteMetaImpl.FilesByType.throw_exception", throw std::exception());
table_files.clear();
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::file_id_,
......@@ -1157,8 +1194,7 @@ SqliteMetaImpl::FilesByType(const std::string& table_id,
case (int)TableFileSchema::BACKUP:
++backup_count;
break;
default:
return Status(DB_ERROR, "Unknown file type.");
default:break;
}
table_files.emplace_back(file_schema);
......@@ -1167,29 +1203,25 @@ SqliteMetaImpl::FilesByType(const std::string& table_id,
std::string msg = "Get table files by type.";
for (int file_type : file_types) {
switch (file_type) {
case (int)TableFileSchema::RAW:
msg = msg + " raw files:" + std::to_string(raw_count);
case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count);
break;
case (int)TableFileSchema::NEW:
msg = msg + " new files:" + std::to_string(new_count);
case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count);
break;
case (int)TableFileSchema::NEW_MERGE:
msg = msg + " new_merge files:" + std::to_string(new_merge_count);
msg = msg + " new_merge files:"
+ std::to_string(new_merge_count);
break;
case (int)TableFileSchema::NEW_INDEX:
msg = msg + " new_index files:" + std::to_string(new_index_count);
msg = msg + " new_index files:"
+ std::to_string(new_index_count);
break;
case (int)TableFileSchema::TO_INDEX:
msg = msg + " to_index files:" + std::to_string(to_index_count);
case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count);
break;
case (int)TableFileSchema::INDEX:
msg = msg + " index files:" + std::to_string(index_count);
case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count);
break;
case (int)TableFileSchema::BACKUP:
msg = msg + " backup files:" + std::to_string(backup_count);
case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count);
break;
default:
return Status(DB_ERROR, "Unknown file type!");
default:break;
}
}
ENGINE_LOG_DEBUG << msg;
......@@ -1215,6 +1247,8 @@ SqliteMetaImpl::Archive() {
int64_t usecs = limit * DAY * US_PS;
int64_t now = utils::GetMicroSecTimeStamp();
try {
fiu_do_on("SqliteMetaImpl.Archive.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
......@@ -1248,6 +1282,8 @@ Status
SqliteMetaImpl::Size(uint64_t& result) {
result = 0;
try {
fiu_do_on("SqliteMetaImpl.Size.throw_exception", throw std::exception());
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::file_size_)),
where(c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE));
for (auto& total_size : selected) {
......@@ -1287,6 +1323,8 @@ SqliteMetaImpl::CleanUpShadowFiles() {
return true;
});
fiu_do_on("SqliteMetaImpl.CleanUpShadowFiles.fail_commited", commited = false);
fiu_do_on("SqliteMetaImpl.CleanUpShadowFiles.throw_exception", throw std::exception());
if (!commited) {
return HandleException("CleanUp error: sqlite transaction failed");
}
......@@ -1308,6 +1346,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// remove to_delete files
try {
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_ThrowException", throw std::exception());
server::MetricCollector metric;
std::vector<int> file_types = {
......@@ -1359,7 +1399,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// delete file from disk storage
utils::DeleteTableFilePath(options_, table_file);
ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:" << table_file.location_;
ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:"
<< table_file.location_;
table_ids.insert(table_file.table_id_);
++clean_files;
......@@ -1367,6 +1408,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
}
return true;
});
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_FailCommited", commited = false);
if (!commited) {
return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed");
......@@ -1381,6 +1423,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// remove to_delete tables
try {
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_ThrowException", throw std::exception());
server::MetricCollector metric;
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
......@@ -1397,6 +1440,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
return true;
});
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_Failcommited", commited = false);
if (!commited) {
return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed");
......@@ -1412,6 +1456,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// remove deleted table folder
// don't remove table folder until all its files has been deleted
try {
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTableFolder_ThrowException", throw std::exception());
server::MetricCollector metric;
int64_t remove_tables = 0;
......@@ -1437,6 +1482,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
Status
SqliteMetaImpl::Count(const std::string& table_id, uint64_t& result) {
try {
fiu_do_on("SqliteMetaImpl.Count.throw_exception", throw std::exception());
server::MetricCollector metric;
std::vector<int> file_types = {(int)TableFileSchema::RAW, (int)TableFileSchema::TO_INDEX,
......@@ -1486,6 +1533,8 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) {
ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size;
try {
fiu_do_on("SqliteMetaImpl.DiscardFiles.throw_exception", throw std::exception());
server::MetricCollector metric;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
......@@ -1525,7 +1574,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) {
return true;
});
fiu_do_on("SqliteMetaImpl.DiscardFiles.fail_commited", commited = false);
if (!commited) {
return HandleException("DiscardFiles error: sqlite transaction failed");
}
......
core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
浏览文件 @ 4dee7dfa
......@@ -16,6 +16,7 @@
// under the License.
#include <faiss/index_io.h>
#include <fiu-local.h>
#include <utility>
#include "knowhere/common/Exception.h"
......@@ -31,6 +32,7 @@ FaissBaseIndex::FaissBaseIndex(std::shared_ptr<faiss::Index> index) : index_(std
BinarySet
FaissBaseIndex::SerializeImpl() {
try {
fiu_do_on("FaissBaseIndex.SerializeImpl.throw_exception", throw std::exception());
faiss::Index* index = index_.get();
// SealImpl();
......
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include <faiss/IndexFlat.h>
#include <faiss/MetaIndexes.h>
#include <faiss/index_io.h>
#include <fiu-local.h>
#ifdef MILVUS_GPU_VERSION
......@@ -61,6 +62,7 @@ GPUIDMAP::CopyGpuToCpu(const Config& config) {
BinarySet
GPUIDMAP::SerializeImpl() {
try {
fiu_do_on("GPUIDMP.SerializeImpl.throw_exception", throw std::exception());
MemoryIOWriter writer;
{
faiss::Index* index = index_.get();
......
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include <faiss/gpu/GpuIndexIVF.h>
#include <faiss/gpu/GpuIndexIVFFlat.h>
#include <faiss/index_io.h>
#include <fiu-local.h>
#include "knowhere/adapter/VectorAdapter.h"
#include "knowhere/common/Exception.h"
......@@ -81,6 +82,7 @@ GPUIVF::SerializeImpl() {
}
try {
fiu_do_on("GPUIVF.SerializeImpl.throw_exception", throw std::exception());
MemoryIOWriter writer;
{
faiss::Index* index = index_.get();
......@@ -128,7 +130,9 @@ void
GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
std::lock_guard<std::mutex> lk(mutex_);
if (auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_);
fiu_do_on("GPUIVF.search_impl.invald_index", device_index = nullptr);
if (device_index) {
auto search_cfg = std::dynamic_pointer_cast<IVFCfg>(cfg);
device_index->nprobe = search_cfg->nprobe;
// assert(device_index->getNumProbes() == search_cfg->nprobe);
......
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
浏览文件 @ 4dee7dfa
......@@ -29,6 +29,7 @@
#include <faiss/gpu/GpuCloner.h>
#endif
#include <fiu-local.h>
#include <chrono>
#include <memory>
#include <utility>
......@@ -119,6 +120,8 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) {
GETTENSOR(dataset)
try {
fiu_do_on("IVF.Search.throw_std_exception", throw std::exception());
fiu_do_on("IVF.Search.throw_faiss_exception", throw faiss::FaissException(""));
auto elems = rows * search_cfg->k;
size_t p_id_size = sizeof(int64_t) * elems;
......
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp
浏览文件 @ 4dee7dfa
......@@ -21,11 +21,11 @@
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/helpers/FaissIO.h"
#include <utility>
#include <faiss/gpu/GpuCloner.h>
#include <faiss/gpu/GpuIndexIVF.h>
#include <faiss/index_factory.h>
#include <fiu-local.h>
#include <utility>
namespace knowhere {
......@@ -300,7 +300,7 @@ IVFSQHybrid::SerializeImpl() {
if (!index_ || !index_->is_trained) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}
fiu_do_on("IVFSQHybrid.SerializeImpl.zero_gpu_mode", gpu_mode = 0);
if (gpu_mode == 0) {
MemoryIOWriter writer;
faiss::write_index(index_.get(), &writer);
......
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
浏览文件 @ 4dee7dfa
......@@ -19,12 +19,16 @@
#include "knowhere/adapter/VectorAdapter.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Timer.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#endif
#include <fiu-local.h>
#include "knowhere/index/vector_index/IndexIDMAP.h"
#include "knowhere/index/vector_index/IndexIVF.h"
#include "knowhere/index/vector_index/nsg/NSG.h"
......@@ -39,6 +43,7 @@ NSG::Serialize() {
}
try {
fiu_do_on("NSG.Serialize.throw_exception", throw std::exception());
algo::NsgIndex* index = index_.get();
MemoryIOWriter writer;
......@@ -57,6 +62,7 @@ NSG::Serialize() {
void
NSG::Load(const BinarySet& index_binary) {
try {
fiu_do_on("NSG.Load.throw_exception", throw std::exception());
auto binary = index_binary.GetByName("NSG");
MemoryIOReader reader;
......
core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
浏览文件 @ 4dee7dfa
......@@ -17,6 +17,7 @@
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include <fiu-local.h>
#include <utility>
namespace knowhere {
......@@ -83,6 +84,7 @@ FaissGpuResourceMgr::InitResource() {
ResPtr
FaissGpuResourceMgr::GetRes(const int64_t& device_id, const int64_t& alloc_size) {
fiu_return_on("FaissGpuResourceMgr.GetRes.ret_null", nullptr);
InitResource();
auto finder = idle_map_.find(device_id);
......
core/src/index/unittest/CMakeLists.txt
浏览文件 @ 4dee7dfa
......@@ -5,7 +5,7 @@ include_directories(${INDEX_SOURCE_DIR})
set(depend_libs
gtest gmock gtest_main gmock_main
faiss
faiss fiu
)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
......@@ -117,11 +117,18 @@ if (KNOWHERE_GPU_VERSION)
target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs})
endif ()
if (NOT TARGET test_knowhere_common)
add_executable(test_knowhere_common test_common.cpp ${util_srcs})
endif ()
target_link_libraries(test_knowhere_common ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ivf DESTINATION unittest)
install(TARGETS test_binaryivf DESTINATION unittest)
install(TARGETS test_idmap DESTINATION unittest)
install(TARGETS test_binaryidmap DESTINATION unittest)
install(TARGETS test_sptag DESTINATION unittest)
install(TARGETS test_knowhere_common DESTINATION unittest)
if (KNOWHERE_GPU_VERSION)
install(TARGETS test_gpuresource DESTINATION unittest)
install(TARGETS test_customized_index DESTINATION unittest)
......
core/src/index/unittest/test_common.cpp 0 → 100644
浏览文件 @ 4dee7dfa
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "knowhere/common/Dataset.h"
#include "knowhere/common/Timer.h"
#include "knowhere/knowhere/common/Exception.h"
#include "unittest/utils.h"
/*Some unittest for knowhere/common, mainly for improve code coverage.*/
TEST(COMMON_TEST, dataset_test) {
knowhere::Dataset set;
int64_t v1 = 111;
set.Set("key1", v1);
auto get_v1 = set.Get<int64_t>("key1");
ASSERT_EQ(get_v1, v1);
ASSERT_ANY_THROW(set.Get<int8_t>("key1"));
ASSERT_ANY_THROW(set.Get<int64_t>("dummy"));
}
TEST(COMMON_TEST, knowhere_exception) {
const std::string msg = "test";
knowhere::KnowhereException ex(msg);
ASSERT_EQ(ex.what(), msg);
}
TEST(COMMON_TEST, time_recoder) {
InitLog();
knowhere::TimeRecorder recoder("COMMTEST", 0);
sleep(1);
double span = recoder.ElapseFromBegin("get time");
ASSERT_GE(span, 1.0);
}
core/src/index/unittest/test_customized_index.cpp
浏览文件 @ 4dee7dfa
......@@ -15,8 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#include <fiu-control.h>
#include <fiu-local.h>
#include <gtest/gtest.h>
#include <thread>
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "unittest/Helper.h"
#include "unittest/utils.h"
......@@ -55,6 +58,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
auto preprocessor = index_->BuildPreprocessor(base_dataset, conf);
index_->set_preprocessor(preprocessor);
fiu_init(0);
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
index_->Add(base_dataset, conf);
......@@ -82,6 +86,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
auto cpu_idx = std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
cpu_idx->Load(binaryset);
ASSERT_ANY_THROW(cpu_idx->CopyCpuToGpuWithQuantizer(-1, conf));
auto pair = cpu_idx->CopyCpuToGpuWithQuantizer(DEVICEID, conf);
auto gpu_idx = pair.first;
auto quantization = pair.second;
......@@ -96,12 +101,21 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
for (int i = 0; i < 2; ++i) {
auto hybrid_idx = std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
hybrid_idx->Load(binaryset);
auto new_idx = hybrid_idx->LoadData(quantization, quantizer_conf);
auto result = new_idx->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
// PrintResult(result, nq, k);
}
{
// invalid quantizer config
quantizer_conf = std::make_shared<knowhere::QuantizerCfg>();
auto hybrid_idx = std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, nullptr));
ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, quantizer_conf));
quantizer_conf->mode = 2; // only copy data
ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, quantizer_conf));
}
}
{
......
core/src/index/unittest/test_idmap.cpp
浏览文件 @ 4dee7dfa
......@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include <fiu-control.h>
#include <fiu-local.h>
#include <gtest/gtest.h>
#include <iostream>
......@@ -54,6 +56,14 @@ TEST_F(IDMAPTest, idmap_basic) {
conf->k = k;
conf->metric_type = knowhere::METRICTYPE::L2;
// null faiss index
{
ASSERT_ANY_THROW(index_->Serialize());
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
ASSERT_ANY_THROW(index_->Add(nullptr, conf));
ASSERT_ANY_THROW(index_->AddWithoutId(nullptr, conf));
}
index_->Train(conf);
index_->Add(base_dataset, conf);
EXPECT_EQ(index_->Count(), nb);
......@@ -145,6 +155,7 @@ TEST_F(IDMAPTest, copy_test) {
{
// cpu to gpu
ASSERT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(index_, -1, conf));
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, conf);
auto clone_result = clone_index->Search(query_dataset, conf);
AssertAnns(clone_result, nq, k);
......@@ -153,6 +164,11 @@ TEST_F(IDMAPTest, copy_test) {
ASSERT_THROW({ std::static_pointer_cast<knowhere::GPUIDMAP>(clone_index)->GetRawIds(); },
knowhere::KnowhereException);
fiu_init(0);
fiu_enable("GPUIDMP.SerializeImpl.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(clone_index->Serialize());
fiu_disable("GPUIDMP.SerializeImpl.throw_exception");
auto binary = clone_index->Serialize();
clone_index->Load(binary);
auto new_result = clone_index->Search(query_dataset, conf);
......
core/src/index/unittest/test_ivf.cpp
浏览文件 @ 4dee7dfa
......@@ -17,11 +17,15 @@
#include <gtest/gtest.h>
#include <fiu-control.h>
#include <fiu-local.h>
#include <iostream>
#include <thread>
#ifdef MILVUS_GPU_VERSION
#include <faiss/gpu/GpuIndexIVFFlat.h>
#endif
#include "knowhere/adapter/VectorAdapter.h"
......@@ -33,11 +37,13 @@
#include "knowhere/index/vector_index/IndexIVFSQ.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
#include "knowhere/index/vector_index/IndexGPUIVFSQ.h"
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#endif
#include "unittest/Helper.h"
......@@ -54,15 +60,14 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, P
#ifdef MILVUS_GPU_VERSION
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM);
#endif
ParameterType parameter_type;
std::tie(index_type, parameter_type) = GetParam();
std::tie(index_type, parameter_type_) = GetParam();
// Init_with_default();
// nb = 1000000;
// nq = 1000;
// k = 1000;
Generate(DIM, NB, NQ);
index_ = IndexFactory(index_type);
conf = ParamGenerator::GetInstance().Gen(parameter_type);
conf = ParamGenerator::GetInstance().Gen(parameter_type_);
conf->Dump();
}
......@@ -77,6 +82,7 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, P
std::string index_type;
knowhere::Config conf;
knowhere::IVFIndexPtr index_ = nullptr;
ParameterType parameter_type_;
};
INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
......@@ -98,6 +104,10 @@ TEST_P(IVFTest, ivf_basic) {
auto preprocessor = index_->BuildPreprocessor(base_dataset, conf);
index_->set_preprocessor(preprocessor);
// null faiss index
ASSERT_ANY_THROW(index_->Add(base_dataset, conf));
ASSERT_ANY_THROW(index_->AddWithoutIds(base_dataset, conf));
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
index_->Add(base_dataset, conf);
......@@ -107,9 +117,14 @@ TEST_P(IVFTest, ivf_basic) {
auto result = index_->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
// PrintResult(result, nq, k);
#ifdef MILVUS_GPU_VERSION
knowhere::FaissGpuResourceMgr::GetInstance().Dump();
#endif
}
TEST_P(IVFTest, ivf_serialize) {
fiu_init(0);
auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) {
FileIOWriter writer(filename);
writer(static_cast<void*>(bin->data.get()), bin->size);
......@@ -118,12 +133,23 @@ TEST_P(IVFTest, ivf_serialize) {
reader(ret, bin->size);
};
{
// null faisss index serialize
ASSERT_ANY_THROW(index_->Serialize());
knowhere::IVFIndexModel model(nullptr);
ASSERT_ANY_THROW(model.Serialize());
}
{
// serialize index-model
auto model = index_->Train(base_dataset, conf);
auto binaryset = model->Serialize();
auto bin = binaryset.GetByName("IVF");
fiu_enable("FaissBaseIndex.SerializeImpl.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(model->Serialize());
fiu_disable("FaissBaseIndex.SerializeImpl.throw_exception");
std::string filename = "/tmp/ivf_test_model_serialize.bin";
auto load_data = new uint8_t[bin->size];
serialize(filename, bin, load_data);
......@@ -250,6 +276,7 @@ TEST_P(IVFTest, clone_test) {
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
});
EXPECT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(index_, -1, knowhere::Config()));
} else {
EXPECT_THROW(
{
......@@ -260,6 +287,7 @@ TEST_P(IVFTest, clone_test) {
}
}
}
#endif
#ifdef MILVUS_GPU_VERSION
......@@ -276,6 +304,9 @@ TEST_P(IVFTest, gpu_seal_test) {
auto preprocessor = index_->BuildPreprocessor(base_dataset, conf);
index_->set_preprocessor(preprocessor);
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
ASSERT_ANY_THROW(index_->Seal());
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
index_->Add(base_dataset, conf);
......@@ -284,6 +315,14 @@ TEST_P(IVFTest, gpu_seal_test) {
auto result = index_->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
fiu_init(0);
fiu_enable("IVF.Search.throw_std_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
fiu_disable("IVF.Search.throw_std_exception");
fiu_enable("IVF.Search.throw_faiss_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
fiu_disable("IVF.Search.throw_faiss_exception");
auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
knowhere::TimeRecorder tc("CopyToGpu");
......@@ -294,6 +333,89 @@ TEST_P(IVFTest, gpu_seal_test) {
knowhere::cloner::CopyCpuToGpu(cpu_idx, DEVICEID, knowhere::Config());
auto with_seal = tc.RecordSection("With seal");
ASSERT_GE(without_seal, with_seal);
// copy to GPU with invalid device id
ASSERT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(cpu_idx, -1, knowhere::Config()));
}
#endif
TEST_P(IVFTest, invalid_gpu_source) {
std::vector<std::string> support_idx_vec{"GPUIVF", "GPUIVFPQ", "GPUIVFSQ"};
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder == support_idx_vec.cend()) {
return;
}
auto invalid_conf = ParamGenerator::GetInstance().Gen(parameter_type_);
invalid_conf->gpu_id = -1;
if (index_type == "GPUIVF") {
// null faiss index
knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
}
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
auto binaryset = model->Serialize();
fiu_init(0);
fiu_enable("GPUIVF.SerializeImpl.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Serialize());
fiu_disable("GPUIVF.SerializeImpl.throw_exception");
fiu_enable("GPUIVF.search_impl.invald_index", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Search(base_dataset, invalid_conf));
fiu_disable("GPUIVF.search_impl.invald_index");
auto ivf_index = std::dynamic_pointer_cast<knowhere::GPUIVF>(index_);
if (ivf_index) {
auto gpu_index = std::dynamic_pointer_cast<knowhere::GPUIndex>(ivf_index);
gpu_index->SetGpuDevice(-1);
ASSERT_EQ(gpu_index->GetGpuDevice(), -1);
ASSERT_ANY_THROW(index_->set_index_model(model));
}
ASSERT_ANY_THROW(index_->Load(binaryset));
ASSERT_ANY_THROW(index_->Train(base_dataset, invalid_conf));
}
#ifdef CUSTOMIZATION
TEST_P(IVFTest, IVFSQHybrid_test) {
std::vector<std::string> support_idx_vec{"IVFSQHybrid"};
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder == support_idx_vec.cend()) {
return;
}
fiu_init(0);
knowhere::cloner::CopyGpuToCpu(index_, conf);
ASSERT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(index_, -1, conf));
fiu_enable("FaissGpuResourceMgr.GetRes.ret_null", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Train(base_dataset, conf));
ASSERT_ANY_THROW(index_->CopyCpuToGpu(DEVICEID, conf));
fiu_disable("FaissGpuResourceMgr.GetRes.ret_null");
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
auto index = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_);
ASSERT_TRUE(index != nullptr);
ASSERT_ANY_THROW(index->UnsetQuantizer());
knowhere::QuantizerConfig config = std::make_shared<knowhere::QuantizerCfg>();
config->gpu_id = knowhere::INVALID_VALUE;
// mode = -1
ASSERT_ANY_THROW(index->LoadQuantizer(config));
config->mode = 1;
ASSERT_ANY_THROW(index->LoadQuantizer(config));
config->gpu_id = DEVICEID;
// index->LoadQuantizer(config);
ASSERT_ANY_THROW(index->SetQuantizer(nullptr));
}
#endif
#endif
core/src/index/unittest/test_nsg/test_nsg.cpp
浏览文件 @ 4dee7dfa
......@@ -30,6 +30,8 @@
#include "knowhere/common/Timer.h"
#include "knowhere/index/vector_index/nsg/NSGIO.h"
#include <fiu-control.h>
#include <fiu-local.h>
#include "unittest/utils.h"
using ::testing::Combine;
......@@ -42,7 +44,7 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
protected:
void
SetUp() override {
// Init_with_default();
// Init_with_default();
#ifdef MILVUS_GPU_VERSION
int64_t MB = 1024 * 1024;
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, MB * 200, MB * 600, 1);
......@@ -85,14 +87,34 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
TEST_F(NSGInterfaceTest, basic_test) {
assert(!xb.empty());
fiu_init(0);
// untrained index
{
ASSERT_ANY_THROW(index_->Search(query_dataset, search_conf));
ASSERT_ANY_THROW(index_->Serialize());
}
train_conf->gpu_id = knowhere::INVALID_VALUE;
auto model_invalid_gpu = index_->Train(base_dataset, train_conf);
train_conf->gpu_id = DEVICEID;
auto model = index_->Train(base_dataset, train_conf);
auto result = index_->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);
auto binaryset = index_->Serialize();
{
fiu_enable("NSG.Serialize.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Serialize());
fiu_disable("NSG.Serialize.throw_exception");
}
auto new_index = std::make_shared<knowhere::NSG>();
new_index->Load(binaryset);
{
fiu_enable("NSG.Load.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(new_index->Load(binaryset));
fiu_disable("NSG.Load.throw_exception");
}
auto new_result = new_index->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);
......
core/src/index/unittest/test_sptag.cpp
浏览文件 @ 4dee7dfa
......@@ -95,6 +95,16 @@ TEST_P(SPTAGTest, sptag_basic) {
std::cout << "id\n" << ss_id.str() << std::endl;
std::cout << "dist\n" << ss_dist.str() << std::endl;
}
// Though these functions do nothing, use them to improve code coverage
{
index_->Seal();
knowhere::CPUSPTAGRNGIndexModel index_model;
// Function Serialize's implementation do'nt have return value,
// which will cause undefined behavior.
// index_model.Serialize();
index_model.Load(knowhere::BinarySet());
}
}
TEST_P(SPTAGTest, sptag_serialize) {
......
core/src/index/unittest/utils.cpp
浏览文件 @ 4dee7dfa
......@@ -72,6 +72,8 @@ BinaryDataGen::Generate(const int& dim, const int& nb, const int& nq) {
query_dataset = generate_binary_query_dataset(nq, dim, xq.data());
}
// not used
#if 0
knowhere::DatasetPtr
DataGen::GenQuery(const int& nq) {
xq.resize(nq * dim);
......@@ -80,6 +82,7 @@ DataGen::GenQuery(const int& nq) {
}
return generate_query_dataset(nq, dim, xq.data());
}
#endif
void
GenAll(const int64_t dim, const int64_t& nb, std::vector<float>& xb, std::vector<int64_t>& ids, const int64_t& nq,
......@@ -236,6 +239,8 @@ PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
std::cout << "dist\n" << ss_dist.str() << std::endl;
}
// not used
#if 0
void
Load_nns_graph(std::vector<std::vector<int64_t>>& final_graph, const char* filename) {
std::vector<std::vector<unsigned>> knng;
......@@ -304,3 +309,4 @@ int* // not very clean, but works as long as sizeof(int) == sizeof(float)
ivecs_read(const char* fname, size_t* d_out, size_t* n_out) {
return (int*)fvecs_read(fname, d_out, n_out);
}
#endif
core/src/metrics/SystemInfo.cpp
浏览文件 @ 4dee7dfa
......@@ -20,13 +20,16 @@
#include "utils/Log.h"
#include <dirent.h>
#include <fiu-local.h>
#include <sys/sysinfo.h>
#include <sys/times.h>
#include <unistd.h>
#include <map>
#ifdef MILVUS_GPU_VERSION
#include <nvml.h>
#endif
namespace milvus {
......@@ -64,11 +67,13 @@ SystemInfo::Init() {
// initialize GPU information
nvmlReturn_t nvmlresult;
nvmlresult = nvmlInit();
fiu_do_on("SystemInfo.Init.nvmInit_fail", nvmlresult = NVML_ERROR_NOT_FOUND);
if (NVML_SUCCESS != nvmlresult) {
SERVER_LOG_ERROR << "System information initilization failed";
return;
}
nvmlresult = nvmlDeviceGetCount(&num_device_);
fiu_do_on("SystemInfo.Init.nvm_getDevice_fail", nvmlresult = NVML_ERROR_NOT_FOUND);
if (NVML_SUCCESS != nvmlresult) {
SERVER_LOG_ERROR << "Unable to get devidce number";
return;
......@@ -127,6 +132,7 @@ SystemInfo::GetProcessUsedMemory() {
double
SystemInfo::MemoryPercent() {
fiu_do_on("SystemInfo.MemoryPercent.mock", initialized_ = false);
if (!initialized_) {
Init();
}
......@@ -144,7 +150,7 @@ SystemInfo::CPUCorePercent() {
std::vector<uint64_t> cur_total_time_array = getTotalCpuTime(cur_work_time_array);
std::vector<double> cpu_core_percent;
for (int i = 1; i < num_processors_; i++) {
for (int i = 0; i < cur_total_time_array.size(); i++) {
double total_cpu_time = cur_total_time_array[i] - prev_total_time_array[i];
double cpu_work_time = cur_work_time_array[i] - prev_work_time_array[i];
cpu_core_percent.push_back((cpu_work_time / total_cpu_time) * 100);
......@@ -156,6 +162,7 @@ std::vector<uint64_t>
SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
std::vector<uint64_t> total_time_array;
FILE* file = fopen("/proc/stat", "r");
fiu_do_on("SystemInfo.getTotalCpuTime.open_proc", file = NULL);
if (file == NULL) {
SERVER_LOG_ERROR << "Could not open stat file";
return total_time_array;
......@@ -167,6 +174,7 @@ SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
for (int i = 0; i < num_processors_; i++) {
char buffer[1024];
char* ret = fgets(buffer, sizeof(buffer) - 1, file);
fiu_do_on("SystemInfo.getTotalCpuTime.read_proc", ret = NULL);
if (ret == NULL) {
SERVER_LOG_ERROR << "Could not read stat file";
fclose(file);
......@@ -186,6 +194,7 @@ SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
double
SystemInfo::CPUPercent() {
fiu_do_on("SystemInfo.CPUPercent.mock", initialized_ = false);
if (!initialized_) {
Init();
}
......@@ -212,6 +221,7 @@ SystemInfo::CPUPercent() {
std::vector<uint64_t>
SystemInfo::GPUMemoryTotal() {
// get GPU usage percent
fiu_do_on("SystemInfo.GPUMemoryTotal.mock", initialized_ = false);
if (!initialized_)
Init();
std::vector<uint64_t> result;
......@@ -232,6 +242,7 @@ SystemInfo::GPUMemoryTotal() {
std::vector<uint64_t>
SystemInfo::GPUTemperature() {
fiu_do_on("SystemInfo.GPUTemperature.mock", initialized_ = false);
if (!initialized_)
Init();
std::vector<uint64_t> result;
......@@ -258,6 +269,7 @@ SystemInfo::CPUTemperature() {
DIR* dir = NULL;
dir = opendir(path.c_str());
fiu_do_on("SystemInfo.CPUTemperature.opendir", dir = NULL);
if (!dir) {
SERVER_LOG_ERROR << "Could not open hwmon directory";
return result;
......@@ -275,6 +287,7 @@ SystemInfo::CPUTemperature() {
std::string object = filename;
object += "/temp1_input";
FILE* file = fopen(object.c_str(), "r");
fiu_do_on("SystemInfo.CPUTemperature.openfile", file = NULL);
if (file == nullptr) {
SERVER_LOG_ERROR << "Could not open temperature file";
return result;
......@@ -292,6 +305,7 @@ SystemInfo::CPUTemperature() {
std::vector<uint64_t>
SystemInfo::GPUMemoryUsed() {
// get GPU memory used
fiu_do_on("SystemInfo.GPUMemoryUsed.mock", initialized_ = false);
if (!initialized_)
Init();
......
core/src/scheduler/SchedInst.cpp
浏览文件 @ 4dee7dfa
......@@ -20,6 +20,7 @@
#include "Utils.h"
#include "server/Config.h"
#include <fiu-local.h>
#include <set>
#include <string>
#include <utility>
......@@ -52,7 +53,7 @@ load_simple_config() {
ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0));
ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
// get resources
// get resources
#ifdef MILVUS_GPU_VERSION
bool enable_gpu = false;
server::Config& config = server::Config::GetInstance();
......@@ -63,6 +64,7 @@ load_simple_config() {
std::vector<int64_t> build_gpu_ids;
config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids);
auto pcie = Connection("pcie", 12000);
fiu_do_on("load_simple_config_mock", build_gpu_ids.push_back(1));
std::vector<int64_t> not_find_build_ids;
for (auto& build_id : build_gpu_ids) {
......
core/src/scheduler/TaskTable.cpp
浏览文件 @ 4dee7dfa
......@@ -283,7 +283,7 @@ TaskTable::TaskToExecute() {
auto begin = table_.front() + 1;
for (size_t i = 0; i < table_.size(); ++i) {
auto index = begin + i;
if (table_[index]->state == TaskTableItemState::LOADED) {
if (table_[index] && table_[index]->state == TaskTableItemState::LOADED) {
++count;
}
}
......
core/src/scheduler/action/PushTaskToNeighbour.cpp
浏览文件 @ 4dee7dfa
......@@ -42,6 +42,8 @@ get_neighbours(const ResourcePtr& self) {
return neighbours;
}
// This function has not been invoked, comment it for code coverage
#if 0
std::vector<std::pair<ResourcePtr, Connection>>
get_neighbours_with_connetion(const ResourcePtr& self) {
std::vector<std::pair<ResourcePtr, Connection>> neighbours;
......@@ -87,6 +89,7 @@ Action::PushTaskToNeighbourRandomly(TaskTableItemPtr task_item, const ResourcePt
// TODO(wxyu): process
}
}
#endif
void
Action::PushTaskToAllNeighbour(TaskTableItemPtr task_item, const ResourcePtr& self) {
......@@ -96,10 +99,12 @@ Action::PushTaskToAllNeighbour(TaskTableItemPtr task_item, const ResourcePtr& se
}
}
#if 0
void
Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) {
dest->task_table().Put(task_item->task, task_item);
}
#endif
void
Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource,
......
core/src/scheduler/optimizer/BuildIndexPass.cpp
浏览文件 @ 4dee7dfa
......@@ -14,10 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#ifdef MILVUS_GPU_VERSION
namespace milvus {
......@@ -27,8 +28,9 @@ void
BuildIndexPass::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_);
fiu_do_on("BuildIndexPass.Init.get_config_fail", s = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!s.ok()) {
throw;
throw std::exception();
}
}
......@@ -36,7 +38,7 @@ bool
BuildIndexPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::BuildIndexTask)
return false;
fiu_do_on("BuildIndexPass.Run.empty_gpu_ids", build_gpu_ids_.clear());
if (build_gpu_ids_.empty()) {
SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!";
return false;
......
core/src/scheduler/optimizer/FaissFlatPass.cpp
浏览文件 @ 4dee7dfa
......@@ -24,6 +24,7 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
namespace milvus {
namespace scheduler {
......@@ -36,7 +37,7 @@ FaissFlatPass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
}
......
core/src/scheduler/optimizer/FaissIVFFlatPass.cpp
浏览文件 @ 4dee7dfa
......@@ -37,7 +37,7 @@ FaissIVFFlatPass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
#endif
}
......
core/src/scheduler/optimizer/FaissIVFPQPass.cpp
浏览文件 @ 4dee7dfa
......@@ -24,6 +24,8 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
namespace milvus {
namespace scheduler {
......@@ -37,7 +39,7 @@ FaissIVFPQPass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
#endif
}
......
core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp
浏览文件 @ 4dee7dfa
......@@ -37,7 +37,7 @@ FaissIVFSQ8Pass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
#endif
}
......
core/src/scheduler/task/BuildIndexTask.cpp
浏览文件 @ 4dee7dfa
......@@ -23,6 +23,7 @@
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <thread>
......@@ -60,12 +61,13 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
error_msg = "Wrong load type";
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
fiu_do_on("XBuildIndexTask.Load.throw_std_exception", throw std::exception());
} catch (std::exception& ex) {
// typical error: out of disk space or permition denied
error_msg = "Failed to load to_index file: " + std::string(ex.what());
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
fiu_do_on("XBuildIndexTask.Load.out_of_memory", stat = Status(SERVER_UNEXPECTED_ERROR, "out of memory"));
if (!stat.ok()) {
Status s;
if (stat.ToString().find("out of memory") != std::string::npos) {
......@@ -117,7 +119,8 @@ XBuildIndexTask::Execute() {
table_file.file_type_ = engine::meta::TableFileSchema::NEW_INDEX;
engine::meta::MetaPtr meta_ptr = build_index_job->meta();
Status status = build_index_job->meta()->CreateTableFile(table_file);
Status status = meta_ptr->CreateTableFile(table_file);
fiu_do_on("XBuildIndexTask.Execute.create_table_success", status = Status::OK());
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
build_index_job->BuildIndexDone(to_index_id_);
......@@ -130,6 +133,7 @@ XBuildIndexTask::Execute() {
try {
ENGINE_LOG_DEBUG << "Begin build index for file:" + table_file.location_;
index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
fiu_do_on("XBuildIndexTask.Execute.build_index_fail", index = nullptr);
if (index == nullptr) {
throw Exception(DB_ERROR, "index NULL");
}
......@@ -150,6 +154,8 @@ XBuildIndexTask::Execute() {
// step 4: if table has been deleted, dont save index file
bool has_table = false;
meta_ptr->HasTable(file_->table_id_, has_table);
fiu_do_on("XBuildIndexTask.Execute.has_table", has_table = true);
if (!has_table) {
meta_ptr->DeleteTableFiles(file_->table_id_);
......@@ -161,6 +167,7 @@ XBuildIndexTask::Execute() {
// step 5: save index file
try {
fiu_do_on("XBuildIndexTask.Execute.throw_std_exception", throw std::exception());
status = index->Serialize();
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
......@@ -171,6 +178,7 @@ XBuildIndexTask::Execute() {
status = Status(DB_ERROR, msg);
}
fiu_do_on("XBuildIndexTask.Execute.save_index_file_success", status = Status::OK());
if (!status.ok()) {
// if failed to serialize index file to disk
// typical error: out of disk space, out of memory or permition denied
......@@ -201,6 +209,7 @@ XBuildIndexTask::Execute() {
status = meta_ptr->UpdateTableFiles(update_files);
}
fiu_do_on("XBuildIndexTask.Execute.update_table_file_fail", status = Status(SERVER_UNEXPECTED_ERROR, ""));
if (status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize()
<< " bytes"
......
core/src/scheduler/task/Path.h
浏览文件 @ 4dee7dfa
......@@ -64,7 +64,7 @@ class Path {
if (!path_.empty()) {
return path_[0];
} else {
return nullptr;
return "";
}
}
......
core/src/scheduler/task/SearchTask.cpp
浏览文件 @ 4dee7dfa
......@@ -15,10 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/task/SearchTask.h"
#include <src/scheduler/SchedInst.h>
#include <fiu-local.h>
#include <algorithm>
#include <memory>
#include <string>
......@@ -27,7 +24,9 @@
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/SchedInst.h"
#include "scheduler/job/SearchJob.h"
#include "scheduler/task/SearchTask.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
......@@ -123,6 +122,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
std::string type_str;
try {
fiu_do_on("XSearchTask.Load.throw_std_exception", throw std::exception());
if (type == LoadType::DISK2CPU) {
stat = index_engine_->Load();
type_str = "DISK2CPU";
......@@ -145,6 +145,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
error_msg = "Failed to load index file: " + std::string(ex.what());
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
fiu_do_on("XSearchTask.Load.out_of_memory", stat = Status(SERVER_UNEXPECTED_ERROR, "out of memory"));
if (!stat.ok()) {
Status s;
......@@ -217,6 +218,7 @@ XSearchTask::Execute() {
"job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk);
try {
fiu_do_on("XSearchTask.Execute.throw_std_exception", throw std::exception());
// step 2: search
bool hybrid = false;
if (index_engine_->IndexEngineType() == engine::EngineType::FAISS_IVFSQ8H &&
......@@ -231,6 +233,8 @@ XSearchTask::Execute() {
s = index_engine_->Search(nq, vectors.binary_data_.data(), topk, nprobe, output_distance.data(),
output_ids.data(), hybrid);
}
fiu_do_on("XSearchTask.Execute.search_fail", s = Status(SERVER_UNEXPECTED_ERROR, ""));
if (!s.ok()) {
search_job->GetStatus() = s;
search_job->SearchDone(index_id_);
......
core/src/server/Config.cpp
浏览文件 @ 4dee7dfa
......@@ -32,6 +32,8 @@
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
namespace milvus {
namespace server {
......@@ -57,15 +59,10 @@ Config::LoadConfigFile(const std::string& filename) {
return Status(SERVER_FILE_NOT_FOUND, str);
}
try {
ConfigMgr* mgr = YamlConfigMgr::GetInstance();
Status s = mgr->LoadConfigFile(filename);
if (!s.ok()) {
return s;
}
} catch (YAML::Exception& e) {
std::string str = "Exception occurs when loading config file: " + filename;
return Status(SERVER_UNEXPECTED_ERROR, str);
ConfigMgr* mgr = YamlConfigMgr::GetInstance();
Status s = mgr->LoadConfigFile(filename);
if (!s.ok()) {
return s;
}
return Status::OK();
......@@ -349,7 +346,9 @@ Config::ProcessConfigCli(std::string& result, const std::string& cmd) {
////////////////////////////////////////////////////////////////////////////////
Status
Config::CheckConfigVersion(const std::string& value) {
if (milvus_config_version_map.at(MILVUS_VERSION) != value) {
bool exist_error = milvus_config_version_map.at(MILVUS_VERSION) != value;
fiu_do_on("check_config_version_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid config version: " + value +
". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION);
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -360,7 +359,10 @@ Config::CheckConfigVersion(const std::string& value) {
/* server config */
Status
Config::CheckServerConfigAddress(const std::string& value) {
if (!ValidationUtil::ValidateIpAddress(value).ok()) {
auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok();
fiu_do_on("check_config_address_fail", exist_error = true);
if (exist_error) {
std::string msg =
"Invalid server IP address: " + value + ". Possible reason: server_config.address is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -370,7 +372,10 @@ Config::CheckServerConfigAddress(const std::string& value) {
Status
Config::CheckServerConfigPort(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_port_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid server port: " + value + ". Possible reason: server_config.port is not a number.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
......@@ -386,6 +391,10 @@ Config::CheckServerConfigPort(const std::string& value) {
Status
Config::CheckServerConfigDeployMode(const std::string& value) {
fiu_return_on("check_config_deploy_mode_fail",
Status(SERVER_INVALID_ARGUMENT,
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."));
if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") {
return Status(SERVER_INVALID_ARGUMENT,
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.");
......@@ -395,6 +404,9 @@ Config::CheckServerConfigDeployMode(const std::string& value) {
Status
Config::CheckServerConfigTimeZone(const std::string& value) {
fiu_return_on("check_config_time_zone_fail",
Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value));
if (value.length() <= 3) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value);
} else {
......@@ -431,7 +443,10 @@ Config::CheckServerConfigWebPort(const std::string& value) {
/* DB config */
Status
Config::CheckDBConfigBackendUrl(const std::string& value) {
if (!ValidationUtil::ValidateDbURI(value).ok()) {
auto exist_error = !ValidationUtil::ValidateDbURI(value).ok();
fiu_do_on("check_config_backend_url_fail", exist_error = true);
if (exist_error) {
std::string msg =
"Invalid backend url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " +
"The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus.";
......@@ -442,7 +457,10 @@ Config::CheckDBConfigBackendUrl(const std::string& value) {
Status
Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_archive_disk_threshold_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid archive disk threshold: " + value +
". Possible reason: db_config.archive_disk_threshold is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -452,7 +470,10 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
Status
Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_archive_days_threshold_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid archive days threshold: " + value +
". Possible reason: db_config.archive_days_threshold is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -463,6 +484,7 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
/* storage config */
Status
Config::CheckStorageConfigPrimaryPath(const std::string& value) {
fiu_return_on("check_config_primary_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.db_path is empty.");
}
......@@ -471,6 +493,7 @@ Config::CheckStorageConfigPrimaryPath(const std::string& value) {
Status
Config::CheckStorageConfigSecondaryPath(const std::string& value) {
fiu_return_on("check_config_secondary_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));
return Status::OK();
}
......@@ -536,7 +559,10 @@ Config::CheckStorageConfigS3Bucket(const std::string& value) {
/* metric config */
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_config_enable_monitor_fail", exist_error = true);
if (exist_error) {
std::string msg =
"Invalid metric config: " + value + ". Possible reason: metric_config.enable_monitor is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -572,6 +598,8 @@ Config::CheckMetricConfigPort(const std::string& value) {
/* cache config */
Status
Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
fiu_return_on("check_config_cpu_cache_capacity_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid cpu cache capacity: " + value +
". Possible reason: cache_config.cpu_cache_capacity is not a positive integer.";
......@@ -598,6 +626,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
CONFIG_CHECK(GetCacheConfigInsertBufferSize(buffer_value));
int64_t insert_buffer_size = buffer_value * GB;
fiu_do_on("Config.CheckCacheConfigCpuCacheCapacity.large_insert_buffer", insert_buffer_size = total_mem + 1);
if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
std::string msg = "Invalid cpu cache capacity: " + value +
". Possible reason: sum of cache_config.cpu_cache_capacity and "
......@@ -610,6 +639,8 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
Status
Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
fiu_return_on("check_config_cpu_cache_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid cpu cache threshold: " + value +
". Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0].";
......@@ -627,6 +658,7 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
Status
Config::CheckCacheConfigInsertBufferSize(const std::string& value) {
fiu_return_on("check_config_insert_buffer_size_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid insert buffer size: " + value +
". Possible reason: cache_config.insert_buffer_size is not a positive integer.";
......@@ -652,6 +684,8 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) {
Status
Config::CheckCacheConfigCacheInsertData(const std::string& value) {
fiu_return_on("check_config_cache_insert_data_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg = "Invalid cache insert data option: " + value +
". Possible reason: cache_config.cache_insert_data is not a boolean.";
......@@ -663,6 +697,8 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) {
/* engine config */
Status
Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
fiu_return_on("check_config_use_blas_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid use blas threshold: " + value +
". Possible reason: engine_config.use_blas_threshold is not a positive integer.";
......@@ -673,6 +709,8 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
Status
Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
fiu_return_on("check_config_omp_thread_num_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid omp thread num: " + value +
". Possible reason: engine_config.omp_thread_num is not a positive integer.";
......@@ -694,6 +732,8 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
Status
Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) {
fiu_return_on("check_config_gpu_search_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu search threshold: " + value +
". Possible reason: engine_config.gpu_search_threshold is not a positive integer.";
......@@ -705,6 +745,8 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) {
/* gpu resource config */
Status
Config::CheckGpuResourceConfigEnable(const std::string& value) {
fiu_return_on("check_config_gpu_resource_enable_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg =
"Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean.";
......@@ -715,6 +757,8 @@ Config::CheckGpuResourceConfigEnable(const std::string& value) {
Status
Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
fiu_return_on("check_gpu_resource_config_cache_capacity_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu cache capacity: " + value +
". Possible reason: gpu_resource_config.cache_capacity is not a positive integer.";
......@@ -743,6 +787,8 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
Status
Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) {
fiu_return_on("check_config_gpu_resource_cache_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid gpu cache threshold: " + value +
". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0].";
......@@ -785,6 +831,8 @@ CheckGpuResource(const std::string& value) {
Status
Config::CheckGpuResourceConfigSearchResources(const std::vector<std::string>& value) {
fiu_return_on("check_gpu_resource_config_search_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
std::string msg =
"Invalid gpu search resource. "
......@@ -810,6 +858,8 @@ Config::CheckGpuResourceConfigSearchResources(const std::vector<std::string>& va
Status
Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>& value) {
fiu_return_on("check_gpu_resource_config_build_index_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
std::string msg =
"Invalid gpu build index resource. "
......@@ -852,10 +902,7 @@ Config::ConfigNodeValid(const std::string& parent_key, const std::string& child_
if (config_map_.find(parent_key) == config_map_.end()) {
return false;
}
if (config_map_[parent_key].count(child_key) == 0) {
return false;
}
return true;
return config_map_[parent_key].count(child_key) != 0;
}
Status
......@@ -1111,10 +1158,12 @@ Config::GetEngineConfigGpuSearchThreshold(int64_t& value) {
value = std::stoll(str);
return Status::OK();
}
#endif
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
Status
Config::GetGpuResourceConfigEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT);
......@@ -1128,6 +1177,7 @@ Status
Config::GetGpuResourceConfigCacheCapacity(int64_t& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("Config.GetGpuResourceConfigCacheCapacity.diable_gpu_resource", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
......@@ -1143,6 +1193,7 @@ Status
Config::GetGpuResourceConfigCacheThreshold(float& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
......@@ -1158,6 +1209,7 @@ Status
Config::GetGpuResourceConfigSearchResources(std::vector<int64_t>& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("get_gpu_config_search_resources.disable_gpu_resource_fail", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
......@@ -1177,6 +1229,7 @@ Status
Config::GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("get_gpu_config_build_index_resources.disable_gpu_resource_fail", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
......@@ -1199,6 +1252,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value) {
Status
Config::GetTracingConfigJsonConfigPath(std::string& value) {
value = GetConfigStr(CONFIG_TRACING, CONFIG_TRACING_JSON_CONFIG_PATH, "");
fiu_do_on("get_config_json_config_path_fail", value = "error_config_json_path");
if (!value.empty()) {
std::ifstream tracer_config(value);
Status s = tracer_config.good() ? Status::OK()
......@@ -1374,10 +1428,12 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(value));
return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
}
#endif
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
Status
Config::SetGpuResourceConfigEnable(const std::string& value) {
CONFIG_CHECK(CheckGpuResourceConfigEnable(value));
......@@ -1411,6 +1467,7 @@ Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) {
CONFIG_CHECK(CheckGpuResourceConfigBuildIndexResources(res_vec));
return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value);
}
#endif
} // namespace server
......
core/src/server/delivery/RequestScheduler.cpp
浏览文件 @ 4dee7dfa
......@@ -18,6 +18,8 @@
#include "server/delivery/RequestScheduler.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <unistd.h>
#include <utility>
namespace milvus {
......@@ -74,7 +76,6 @@ RequestScheduler::Stop() {
for (auto& iter : execute_threads_) {
if (iter == nullptr)
continue;
iter->join();
}
request_groups_.clear();
......@@ -90,6 +91,8 @@ RequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) {
}
auto status = PutToQueue(request_ptr);
fiu_do_on("RequestScheduler.ExecuteRequest.push_queue_fail", status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!status.ok()) {
SERVER_LOG_ERROR << "Put request to queue failed with code: " << status.ToString();
return status;
......@@ -98,7 +101,6 @@ RequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) {
if (request_ptr->IsAsync()) {
return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere
}
return request_ptr->WaitToFinish(); // sync execution
}
......@@ -116,7 +118,10 @@ RequestScheduler::TakeToExecute(RequestQueuePtr request_queue) {
}
try {
fiu_do_on("RequestScheduler.TakeToExecute.throw_std_exception1", throw std::exception());
auto status = request->Execute();
fiu_do_on("RequestScheduler.TakeToExecute.throw_std_exception", throw std::exception());
fiu_do_on("RequestScheduler.TakeToExecute.execute_fail", status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!status.ok()) {
SERVER_LOG_ERROR << "Request failed with code: " << status.ToString();
}
......@@ -137,9 +142,12 @@ RequestScheduler::PutToQueue(const BaseRequestPtr& request_ptr) {
RequestQueuePtr queue = std::make_shared<RequestQueue>();
queue->Put(request_ptr);
request_groups_.insert(std::make_pair(group_name, queue));
fiu_do_on("RequestScheduler.PutToQueue.null_queue", queue = nullptr);
// start a thread
ThreadPtr thread = std::make_shared<std::thread>(&RequestScheduler::TakeToExecute, this, queue);
fiu_do_on("RequestScheduler.PutToQueue.push_null_thread", execute_threads_.push_back(nullptr));
execute_threads_.push_back(thread);
SERVER_LOG_INFO << "Create new thread for request group: " << group_name;
}
......
core/src/server/delivery/request/CountTableRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -22,6 +22,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -52,8 +53,11 @@ CountTableRequest::OnExecute() {
// step 2: get row count
uint64_t row_count = 0;
status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count);
fiu_do_on("CountTableRequest.OnExecute.db_not_found", status = Status(DB_NOT_FOUND, ""));
fiu_do_on("CountTableRequest.OnExecute.status_error", status = Status(SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CountTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code(), DB_NOT_FOUND) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
} else {
return status;
......
core/src/server/delivery/request/CreateIndexRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -22,6 +22,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
......@@ -53,6 +54,8 @@ CreateIndexRequest::OnExecute() {
bool has_table = false;
status = DBWrapper::DB()->HasTable(table_name_, has_table);
fiu_do_on("CreateIndexRequest.OnExecute.not_has_table", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreateIndexRequest.OnExecute.throw_std.exception", throw std::exception());
if (!status.ok()) {
return status;
}
......@@ -92,6 +95,9 @@ CreateIndexRequest::OnExecute() {
bool enable_gpu = false;
server::Config& config = server::Config::GetInstance();
s = config.GetGpuResourceConfigEnable(enable_gpu);
fiu_do_on("CreateIndexRequest.OnExecute.ip_meteric",
table_info.metric_type_ = static_cast<int>(engine::MetricType::IP));
if (s.ok() && adapter_index_type == (int)engine::EngineType::FAISS_PQ &&
table_info.metric_type_ == (int)engine::MetricType::IP) {
return Status(SERVER_UNEXPECTED_ERROR, "PQ not support IP in GPU version!");
......@@ -103,6 +109,8 @@ CreateIndexRequest::OnExecute() {
index.engine_type_ = adapter_index_type;
index.nlist_ = nlist_;
status = DBWrapper::DB()->CreateIndex(table_name_, index);
fiu_do_on("CreateIndexRequest.OnExecute.create_index_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/CreatePartitionRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
......@@ -47,22 +48,32 @@ CreatePartitionRequest::OnExecute() {
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionName(partition_name_);
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_partition_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionTags({tag_});
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
// step 2: create partition
status = DBWrapper::DB()->CreatePartition(table_name_, partition_name_, tag_);
fiu_do_on("CreatePartitionRequest.OnExecute.db_already_exist", status = Status(milvus::DB_ALREADY_EXIST, ""));
fiu_do_on("CreatePartitionRequest.OnExecute.create_partition_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreatePartitionRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
// partition could exist
if (status.code() == DB_ALREADY_EXIST) {
......
core/src/server/delivery/request/CreateTableRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -22,6 +22,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
......@@ -62,6 +63,8 @@ CreateTableRequest::OnExecute() {
}
status = ValidationUtil::ValidateTableIndexFileSize(index_file_size_);
fiu_do_on("CreateTableRequest.OnExecute.invalid_index_file_size",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -89,6 +92,10 @@ CreateTableRequest::OnExecute() {
// step 3: create table
status = DBWrapper::DB()->CreateTable(table_info);
fiu_do_on("CreateTableRequest.OnExecute.db_already_exist", status = Status(milvus::DB_ALREADY_EXIST, ""));
fiu_do_on("CreateTableRequest.OnExecute.create_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreateTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
// table could exist
if (status.code() == DB_ALREADY_EXIST) {
......
core/src/server/delivery/request/DeleteByDateRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <vector>
......@@ -54,8 +55,13 @@ DeleteByDateRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DeleteByDateRequest.OnExecute.db_not_found", status = Status(milvus::DB_NOT_FOUND, ""));
fiu_do_on("DeleteByDateRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("DeleteByDateRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code(), DB_NOT_FOUND) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
} else {
return status;
......@@ -80,6 +86,8 @@ DeleteByDateRequest::OnExecute() {
ProfilerStart(fname.c_str());
#endif
status = DBWrapper::DB()->DropTable(table_name_, dates);
fiu_do_on("DeleteByDateRequest.OnExecute.drop_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DescribeIndexRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -40,6 +41,7 @@ DescribeIndexRequest::Create(const std::shared_ptr<Context>& context, const std:
Status
DescribeIndexRequest::OnExecute() {
try {
fiu_do_on("DescribeIndexRequest.OnExecute.throw_std_exception", throw std::exception());
std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")";
TimeRecorderAuto rc(hdr);
......
core/src/server/delivery/request/DescribeTableRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -53,6 +54,9 @@ DescribeTableRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DescribeTableRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("DescribeTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DropIndexRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -38,6 +39,7 @@ DropIndexRequest::Create(const std::shared_ptr<Context>& context, const std::str
Status
DropIndexRequest::OnExecute() {
try {
fiu_do_on("DropIndexRequest.OnExecute.throw_std_exception", throw std::exception());
std::string hdr = "DropIndexRequest(table=" + table_name_ + ")";
TimeRecorderAuto rc(hdr);
......@@ -49,6 +51,7 @@ DropIndexRequest::OnExecute() {
bool has_table = false;
status = DBWrapper::DB()->HasTable(table_name_, has_table);
fiu_do_on("DropIndexRequest.OnExecute.table_not_exist", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -59,6 +62,7 @@ DropIndexRequest::OnExecute() {
// step 2: check table existence
status = DBWrapper::DB()->DropIndex(table_name_);
fiu_do_on("DropIndexRequest.OnExecute.drop_index_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DropPartitionRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
......@@ -60,6 +61,8 @@ DropPartitionRequest::OnExecute() {
if (!partition_name.empty()) {
status = ValidationUtil::ValidateTableName(partition_name);
fiu_do_on("DropPartitionRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -68,6 +71,8 @@ DropPartitionRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = partition_name;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DropPartitionRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST,
......@@ -80,11 +85,15 @@ DropPartitionRequest::OnExecute() {
return DBWrapper::DB()->DropPartition(partition_name);
} else {
status = ValidationUtil::ValidateTableName(table_name);
fiu_do_on("DropPartitionRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionTags({partition_tag});
fiu_do_on("DropPartitionRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DropTableRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <vector>
......@@ -52,6 +53,10 @@ DropTableRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DropTableRequest.OnExecute.db_not_found", status = Status(milvus::DB_NOT_FOUND, ""));
fiu_do_on("DropTableRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("DropTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
......@@ -65,6 +70,7 @@ DropTableRequest::OnExecute() {
// step 3: Drop table
std::vector<DB_DATE> dates;
status = DBWrapper::DB()->DropTable(table_name_, dates);
fiu_do_on("DropTableRequest.OnExecute.drop_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/HasTableRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -50,6 +51,8 @@ HasTableRequest::OnExecute() {
// step 2: check table existence
status = DBWrapper::DB()->HasTable(table_name_, has_table_);
fiu_do_on("HasTableRequest.OnExecute.table_not_exist", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("HasTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/InsertRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <vector>
......@@ -46,6 +47,7 @@ Status
InsertRequest::OnExecute() {
try {
int64_t vector_count = vectors_data_.vector_count_;
fiu_do_on("InsertRequest.OnExecute.throw_std_exception", throw std::exception());
std::string hdr = "InsertRequest(table=" + table_name_ + ", n=" + std::to_string(vector_count) +
", partition_tag=" + partition_tag_ + ")";
TimeRecorder rc(hdr);
......@@ -60,6 +62,7 @@ InsertRequest::OnExecute() {
"The vector array is empty. Make sure you have entered vector records.");
}
fiu_do_on("InsertRequest.OnExecute.id_array_error", vectors_data_.id_array_.resize(vector_count + 1));
if (!vectors_data_.id_array_.empty()) {
if (vectors_data_.id_array_.size() != vector_count) {
return Status(SERVER_ILLEGAL_VECTOR_ID,
......@@ -71,6 +74,8 @@ InsertRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("InsertRequest.OnExecute.db_not_found", status = Status(milvus::DB_NOT_FOUND, ""));
fiu_do_on("InsertRequest.OnExecute.describe_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
......@@ -82,12 +87,16 @@ InsertRequest::OnExecute() {
// step 3: check table flag
// all user provide id, or all internal id
bool user_provide_ids = !vectors_data_.id_array_.empty();
fiu_do_on("InsertRequest.OnExecute.illegal_vector_id", user_provide_ids = false;
table_info.flag_ = engine::meta::FLAG_MASK_HAS_USERID);
// user already provided id before, all insert action require user id
if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) {
return Status(SERVER_ILLEGAL_VECTOR_ID,
"Table vector IDs are user-defined. Please provide IDs for all vectors of this table.");
}
fiu_do_on("InsertRequest.OnExecute.illegal_vector_id2", user_provide_ids = true;
table_info.flag_ = engine::meta::FLAG_MASK_NO_USERID);
// user didn't provided id before, no need to provide user id
if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) {
return Status(
......@@ -114,6 +123,7 @@ InsertRequest::OnExecute() {
"The vector dimension must be equal to the table dimension.");
}
fiu_do_on("InsertRequest.OnExecute.invalid_dim", table_info.dimension_ = -1);
if (vectors_data_.float_data_.size() / vector_count != table_info.dimension_) {
return Status(SERVER_INVALID_VECTOR_DIMENSION,
"The vector dimension must be equal to the table dimension.");
......@@ -140,11 +150,13 @@ InsertRequest::OnExecute() {
rc.RecordSection("prepare vectors data");
status = DBWrapper::DB()->InsertVectors(table_name_, partition_tag_, vectors_data_);
fiu_do_on("InsertRequest.OnExecute.insert_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
auto ids_size = vectors_data_.id_array_.size();
fiu_do_on("InsertRequest.OnExecute.invalid_ids_size", ids_size = vec_count - 1);
if (ids_size != vec_count) {
std::string msg =
"Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id";
......
core/src/server/delivery/request/PreloadTableRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -49,6 +50,9 @@ PreloadTableRequest::OnExecute() {
// step 2: check table existence
status = DBWrapper::DB()->PreloadTable(table_name_);
fiu_do_on("PreloadTableRequest.OnExecute.preload_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("PreloadTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/SearchRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
......@@ -53,6 +54,7 @@ SearchRequest::Create(const std::shared_ptr<Context>& context, const std::string
Status
SearchRequest::OnExecute() {
try {
fiu_do_on("SearchRequest.OnExecute.throw_std_exception", throw std::exception());
uint64_t vector_count = vectors_data_.vector_count_;
auto pre_query_ctx = context_->Child("Pre query");
......@@ -71,6 +73,7 @@ SearchRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("SearchRequest.OnExecute.describe_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
......@@ -117,11 +120,13 @@ SearchRequest::OnExecute() {
}
} else {
// check prepared float data
fiu_do_on("SearchRequest.OnExecute.invalod_rowrecord_array",
vector_count = vectors_data_.float_data_.size() + 1);
if (vectors_data_.float_data_.size() % vector_count != 0) {
return Status(SERVER_INVALID_ROWRECORD_ARRAY,
"The vector dimension must be equal to the table dimension.");
}
fiu_do_on("SearchRequest.OnExecute.invalid_dim", table_info.dimension_ = -1);
if (vectors_data_.float_data_.size() / vector_count != table_info.dimension_) {
return Status(SERVER_INVALID_VECTOR_DIMENSION,
"The vector dimension must be equal to the table dimension.");
......@@ -144,6 +149,8 @@ SearchRequest::OnExecute() {
if (file_id_list_.empty()) {
status = ValidationUtil::ValidatePartitionTags(partition_list_);
fiu_do_on("SearchRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -160,10 +167,11 @@ SearchRequest::OnExecute() {
#endif
rc.RecordSection("search vectors from engine");
fiu_do_on("SearchRequest.OnExecute.query_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
fiu_do_on("SearchRequest.OnExecute.empty_result_ids", result_ids.clear());
if (result_ids.empty()) {
return Status::OK(); // empty table
}
......
core/src/server/delivery/request/ShowPartitionsRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <vector>
......@@ -44,6 +45,8 @@ ShowPartitionsRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
auto status = ValidationUtil::ValidateTableName(table_name_);
fiu_do_on("ShowPartitionsRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......@@ -60,6 +63,8 @@ ShowPartitionsRequest::OnExecute() {
std::vector<engine::meta::TableSchema> schema_array;
status = DBWrapper::DB()->ShowPartitions(table_name_, schema_array);
fiu_do_on("ShowPartitionsRequest.OnExecute.show_partition_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/ShowTablesRequest.cpp
浏览文件 @ 4dee7dfa
......@@ -20,6 +20,7 @@
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <vector>
......@@ -41,9 +42,10 @@ ShowTablesRequest::OnExecute() {
TimeRecorderAuto rc("ShowTablesRequest");
std::vector<engine::meta::TableSchema> schema_array;
auto statuts = DBWrapper::DB()->AllTables(schema_array);
if (!statuts.ok()) {
return statuts;
auto status = DBWrapper::DB()->AllTables(schema_array);
fiu_do_on("ShowTablesRequest.OnExecute.show_tables_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
for (auto& schema : schema_array) {
......
core/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @ 4dee7dfa
......@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <memory>
#include <unordered_map>
#include <vector>
......@@ -290,6 +290,7 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc:
// step 3: search vectors
std::vector<std::string> file_ids;
TopKQueryResult result;
fiu_do_on("GrpcRequestHandler.Search.not_empty_file_ids", file_ids.emplace_back("test_file_id"));
Status status = request_handler_.Search(context_map_[context], request->table_name(), vectors, ranges,
request->topk(), request->nprobe(), partitions, file_ids, result);
......
core/src/utils/CommonUtil.cpp
浏览文件 @ 4dee7dfa
......@@ -42,6 +42,8 @@
#define THREAD_MULTIPLY_CPU 1
#endif
#include <fiu-local.h>
namespace milvus {
namespace server {
......@@ -62,6 +64,8 @@ CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("CommonUtil.GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
......@@ -95,6 +99,7 @@ CommonUtil::CreateDirectory(const std::string& path) {
fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path();
Status err_status = CreateDirectory(parent_path.string());
fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!err_status.ok()) {
return err_status;
}
......@@ -105,6 +110,7 @@ CommonUtil::CreateDirectory(const std::string& path) {
}
int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH);
fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1);
if (makeOK != 0) {
return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path);
}
......@@ -183,6 +189,7 @@ CommonUtil::GetExePath() {
const size_t buf_len = 1024;
char buf[buf_len];
size_t cnt = readlink("/proc/self/exe", buf, buf_len);
fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1);
if (cnt < 0 || cnt >= buf_len) {
return "";
}
......@@ -190,6 +197,7 @@ CommonUtil::GetExePath() {
buf[cnt] = '\0';
std::string exe_path = buf;
fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/");
if (exe_path.rfind('/') != exe_path.length() - 1) {
std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
return sub_str + "/";
......
core/src/utils/StringHelpFunctions.cpp
浏览文件 @ 4dee7dfa
......@@ -17,6 +17,7 @@
#include "utils/StringHelpFunctions.h"
#include <fiu-local.h>
#include <regex>
#include <string>
......@@ -104,6 +105,8 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
last = index + 1;
std::string postfix = process_str.substr(last);
index = postfix.find_first_of(quote, 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos);
if (index == std::string::npos) {
return Status(SERVER_UNEXPECTED_ERROR, "");
}
......@@ -112,6 +115,9 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
last = index + 1;
index = postfix.find_first_of(delimeter, last);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos);
if (index != std::string::npos) {
if (index > last) {
append_prefix += postfix.substr(last, index - last);
......@@ -120,6 +126,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
append_prefix += postfix.substr(last);
}
result.emplace_back(append_prefix);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length());
if (last == postfix.length()) {
return Status::OK();
......
core/src/utils/ThreadPool.h
浏览文件 @ 4dee7dfa
......@@ -17,6 +17,7 @@
#pragma once
#include <fiu-local.h>
#include <condition_variable>
#include <functional>
#include <future>
......@@ -89,7 +90,7 @@ ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_o
auto task = std::make_shared<std::packaged_task<return_type()> >(
std::bind(std::forward<F>(f), std::forward<Args>(args)...));
fiu_do_on("ThreadPool.enqueue.stop_is_true", stop = true);
std::future<return_type> res = task->get_future();
{
std::unique_lock<std::mutex> lock(queue_mutex_);
......
core/src/utils/TimeRecorder.cpp
浏览文件 @ 4dee7dfa
......@@ -24,8 +24,7 @@ TimeRecorder::TimeRecorder(const std::string& header, int64_t log_level) : heade
start_ = last_ = stdclock::now();
}
TimeRecorder::~TimeRecorder() {
}
TimeRecorder::~TimeRecorder() = default;
std::string
TimeRecorder::GetTimeSpanStr(double span) {
......
core/src/utils/TimeRecorder.h
浏览文件 @ 4dee7dfa
......@@ -54,7 +54,7 @@ class TimeRecorderAuto : public TimeRecorder {
public:
explicit TimeRecorderAuto(const std::string& header, int64_t log_level = 1);
~TimeRecorderAuto();
~TimeRecorderAuto() override;
};
} // namespace milvus
core/src/utils/ValidationUtil.cpp
浏览文件 @ 4dee7dfa
......@@ -28,6 +28,7 @@
#endif
#include <fiu-local.h>
#include <algorithm>
#include <cmath>
#include <regex>
......@@ -245,6 +246,8 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("ValidationUtil.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
SERVER_LOG_ERROR << msg;
......@@ -265,6 +268,8 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) {
Status
ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) {
fiu_return_on("ValidationUtil.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
......@@ -285,6 +290,7 @@ ValidationUtil::ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("ValidationUtil.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
......@@ -309,6 +315,7 @@ ValidationUtil::ValidateStringIsNumber(const std::string& str) {
}
try {
int64_t value = std::stol(str);
fiu_do_on("ValidationUtil.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
......@@ -320,6 +327,7 @@ ValidationUtil::ValidateStringIsNumber(const std::string& str) {
Status
ValidationUtil::ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
......
core/src/wrapper/ConfAdapter.cpp
浏览文件 @ 4dee7dfa
......@@ -17,6 +17,7 @@
#include "wrapper/ConfAdapter.h"
#include <fiu-local.h>
#include <cmath>
#include <memory>
#include <vector>
......@@ -160,7 +161,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) {
}
}
}
fiu_do_on("IVFPQConfAdapter.Match.empty_resset", resset.clear());
if (resset.empty()) {
// todo(linxj): throw exception here.
WRAPPER_LOG_ERROR << "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-"
......
core/src/wrapper/KnowhereResource.cpp
浏览文件 @ 4dee7dfa
......@@ -23,6 +23,7 @@
#include "scheduler/Utils.h"
#include "server/Config.h"
#include <fiu-local.h>
#include <map>
#include <set>
#include <string>
......@@ -43,7 +44,7 @@ KnowhereResource::Initialize() {
s = config.GetGpuResourceConfigEnable(enable_gpu);
if (!s.ok())
return s;
fiu_do_on("KnowhereResource.Initialize.disable_gpu", enable_gpu = false);
if (not enable_gpu)
return Status::OK();
......
core/src/wrapper/VecImpl.cpp
浏览文件 @ 4dee7dfa
......@@ -32,6 +32,7 @@
#endif
#include <fiu-local.h>
/*
* no parameter check in this layer.
* only responsible for index combination
......@@ -46,6 +47,8 @@ VecIndexImpl::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, c
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
fiu_do_on("VecIndexImpl.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndexImpl.BuildAll.throw_std_exception", throw std::exception());
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
......@@ -66,7 +69,8 @@ Status
VecIndexImpl::Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg) {
try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
fiu_do_on("VecIndexImpl.Add.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndexImpl.Add.throw_std_exception", throw std::exception());
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
......@@ -86,6 +90,9 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
Config search_cfg = cfg;
fiu_do_on("VecIndexImpl.Search.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndexImpl.Search.throw_std_exception", throw std::exception());
auto res = index_->Search(dataset, search_cfg);
//{
// auto& ids = ids_array;
......@@ -154,7 +161,7 @@ VecIndexImpl::GetType() const {
VecIndexPtr
VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) {
// TODO(linxj): exception handle
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
......@@ -168,7 +175,7 @@ VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) {
VecIndexPtr
VecIndexImpl::CopyToCpu(const Config& cfg) {
// TODO(linxj): exception handle
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg);
auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
......@@ -217,6 +224,8 @@ BFIndex::GetRawIds() {
ErrorCode
BFIndex::Build(const Config& cfg) {
try {
fiu_do_on("BFIndex.Build.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("BFIndex.Build.throw_std_exception", throw std::exception());
dim = cfg->d;
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
} catch (knowhere::KnowhereException& e) {
......@@ -235,6 +244,8 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
fiu_do_on("BFIndex.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("BFIndex.BuildAll.throw_std_exception", throw std::exception());
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
index_->Add(dataset, cfg);
......
core/src/wrapper/VecIndex.cpp
浏览文件 @ 4dee7dfa
......@@ -49,6 +49,8 @@
#include "wrapper/gpu/GPUVecImpl.h"
#endif
#include <fiu-local.h>
namespace milvus {
namespace engine {
......@@ -142,6 +144,7 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
config.GetGpuResourceConfigEnable(gpu_resource_enable);
if (gpu_resource_enable) {
index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
fiu_do_on("GetVecIndexFactory.IVFSQHybrid.mock", index = std::make_shared<knowhere::IVF>());
return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
} else {
throw Exception(DB_ERROR, "No GPU resources for IndexType::FAISS_IVFSQ8_HYBRID");
......@@ -171,6 +174,8 @@ LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binar
VecIndexPtr
read_index(const std::string& location) {
fiu_return_on("read_null_index", nullptr);
fiu_do_on("vecIndex.throw_read_exception", throw std::exception());
TimeRecorder recorder("read_index");
knowhere::BinarySet load_data_list;
......@@ -242,6 +247,11 @@ write_index(VecIndexPtr index, const std::string& location) {
auto binaryset = index->Serialize();
auto index_type = index->GetType();
fiu_do_on("VecIndex.write_index.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndex.write_index.throw_std_exception", throw std::exception());
fiu_do_on("VecIndex.write_index.throw_no_space_exception",
throw Exception(SERVER_INVALID_ARGUMENT, "No space left on device"));
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigS3Enable(s3_enable);
......
core/src/wrapper/gpu/GPUVecImpl.cpp
浏览文件 @ 4dee7dfa
......@@ -26,6 +26,7 @@
#include "utils/Log.h"
#include "wrapper/VecImpl.h"
#include <fiu-local.h>
/*
* no parameter check in this layer.
* only responible for index combination
......@@ -39,9 +40,11 @@ Status
IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) {
try {
fiu_do_on("IVFMixIndex.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFMixIndex.BuildAll.throw_std_exception", throw std::exception());
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
auto model = index_->Train(dataset, cfg);
......@@ -81,12 +84,16 @@ IVFHybridIndex::LoadQuantizer(const Config& conf) {
return new_idx->LoadQuantizer(conf);
} else {
WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type);
return nullptr;
}
}
Status
IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) {
try {
fiu_do_on("IVFHybridIndex.SetQuantizer.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.SetQuantizer.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
new_idx->SetQuantizer(q);
......@@ -107,6 +114,9 @@ IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) {
Status
IVFHybridIndex::UnsetQuantizer() {
try {
fiu_do_on("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.UnsetQuantizer.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
new_idx->UnsetQuantizer();
......@@ -127,6 +137,9 @@ IVFHybridIndex::UnsetQuantizer() {
VecIndexPtr
IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
try {
fiu_do_on("IVFHybridIndex.LoadData.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.LoadData.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
return std::make_shared<IVFHybridIndex>(new_idx->LoadData(q, conf), type);
......@@ -144,6 +157,10 @@ IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
std::pair<VecIndexPtr, knowhere::QuantizerPtr>
IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) {
try {
fiu_do_on("IVFHybridIndex.CopyToGpuWithQuantizer.throw_knowhere_exception",
throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.CopyToGpuWithQuantizer.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto hybrid_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg);
......
core/unittest/db/test_db.cpp
浏览文件 @ 4dee7dfa
......@@ -16,6 +16,8 @@
// under the License.
#include <gtest/gtest.h>
#include <fiu-control.h>
#include <fiu-local.h>
#include <boost/filesystem.hpp>
#include <random>
......@@ -40,7 +42,6 @@ static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;
milvus::engine::meta::TableSchema
BuildTableSchema() {
milvus::engine::meta::TableSchema table_info;
......@@ -125,6 +126,18 @@ TEST_F(DBTest, CONFIG_TEST) {
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 0);
}
{
fiu_init(0);
fiu_enable("ArchiveConf.ParseCritirias.OptionsParseCritiriasOutOfRange", 1, NULL, 0);
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf("swap", "disk:"));
fiu_disable("ArchiveConf.ParseCritirias.OptionsParseCritiriasOutOfRange");
}
{
fiu_enable("ArchiveConf.ParseCritirias.empty_tokens", 1, NULL, 0);
milvus::engine::ArchiveConf conf("swap", "");
ASSERT_TRUE(conf.GetCriterias().empty());
fiu_disable("ArchiveConf.ParseCritirias.empty_tokens");
}
{
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "disk:"));
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "disk:a"));
......@@ -228,6 +241,19 @@ TEST_F(DBTest, DB_TEST) {
stat = db_->GetTableRowCount(TABLE_NAME, count);
ASSERT_TRUE(stat.ok());
ASSERT_GT(count, 0);
// test invalid build db
{
auto options = GetOptions();
options.meta_.backend_uri_ = "dummy";
ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
options.meta_.backend_uri_ = "mysql://root:123456@127.0.0.1:3306/test";
ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
options.meta_.backend_uri_ = "dummy://root:123456@127.0.0.1:3306/test";
ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
}
}
TEST_F(DBTest, SEARCH_TEST) {
......@@ -341,6 +367,18 @@ TEST_F(DBTest, SEARCH_TEST) {
stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, dates, result_ids,
result_distances);
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, dates, result_ids,
result_distances);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
FIU_ENABLE_FIU("DBImpl.QueryByFileID.empty_files_array");
stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, dates, result_ids,
result_distances);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.QueryByFileID.empty_files_array");
}
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
......@@ -352,6 +390,13 @@ TEST_F(DBTest, SEARCH_TEST) {
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances);
ASSERT_TRUE(stat.ok());
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances);
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
}
#ifdef CUSTOMIZATION
......@@ -389,6 +434,8 @@ TEST_F(DBTest, SEARCH_TEST) {
}
TEST_F(DBTest, PRELOADTABLE_TEST) {
fiu_init(0);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
......@@ -418,6 +465,32 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
ASSERT_TRUE(stat.ok());
int64_t cur_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage();
ASSERT_TRUE(prev_cache_usage < cur_cache_usage);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
//create a partition
stat = db_->CreatePartition(TABLE_NAME, "part0", "0");
ASSERT_TRUE(stat.ok());
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("DBImpl.PreloadTable.null_engine");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.PreloadTable.null_engine");
FIU_ENABLE_FIU("DBImpl.PreloadTable.exceed_cache");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.PreloadTable.exceed_cache");
FIU_ENABLE_FIU("DBImpl.PreloadTable.engine_throw_exception");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.PreloadTable.engine_throw_exception");
}
TEST_F(DBTest, SHUTDOWN_TEST) {
......@@ -430,6 +503,26 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->DescribeTable(table_info);
ASSERT_FALSE(stat.ok());
stat = db_->UpdateTableFlag(TABLE_NAME, 0);
ASSERT_FALSE(stat.ok());
stat = db_->CreatePartition(TABLE_NAME, "part0", "0");
ASSERT_FALSE(stat.ok());
stat = db_->DropPartition("part0");
ASSERT_FALSE(stat.ok());
stat = db_->DropPartitionByTag(TABLE_NAME, "0");
ASSERT_FALSE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> partition_schema_array;
stat = db_->ShowPartitions(TABLE_NAME, partition_schema_array);
ASSERT_FALSE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> table_infos;
stat = db_->AllTables(table_infos);
ASSERT_EQ(stat.code(), milvus::DB_ERROR);
bool has_table = false;
stat = db_->HasTable(table_info.table_id_, has_table);
ASSERT_FALSE(stat.ok());
......@@ -452,6 +545,9 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->DescribeIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
stat = db_->DropIndex(TABLE_NAME);
ASSERT_FALSE(stat.ok());
std::vector<std::string> tags;
milvus::engine::meta::DatesT dates;
milvus::engine::ResultIds result_ids;
......@@ -464,10 +560,127 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
result_distances);
ASSERT_FALSE(stat.ok());
stat = db_->Query(dummy_context_, table_info.table_id_, tags, 1, 1,
milvus::engine::VectorsData(), result_ids, result_distances);
ASSERT_FALSE(stat.ok());
stat = db_->DropTable(table_info.table_id_, dates);
ASSERT_FALSE(stat.ok());
}
TEST_F(DBTest, BACK_TIMER_THREAD_1) {
fiu_init(0);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::Status stat;
//test background timer thread
{
FIU_ENABLE_FIU("DBImpl.StartMetricTask.InvalidTotalCache");
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToMerge.throw_exception");
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.StartMetricTask.InvalidTotalCache");
fiu_disable("SqliteMetaImpl.FilesToMerge.throw_exception");
}
FIU_ENABLE_FIU("DBImpl.StartMetricTask.InvalidTotalCache");
db_->Start();
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.StartMetricTask.InvalidTotalCache");
}
TEST_F(DBTest, BACK_TIMER_THREAD_2) {
fiu_init(0);
milvus::Status stat;
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTableFile.throw_exception");
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("SqliteMetaImpl.CreateTableFile.throw_exception");
}
TEST_F(DBTest, BACK_TIMER_THREAD_3) {
fiu_init(0);
milvus::Status stat;
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
FIU_ENABLE_FIU("DBImpl.MergeFiles.Serialize_ThrowException");
db_->Start();
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.MergeFiles.Serialize_ThrowException");
}
TEST_F(DBTest, BACK_TIMER_THREAD_4) {
fiu_init(0);
milvus::Status stat;
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
FIU_ENABLE_FIU("DBImpl.MergeFiles.Serialize_ErrorStatus");
db_->Start();
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.MergeFiles.Serialize_ErrorStatus");
}
TEST_F(DBTest, INDEX_TEST) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
......@@ -489,6 +702,18 @@ TEST_F(DBTest, INDEX_TEST) {
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
fiu_init(0);
FIU_ENABLE_FIU("SqliteMetaImpl.DescribeTableIndex.throw_exception");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.DescribeTableIndex.throw_exception");
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
FIU_ENABLE_FIU("DBImpl.UpdateTableIndexRecursively.fail_update_table_index");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.UpdateTableIndexRecursively.fail_update_table_index");
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
......@@ -542,6 +767,10 @@ TEST_F(DBTest, PARTITION_TEST) {
db_->InsertVectors(table_name, partition_tag, xb);
ASSERT_EQ(vector_ids.size(), INSERT_BATCH);
//insert data into not existed partition
stat = db_->InsertVectors(TABLE_NAME, "notexist", xb);
ASSERT_FALSE(stat.ok());
}
// duplicated partition is not allowed
......@@ -563,10 +792,31 @@ TEST_F(DBTest, PARTITION_TEST) {
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
fiu_init(0);
FIU_ENABLE_FIU("DBImpl.BuildTableIndexRecursively.fail_build_table_Index_for_partition");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.BuildTableIndexRecursively.fail_build_table_Index_for_partition");
FIU_ENABLE_FIU("DBImpl.BuildTableIndexRecursively.not_empty_err_msg");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.BuildTableIndexRecursively.not_empty_err_msg");
uint64_t row_count = 0;
stat = db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, INSERT_BATCH * PARTITION_COUNT);
FIU_ENABLE_FIU("SqliteMetaImpl.Count.throw_exception");
stat = db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.Count.throw_exception");
FIU_ENABLE_FIU("DBImpl.GetTableRowCountRecursively.fail_get_table_rowcount_for_partition");
stat = db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.GetTableRowCountRecursively.fail_get_table_rowcount_for_partition");
}
{ // search
......@@ -607,6 +857,16 @@ TEST_F(DBTest, PARTITION_TEST) {
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
stat = db_->DropIndex(table_info.table_id_);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
FIU_ENABLE_FIU("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
stat = db_->DropIndex(table_info.table_id_);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
stat = db_->DropIndex(table_name);
ASSERT_TRUE(stat.ok());
......@@ -684,6 +944,18 @@ TEST_F(DBTest2, DELETE_TEST) {
stat = db_->CreateIndex(TABLE_NAME, index);
std::vector<milvus::engine::meta::DateT> dates;
//create partition, drop table will drop partition recursively
stat = db_->CreatePartition(TABLE_NAME, "part0", "0");
ASSERT_TRUE(stat.ok());
//fail drop table
fiu_init(0);
FIU_ENABLE_FIU("DBImpl.DropTableRecursively.failed");
stat = db_->DropTable(TABLE_NAME, dates);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.DropTableRecursively.failed");
stat = db_->DropTable(TABLE_NAME, dates);
std::this_thread::sleep_for(std::chrono::seconds(2));
ASSERT_TRUE(stat.ok());
......
core/unittest/db/test_db_mysql.cpp
浏览文件 @ 4dee7dfa
......@@ -16,7 +16,8 @@
// under the License.
#include <gtest/gtest.h>
#include <fiu-control.h>
#include <fiu-local.h>
#include <boost/filesystem.hpp>
#include <random>
#include <thread>
......@@ -204,6 +205,17 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
}
ASSERT_TRUE(bfound);
fiu_init(0);
FIU_ENABLE_FIU("MySQLMetaImpl.AllTable.null_connection");
stat = db_->AllTables(table_schema_array);
ASSERT_FALSE(stat.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.AllTable.throw_exception");
stat = db_->AllTables(table_schema_array);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.AllTable.null_connection");
fiu_disable("MySQLMetaImpl.AllTable.throw_exception");
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
......@@ -231,6 +243,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * milvus::engine::G);
FIU_ENABLE_FIU("MySQLMetaImpl.Size.null_connection");
stat = db_->Size(size);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.Size.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.Size.throw_exception");
stat = db_->Size(size);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.Size.throw_exception");
}
TEST_F(MySqlDBTest, DELETE_TEST) {
......@@ -288,6 +309,12 @@ TEST_F(MySqlDBTest, PARTITION_TEST) {
stat = db_->CreatePartition(table_name, partition_name, partition_tag);
ASSERT_TRUE(stat.ok());
fiu_init(0);
FIU_ENABLE_FIU("MySQLMetaImpl.CreatePartition.aleady_exist");
stat = db_->CreatePartition(table_name, partition_name, partition_tag);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.CreatePartition.aleady_exist");
// not allow nested partition
stat = db_->CreatePartition(partition_name, "dumy", "dummy");
ASSERT_FALSE(stat.ok());
......@@ -366,16 +393,84 @@ TEST_F(MySqlDBTest, PARTITION_TEST) {
ASSERT_EQ(result_ids.size() / topk, nq);
}
stat = db_->DropPartition(table_name + "_0");
ASSERT_TRUE(stat.ok());
fiu_init(0);
{
//create partition with dummy name
stat = db_->CreatePartition(table_name, "", "6");
ASSERT_TRUE(stat.ok());
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_TRUE(stat.ok());
// ensure DescribeTable failed
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.throw_exception");
stat = db_->CreatePartition(table_name, "", "7");
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.throw_exception");
stat = db_->DropIndex(table_name);
ASSERT_TRUE(stat.ok());
//Drop partition will failed,since it firstly drop partition meta table.
FIU_ENABLE_FIU("MySQLMetaImpl.DropTable.null_connection");
stat = db_->DropPartition(table_name + "_5");
//TODO(sjh): add assert expr, since DropPartion always return Status::OK() for now.
//ASSERT_TRUE(stat.ok());
fiu_disable("MySQLMetaImpl.DropTable.null_connection");
milvus::engine::meta::DatesT dates;
stat = db_->DropTable(table_name, dates);
ASSERT_TRUE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> partition_schema_array;
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(partition_schema_array.size(), PARTITION_COUNT + 1);
FIU_ENABLE_FIU("MySQLMetaImpl.ShowPartitions.null_connection");
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_FALSE(stat.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.ShowPartitions.throw_exception");
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_FALSE(stat.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.DropTable.throw_exception");
stat = db_->DropPartition(table_name + "_4");
fiu_disable("MySQLMetaImpl.DropTable.throw_exception");
stat = db_->DropPartition(table_name + "_0");
ASSERT_TRUE(stat.ok());
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.GetPartitionName.null_connection");
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.GetPartitionName.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.GetPartitionName.throw_exception");
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.GetPartitionName.throw_exception");
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_TRUE(stat.ok());
stat = db_->CreatePartition(table_name, table_name + "_1", "1");
FIU_ENABLE_FIU("MySQLMetaImpl.DeleteTableFiles.null_connection");
stat = db_->DropPartition(table_name + "_1");
fiu_disable("MySQLMetaImpl.DeleteTableFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DeleteTableFiles.throw_exception");
stat = db_->DropPartition(table_name + "_1");
fiu_disable("MySQLMetaImpl.DeleteTableFiles.throw_exception");
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.DropTableIndex.null_connection");
stat = db_->DropIndex(table_name);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DropTableIndex.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DropTableIndex.throw_exception");
stat = db_->DropIndex(table_name);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DropTableIndex.throw_exception");
stat = db_->DropIndex(table_name);
ASSERT_TRUE(stat.ok());
}
}
core/unittest/db/test_engine.cpp
浏览文件 @ 4dee7dfa
......@@ -22,6 +22,8 @@
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "db/utils.h"
#include <fiu-local.h>
#include <fiu-control.h>
TEST_F(EngineTest, FACTORY_TEST) {
{
......@@ -62,29 +64,49 @@ TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
fiu_init(0);
// test ExecutionEngineImpl constructor when create vecindex failed
FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.invalid_type");
ASSERT_ANY_THROW(milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024));
fiu_disable("ExecutionEngineImpl.CreatetVecIndex.invalid_type");
}
{
// test ExecutionEngineImpl constructor when build BFindex failed
FIU_ENABLE_FIU("BFIndex.Build.throw_knowhere_exception");
ASSERT_ANY_THROW(milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024));
fiu_disable("BFIndex.Build.throw_knowhere_exception");
}
}
TEST_F(EngineTest, ENGINE_IMPL_TEST) {
fiu_init(0);
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = milvus::engine::EngineFactory::Build(
......@@ -108,18 +130,122 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
ASSERT_EQ(engine_ptr->Dimension(), dimension);
ASSERT_EQ(engine_ptr->Count(), ids.size());
ASSERT_EQ(engine_ptr->GetLocation(), file_path);
ASSERT_EQ(engine_ptr->IndexMetricType(), milvus::engine::MetricType::IP);
ASSERT_ANY_THROW(engine_ptr->BuildIndex(file_path, milvus::engine::EngineType::INVALID));
FIU_ENABLE_FIU("VecIndexImpl.BuildAll.throw_knowhere_exception");
ASSERT_ANY_THROW(engine_ptr->BuildIndex(file_path, milvus::engine::EngineType::SPTAG_KDT));
fiu_disable("VecIndexImpl.BuildAll.throw_knowhere_exception");
auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
#ifndef MILVUS_GPU_VERSION
//PQ don't support IP In gpu version
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ);
#endif
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT);
engine_ptr->BuildIndex("/tmp/milvus_index_SPTAG_BKT", milvus::engine::EngineType::SPTAG_BKT);
#ifdef MILVUS_GPU_VERSION
FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled");
engine_ptr->BuildIndex("/tmp/milvus_index_NSG_MIX", milvus::engine::EngineType::NSG_MIX);
engine_ptr->BuildIndex("/tmp/milvus_index_6", milvus::engine::EngineType::FAISS_IVFFLAT);
engine_ptr->BuildIndex("/tmp/milvus_index_7", milvus::engine::EngineType::FAISS_IVFSQ8);
ASSERT_ANY_THROW(engine_ptr->BuildIndex("/tmp/milvus_index_8", milvus::engine::EngineType::FAISS_IVFSQ8H));
ASSERT_ANY_THROW(engine_ptr->BuildIndex("/tmp/milvus_index_9", milvus::engine::EngineType::FAISS_PQ));
fiu_disable("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled");
#endif
//merge self
status = engine_ptr->Merge(file_path);
ASSERT_FALSE(status.ok());
// FIU_ENABLE_FIU("VecIndexImpl.Add.throw_knowhere_exception");
// status = engine_ptr->Merge("/tmp/milvus_index_2");
// ASSERT_FALSE(status.ok());
// fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
FIU_ENABLE_FIU("vecIndex.throw_read_exception");
status = engine_ptr->Merge("dummy");
ASSERT_FALSE(status.ok());
fiu_disable("vecIndex.throw_read_exception");
//CPU version invoke CopyToCpu will fail
status = engine_ptr->CopyToCpu();
ASSERT_FALSE(status.ok());
#ifdef MILVUS_GPU_VERSION
status = engine_ptr->CopyToGpu(0, false);
ASSERT_TRUE(status.ok());
status = engine_ptr->GpuCache(0);
ASSERT_TRUE(status.ok());
status = engine_ptr->CopyToGpu(0, false);
// ASSERT_TRUE(status.ok());
ASSERT_TRUE(status.ok());
// auto new_engine = engine_ptr->Clone();
// ASSERT_EQ(new_engine->Dimension(), dimension);
// ASSERT_EQ(new_engine->Count(), ids.size());
status = engine_ptr->CopyToCpu();
// ASSERT_TRUE(status.ok());
ASSERT_TRUE(status.ok());
engine_ptr->CopyToCpu();
ASSERT_TRUE(status.ok());
#endif
}
auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT);
// ASSERT_TRUE(status.ok());
TEST_F(EngineTest, ENGINE_IMPL_NULL_INDEX_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024);
fiu_init(0); // init
fiu_enable("read_null_index", 1, NULL, 0);
engine_ptr->Load(true);
auto count = engine_ptr->Count();
ASSERT_EQ(count, 0);
auto dim = engine_ptr->Dimension();
ASSERT_EQ(dim, dimension);
auto status = engine_ptr->Merge("/tmp/milvus_index_2");
ASSERT_FALSE(status.ok());
auto build_index = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IDMAP);
ASSERT_EQ(build_index, nullptr);
int64_t n = 0;
const float* data = nullptr;
int64_t k = 10;
int64_t nprobe = 0;
float* distances = nullptr;
int64_t* labels = nullptr;
bool hybrid = false;
status = engine_ptr->Search(n, data, k, nprobe, distances, labels, hybrid);
ASSERT_FALSE(status.ok());
fiu_disable("read_null_index");
}
TEST_F(EngineTest, ENGINE_IMPL_THROW_EXCEPTION_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/invalid_file";
fiu_init(0); // init
fiu_enable("ValidateStringNotBool", 1, NULL, 0);
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024);
fiu_disable("ValidateStringNotBool");
fiu_init(0); // init
fiu_enable("vecIndex.throw_read_exception", 1, NULL, 0);
engine_ptr->Load(true);
engine_ptr->CopyToGpu(0, true);
engine_ptr->CopyToCpu();
fiu_disable("vecIndex.throw_read_exception");
}
core/unittest/db/test_mem.cpp
浏览文件 @ 4dee7dfa
......@@ -22,6 +22,9 @@
#include <iostream>
#include <random>
#include <thread>
#include <fiu-control.h>
#include <fiu-local.h>
#include <src/db/DBFactory.h>
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
......@@ -98,10 +101,23 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
ASSERT_TRUE(status.ok());
ASSERT_EQ(num_vectors_added, 50);
ASSERT_EQ(source.GetVectorIds().size(), 100);
auto current_num_vectors_added = source.GetNumVectorsAdded();
ASSERT_EQ(current_num_vectors_added, 100);
vectors.id_array_ = source.GetVectorIds();
ASSERT_EQ(vectors.id_array_.size(), 100);
fiu_init(0);
FIU_ENABLE_FIU("VecIndexImpl.Add.throw_knowhere_exception");
status = source.Add(execution_engine_, table_file_schema, 60, num_vectors_added);
ASSERT_FALSE(status.ok());
fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
}
TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
auto options = GetOptions();
fiu_init(0);
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
......@@ -115,7 +131,6 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
milvus::engine::VectorSourcePtr source = std::make_shared<milvus::engine::VectorSource>(vectors_100);
milvus::engine::IDNumbers vector_ids;
status = mem_table_file.Add(source);
ASSERT_TRUE(status.ok());
......@@ -130,12 +145,38 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(vectors_128M);
vector_ids.clear();
status = mem_table_file.Add(source_128M);
ASSERT_EQ(source_128M->GetVectorIds().size(), n_max - n_100);
ASSERT_TRUE(mem_table_file.IsFull());
//mem_table_file has no memory left = 0
status = mem_table_file.Add(source_128M);
ASSERT_TRUE(status.ok());
{
//test fail create table file
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTableFile.throw_exception");
milvus::engine::MemTableFile mem_table_file_1(GetTableName(), impl_, options);
fiu_disable("SqliteMetaImpl.CreateTableFile.throw_exception");
status = mem_table_file_1.Add(source);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), milvus::DB_ERROR);
}
{
options.insert_cache_immediately_ = true;
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
table_schema.table_id_ = "faiss_pq";
table_schema.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
milvus::engine::MemTableFile mem_table_file_1("faiss_pq", impl_, options);
mem_table_file_1.Serialize();
}
}
TEST_F(MemManagerTest, MEM_TABLE_TEST) {
......@@ -154,7 +195,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
milvus::engine::MemTable mem_table(GetTableName(), impl_, options);
milvus::engine::IDNumbers vector_ids;
status = mem_table.Add(source_100);
ASSERT_TRUE(status.ok());
ASSERT_EQ(source_100->GetVectorIds().size(), 100);
......@@ -168,7 +208,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
milvus::engine::VectorsData vectors_128M;
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(vectors_128M);
status = mem_table.Add(source_128M);
......@@ -187,7 +226,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
milvus::engine::VectorSourcePtr source_1G = std::make_shared<milvus::engine::VectorSource>(vectors_1G);
vector_ids.clear();
status = mem_table.Add(source_1G);
ASSERT_TRUE(status.ok());
......@@ -198,6 +236,24 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
status = mem_table.Serialize();
ASSERT_TRUE(status.ok());
milvus::engine::VectorsData vectors_10;
BuildVectors(10, vectors_10);
milvus::engine::VectorSourcePtr source_10 = std::make_shared<milvus::engine::VectorSource>(vectors_10);
fiu_init(0);
FIU_ENABLE_FIU("VecIndexImpl.Add.throw_knowhere_exception");
status = mem_table.Add(source_10);
ASSERT_FALSE(status.ok());
fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
status = mem_table.Add(source_10);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFile.throw_exception");
status = mem_table.Serialize();
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.UpdateTableFile.throw_exception");
}
TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
......
core/unittest/db/test_meta.cpp
浏览文件 @ 4dee7dfa
......@@ -25,6 +25,10 @@
#include <stdlib.h>
#include <time.h>
#include <thread>
#include <fiu-local.h>
#include <fiu-control.h>
#include <boost/filesystem/operations.hpp>
#include "src/db/OngoingFileChecker.h"
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
......@@ -49,11 +53,352 @@ TEST_F(MetaTest, TABLE_TEST) {
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST);
status = impl_->DropTable(table.table_id_);
ASSERT_TRUE(status.ok());
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), milvus::DB_ERROR);
table.table_id_ = "";
status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
}
TEST_F(MetaTest, FALID_TEST) {
fiu_init(0);
auto options = GetOptions();
auto table_id = "meta_test_table";
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
milvus::Status status;
{
FIU_ENABLE_FIU("SqliteMetaImpl.ValidateMetaSchema.NullConnection");
milvus::engine::meta::SqliteMetaImpl impl(options.meta_);
fiu_disable("SqliteMetaImpl.ValidateMetaSchema.NullConnection");
}
{
//failed initialize
auto options_1 = options;
options_1.meta_.path_ = options.meta_.path_ + "1";
if (boost::filesystem::is_directory(options_1.meta_.path_)) {
boost::filesystem::remove_all(options_1.meta_.path_);
}
FIU_ENABLE_FIU("SqliteMetaImpl.Initialize.fail_create_directory");
ASSERT_ANY_THROW(milvus::engine::meta::SqliteMetaImpl impl(options_1.meta_));
fiu_disable("SqliteMetaImpl.Initialize.fail_create_directory");
boost::filesystem::remove_all(options_1.meta_.path_);
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTable.throw_exception");
status = impl_->CreateTable(table);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CreateTable.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTable.insert_throw_exception");
table.table_id_ = "";
status = impl_->CreateTable(table);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.CreateTable.insert_throw_exception");
//success create table
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DescribeTable.throw_exception");
status = impl_->DescribeTable(table);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DescribeTable.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.HasTable.throw_exception");
bool has = false;
status = impl_->HasTable(table.table_id_, has);
ASSERT_FALSE(status.ok());
ASSERT_FALSE(has);
fiu_disable("SqliteMetaImpl.HasTable.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.AllTables.throw_exception");
std::vector<milvus::engine::meta::TableSchema> table_schema_array;
status = impl_->AllTables(table_schema_array);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.AllTables.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DropTable.throw_exception");
status = impl_->DropTable(table.table_id_);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DropTable.throw_exception");
}
{
milvus::engine::meta::TableFileSchema schema;
schema.table_id_ = "notexist";
status = impl_->CreateTableFile(schema);
ASSERT_FALSE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTableFile.throw_exception");
schema.table_id_ = table_id;
status = impl_->CreateTableFile(schema);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.CreateTableFile.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DeleteTableFiles.throw_exception");
status = impl_->DeleteTableFiles(table.table_id_);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DeleteTableFiles.throw_exception");
}
{
milvus::engine::meta::DatesT dates;
status = impl_->DropDataByDate(table.table_id_, dates);
ASSERT_TRUE(status.ok());
dates.push_back(1);
status = impl_->DropDataByDate("notexist", dates);
ASSERT_FALSE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.DropDataByDate.throw_exception");
status = impl_->DropDataByDate(table.table_id_, dates);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DropDataByDate.throw_exception");
}
{
milvus::engine::meta::TableFilesSchema schemas;
std::vector<size_t> ids;
status = impl_->GetTableFiles("notexist", ids, schemas);
ASSERT_FALSE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.GetTableFiles.throw_exception");
status = impl_->GetTableFiles(table_id, ids, schemas);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.GetTableFiles.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFlag.throw_exception");
status = impl_->UpdateTableFlag(table_id, 0);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFlag.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFile.throw_exception");
milvus::engine::meta::TableFileSchema schema;
schema.table_id_ = table_id;
status = impl_->UpdateTableFile(schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFile.throw_exception");
schema = {};
schema.table_id_ = "notexist";
status = impl_->UpdateTableFile(schema);
ASSERT_TRUE(status.ok());
}
{
milvus::engine::meta::TableFilesSchema schemas;
milvus::engine::meta::TableFileSchema schema;
schema.table_id_ = "notexits";
schemas.emplace_back(schema);
status = impl_->UpdateTableFiles(schemas);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFiles.throw_exception");
status = impl_->UpdateTableFiles(schemas);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFiles.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFiles.fail_commited");
status = impl_->UpdateTableFiles(schemas);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFiles.fail_commited");
}
{
milvus::engine::TableIndex index;
status = impl_->UpdateTableIndex("notexist", index);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableIndex.throw_exception");
status = impl_->UpdateTableIndex("notexist", index);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableIndex.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.DescribeTableIndex.throw_exception");
status = impl_->DescribeTableIndex(table_id, index);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.DescribeTableIndex.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFilesToIndex.throw_exception");
status = impl_->UpdateTableFilesToIndex(table_id);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFilesToIndex.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DropTableIndex.throw_exception");
status = impl_->DropTableIndex(table_id);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.DropTableIndex.throw_exception");
}
{
std::string partition = "part0";
std::string partition_tag = "tag0";
status = impl_->CreatePartition("notexist", partition, partition_tag);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
status = impl_->CreatePartition(table_id, partition, partition_tag);
ASSERT_TRUE(status.ok());
partition_tag = "tag1";
status = impl_->CreatePartition(table_id, partition, partition_tag);
ASSERT_FALSE(status.ok());
//create empty name partition
partition = "";
status = impl_->CreatePartition(table_id, partition, partition_tag);
ASSERT_TRUE(status.ok());
std::vector<milvus::engine::meta::TableSchema> partions_schema;
status = impl_->ShowPartitions(table_id, partions_schema);
ASSERT_TRUE(status.ok());
ASSERT_EQ(partions_schema.size(), 2);
partions_schema.clear();
FIU_ENABLE_FIU("SqliteMetaImpl.ShowPartitions.throw_exception");
status = impl_->ShowPartitions(table_id, partions_schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.ShowPartitions.throw_exception");
std::string partion;
FIU_ENABLE_FIU("SqliteMetaImpl.GetPartitionName.throw_exception");
status = impl_->GetPartitionName(table_id, "tag0", partion);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.GetPartitionName.throw_exception");
}
{
std::vector<size_t> ids;
milvus::engine::meta::DatesT dates;
milvus::engine::meta::DatePartionedTableFilesSchema schema;
status = impl_->FilesToSearch("notexist", ids, dates, schema);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
status = impl_->FilesToSearch(table_id, ids, dates, schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
}
{
milvus::engine::meta::TableFileSchema file;
file.table_id_ = table_id;
file.file_type_ = milvus::engine::meta::TableFileSchema::RAW;
status = impl_->CreateTableFile(file);
ASSERT_TRUE(status.ok());
file.file_size_ = std::numeric_limits<size_t>::max();
status = impl_->UpdateTableFile(file);
ASSERT_TRUE(status.ok());
milvus::engine::meta::DatePartionedTableFilesSchema schema;
status = impl_->FilesToMerge("notexist", schema);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToMerge.throw_exception");
status = impl_->FilesToMerge(table_id, schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesToMerge.throw_exception");
//skip large files
milvus::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[file.date_].size(), 0);
}
{
milvus::engine::meta::TableFileSchema file;
file.table_id_ = table_id;
status = impl_->CreateTableFile(file);
ASSERT_TRUE(status.ok());
file.file_type_ = milvus::engine::meta::TableFileSchema::TO_INDEX;
impl_->UpdateTableFile(file);
milvus::engine::meta::TableFilesSchema files;
FIU_ENABLE_FIU("SqliteMetaImpl_FilesToIndex_TableNotFound");
status = impl_->FilesToIndex(files);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
fiu_disable("SqliteMetaImpl_FilesToIndex_TableNotFound");
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToIndex.throw_exception");
status = impl_->FilesToIndex(files);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesToIndex.throw_exception");
}
{
milvus::engine::meta::TableFilesSchema files;
std::vector<int> file_types;
file_types.push_back(milvus::engine::meta::TableFileSchema::INDEX);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesByType.throw_exception");
status = impl_->FilesByType(table_id, file_types, files);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesByType.throw_exception");
}
{
uint64_t size = 0;
FIU_ENABLE_FIU("SqliteMetaImpl.Size.throw_exception");
status = impl_->Size(size);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.Size.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpShadowFiles.fail_commited");
status = impl_->CleanUpShadowFiles();
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpShadowFiles.fail_commited");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpShadowFiles.throw_exception");
status = impl_->CleanUpShadowFiles();
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpShadowFiles.throw_exception");
}
{
uint64_t count;
status = impl_->Count("notexist", count);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.Count.throw_exception");
status = impl_->Count("notexist", count);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.Count.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_ThrowException");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_ThrowException");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_FailCommited");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_FailCommited");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_Failcommited");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_Failcommited");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_ThrowException");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_ThrowException");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTableFolder_ThrowException");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTableFolder_ThrowException");
}
}
TEST_F(MetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
......@@ -148,6 +493,14 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
ids.push_back(table_file.id_);
}
{
fiu_init(0);
FIU_ENABLE_FIU("SqliteMetaImpl.Archive.throw_exception");
status = impl.Archive();
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.Archive.throw_exception");
}
impl.Archive();
int i = 0;
......@@ -193,6 +546,17 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
ids.push_back(table_file.id_);
}
{
fiu_init(0);
FIU_ENABLE_FIU("SqliteMetaImpl.DiscardFiles.throw_exception");
status = impl.Archive();
fiu_disable("SqliteMetaImpl.DiscardFiles.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.DiscardFiles.fail_commited");
status = impl.Archive();
fiu_disable("SqliteMetaImpl.DiscardFiles.fail_commited");
}
impl.Archive();
int i = 0;
......@@ -312,9 +676,9 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, table_files);
......@@ -337,6 +701,21 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
status = impl_->CleanUpFilesWithTTL(1UL);
ASSERT_TRUE(status.ok());
sleep(1);
std::vector<int> files_to_delete;
milvus::engine::meta::TableFilesSchema files_schema;
files_to_delete.push_back(milvus::engine::meta::TableFileSchema::TO_DELETE);
status = impl_->FilesByType(table_id, files_to_delete, files_schema);
ASSERT_TRUE(status.ok());
table_file.table_id_ = table_id;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_DELETE;
milvus::engine::OngoingFileChecker filter;
table_file.file_id_ = files_schema.front().file_id_;
filter.MarkOngoingFile(table_file);
status = impl_->CleanUpFilesWithTTL(1UL, &filter);
ASSERT_TRUE(status.ok());
}
TEST_F(MetaTest, INDEX_TEST) {
......
core/unittest/db/test_misc.cpp
浏览文件 @ 4dee7dfa
......@@ -28,6 +28,9 @@
#include <boost/filesystem.hpp>
#include <thread>
#include <vector>
#include <fiu-local.h>
#include <fiu-control.h>
#include "db/utils.h"
TEST(DBMiscTest, EXCEPTION_TEST) {
milvus::Exception ex1(100, "error");
......@@ -91,7 +94,20 @@ TEST(DBMiscTest, UTILS_TEST) {
options.slave_paths_.push_back("/tmp/milvus_test/slave_2");
const std::string TABLE_NAME = "test_tbl";
auto status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
fiu_init(0);
milvus::Status status;
FIU_ENABLE_FIU("CommonUtil.CreateDirectory.create_parent_fail");
status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_FALSE(status.ok());
fiu_disable("CommonUtil.CreateDirectory.create_parent_fail");
FIU_ENABLE_FIU("CreateTablePath.creat_slave_path");
status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_FALSE(status.ok());
fiu_disable("CreateTablePath.creat_slave_path");
status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for (auto& path : options.slave_paths_) {
......@@ -120,6 +136,30 @@ TEST(DBMiscTest, UTILS_TEST) {
status = milvus::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
status = milvus::engine::utils::CreateTableFilePath(options, file);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("CreateTableFilePath.fail_create");
status = milvus::engine::utils::CreateTableFilePath(options, file);
ASSERT_FALSE(status.ok());
fiu_disable("CreateTableFilePath.fail_create");
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(file.location_.empty());
FIU_ENABLE_FIU("CommonUtil.CreateDirectory.create_parent_fail");
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(file.location_.empty());
fiu_disable("CommonUtil.CreateDirectory.create_parent_fail");
FIU_ENABLE_FIU("GetTableFilePath.enable_s3");
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(file.location_.empty());
fiu_disable("GetTableFilePath.enable_s3");
status = milvus::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
}
TEST(DBMiscTest, CHECKER_TEST) {
......
core/unittest/db/utils.cpp
浏览文件 @ 4dee7dfa
......@@ -25,6 +25,7 @@
#include <string>
#include <thread>
#include <utility>
#include <fiu-local.h>
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
......@@ -231,7 +232,7 @@ MetaTest::TearDown() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
milvus::engine::DBOptions
MySqlDBTest::GetOptions() {
MySqlDBTest:: GetOptions() {
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = test_env->getURI();
......
core/unittest/db/utils.h
浏览文件 @ 4dee7dfa
......@@ -20,6 +20,8 @@
#include <gtest/gtest.h>
#include <chrono>
#include <memory>
#include <fiu-control.h>
#include <fiu-local.h>
#include "db/DB.h"
#include "db/meta/MySQLMetaImpl.h"
......@@ -44,6 +46,10 @@
#define STOP_TIMER(name)
#endif
#ifdef FIU_ENABLE
#define FIU_ENABLE_FIU(name) fiu_enable(name, 1, nullptr, 0)
#endif
static const char* CONFIG_PATH = "/tmp/milvus_test";
static const char* CONFIG_FILE = "/server_config.yaml";
......
core/unittest/main.cpp
浏览文件 @ 4dee7dfa
......@@ -17,6 +17,7 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <fiu-local.h>
#include "easyloggingpp/easylogging++.h"
......@@ -24,6 +25,7 @@ INITIALIZE_EASYLOGGINGPP
int
main(int argc, char** argv) {
fiu_init(0);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
core/unittest/metrics/test_metricbase.cpp
浏览文件 @ 4dee7dfa
......@@ -56,6 +56,7 @@ TEST(MetricbaseTest, METRICBASE_TEST) {
instance.RAMUsagePercentSet();
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
......
core/unittest/metrics/test_metrics.cpp
浏览文件 @ 4dee7dfa
......@@ -21,13 +21,17 @@
#include <string>
#include <thread>
#include <gtest/gtest.h>
#include <fiu-local.h>
#include <fiu-control.h>
#define private public
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "metrics/Metrics.h"
#include "metrics/utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "metrics/Metrics.h"
namespace {
static constexpr int64_t TABLE_DIM = 256;
......@@ -46,13 +50,30 @@ BuildVectors(uint64_t n, milvus::engine::VectorsData& vectors) {
} // namespace
TEST_F(MetricTest, METRIC_TEST) {
fiu_init(0);
#ifdef MILVUS_GPU_VERSION
FIU_ENABLE_FIU("SystemInfo.Init.nvmInit_fail");
milvus::server::SystemInfo::GetInstance().initialized_ = false;
milvus::server::SystemInfo::GetInstance().Init();
fiu_disable("SystemInfo.Init.nvmInit_fail");
FIU_ENABLE_FIU("SystemInfo.Init.nvm_getDevice_fail");
milvus::server::SystemInfo::GetInstance().initialized_ = false;
milvus::server::SystemInfo::GetInstance().Init();
fiu_disable("SystemInfo.Init.nvm_getDevice_fail");
milvus::server::SystemInfo::GetInstance().initialized_ = false;
#endif
milvus::server::SystemInfo::GetInstance().Init();
milvus::server::Metrics::GetInstance().Init();
std::string system_info;
milvus::server::SystemInfo::GetInstance().GetSysInfoJsonStr(system_info);
milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
std::cout << milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl;
static const char *group_name = "test_group";
static const char* group_name = "test_group";
static const int group_dim = 256;
milvus::engine::meta::TableSchema group_info;
......@@ -90,7 +111,7 @@ TEST_F(MetricTest, METRIC_TEST) {
START_TIMER;
// stat = db_->Query(group_name, tags, k, qb, qxb, result_ids, result_distances);
ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024)
ss << "Search " << j << " With Size " << (float)(count * group_dim * sizeof(float)) / (1024 * 1024)
<< " M";
for (auto k = 0; k < qb; ++k) {
......
core/unittest/metrics/test_prometheus.cpp
浏览文件 @ 4dee7dfa
......@@ -20,8 +20,11 @@
#include <gtest/gtest.h>
#include <iostream>
#include <fiu-control.h>
#include <fiu-local.h>
TEST(PrometheusTest, PROMETHEUS_TEST) {
fiu_init(0);
milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
milvus::server::PrometheusMetrics instance = milvus::server::PrometheusMetrics::GetInstance();
......@@ -57,9 +60,20 @@ TEST(PrometheusTest, PROMETHEUS_TEST) {
instance.QueryVectorResponseSummaryObserve(1.0, 1);
instance.QueryVectorResponsePerSecondGaugeSet(1.0);
instance.CPUUsagePercentSet();
fiu_enable("SystemInfo.CPUPercent.mock", 1, nullptr, 0);
instance.CPUUsagePercentSet();
fiu_disable("SystemInfo.CPUPercent.mock");
instance.RAMUsagePercentSet();
fiu_enable("SystemInfo.MemoryPercent.mock", 1, nullptr, 0);
instance.RAMUsagePercentSet();
fiu_disable("SystemInfo.MemoryPercent.mock");
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
fiu_enable("SystemInfo.GPUMemoryTotal.mock", 1, nullptr, 0);
fiu_enable("SystemInfo.GPUMemoryUsed.mock", 1, nullptr, 0);
instance.GPUPercentGaugeSet();
fiu_disable("SystemInfo.GPUMemoryTotal.mock");
fiu_disable("SystemInfo.GPUMemoryUsed.mock");
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
......@@ -71,8 +85,23 @@ TEST(PrometheusTest, PROMETHEUS_TEST) {
instance.OctetsSet();
instance.CPUCoreUsagePercentSet();
fiu_enable("SystemInfo.getTotalCpuTime.open_proc", 1, nullptr, 0);
instance.CPUCoreUsagePercentSet();
fiu_disable("SystemInfo.getTotalCpuTime.open_proc");
fiu_enable("SystemInfo.getTotalCpuTime.read_proc", 1, nullptr, 0);
instance.CPUCoreUsagePercentSet();
fiu_disable("SystemInfo.getTotalCpuTime.read_proc");
instance.GPUTemperature();
fiu_enable("SystemInfo.GPUTemperature.mock", 1, nullptr, 0);
instance.GPUTemperature();
fiu_disable("SystemInfo.GPUTemperature.mock");
instance.CPUTemperature();
fiu_enable("SystemInfo.CPUTemperature.opendir", 1, nullptr, 0);
instance.CPUTemperature();
fiu_disable("SystemInfo.CPUTemperature.opendir");
fiu_enable("SystemInfo.CPUTemperature.openfile", 1, nullptr, 0);
instance.CPUTemperature();
fiu_disable("SystemInfo.CPUTemperature.openfile");
milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
instance.Init();
......
core/unittest/metrics/utils.h
浏览文件 @ 4dee7dfa
......@@ -24,7 +24,7 @@
#include "db/DB.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/meta/SqliteMetaImpl.h"
#include <fiu-control.h>
#define TIMING
#ifdef TIMING
......@@ -42,6 +42,10 @@
#define STOP_TIMER(name)
#endif
#ifdef FIU_ENABLE
#define FIU_ENABLE_FIU(name) fiu_enable(name, 1, nullptr, 0)
#endif
void
ASSERT_STATS(milvus::Status& stat);
......
core/unittest/scheduler/CMakeLists.txt
浏览文件 @ 4dee7dfa
......@@ -26,6 +26,8 @@ set(test_files
${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_scheduler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_task.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_job.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_optimizer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_tasktable.cpp)
add_executable(test_scheduler
......
core/unittest/scheduler/test_action.cpp 0 → 100644
浏览文件 @ 4dee7dfa
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <fiu-control.h>
#include <gtest/gtest.h>
#include <src/scheduler/task/BuildIndexTask.h>
#include <src/scheduler/task/SearchTask.h>
#include <src/scheduler/optimizer/FaissIVFFlatPass.h>
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/optimizer/FaissFlatPass.h"
#include "scheduler/optimizer/FaissIVFPQPass.h"
#include "scheduler/optimizer/FaissIVFSQ8HPass.h"
#include "scheduler/optimizer/FaissIVFSQ8Pass.h"
namespace milvus {
namespace scheduler {
#ifdef MILVUS_GPU_VERSION
TEST(Action_Test, TESTACTION) {
}
#endif
} // namespace scheduler
} // namespace milvus
core/unittest/scheduler/test_job.cpp 0 → 100644
浏览文件 @ 4dee7dfa
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "scheduler/job/Job.h"
#include "scheduler/job/BuildIndexJob.h"
#include "scheduler/job/DeleteJob.h"
#include "scheduler/job/SearchJob.h"
namespace milvus {
namespace scheduler {
class TestJob : public Job {
public:
TestJob() : Job(JobType::INVALID) {}
};
TEST(JobTest, TestJob) {
engine::DBOptions options;
auto build_index_ptr = std::make_shared<BuildIndexJob>(nullptr, options);
build_index_ptr->Dump();
build_index_ptr->AddToIndexFiles(nullptr);
TestJob test_job;
test_job.Dump();
auto delete_ptr = std::make_shared<DeleteJob>("table_id", nullptr, 1);
delete_ptr->Dump();
engine::VectorsData vectors;
auto search_ptr = std::make_shared<SearchJob>(nullptr, 1, 1, vectors);
search_ptr->Dump();
search_ptr->AddIndexFile(nullptr);
}
} // namespace scheduler
} // namespace milvus
core/unittest/scheduler/test_optimizer.cpp 0 → 100644
浏览文件 @ 4dee7dfa
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <fiu-control.h>
#include <gtest/gtest.h>
#include "scheduler/task/BuildIndexTask.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/optimizer/FaissIVFFlatPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/resource/CpuResource.h"
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/optimizer/FaissFlatPass.h"
#include "scheduler/optimizer/FaissIVFPQPass.h"
#include "scheduler/optimizer/FaissIVFSQ8HPass.h"
#include "scheduler/optimizer/FaissIVFSQ8Pass.h"
#include "scheduler/optimizer/FallbackPass.h"
namespace milvus {
namespace scheduler {
#ifdef MILVUS_GPU_VERSION
TEST(OptimizerTest, TEST_OPTIMIZER) {
BuildIndexPass build_index_pass;
fiu_init(0);
fiu_enable("BuildIndexPass.Init.get_config_fail", 1, NULL, 0);
ASSERT_ANY_THROW(build_index_pass.Init(););
fiu_disable("BuildIndexPass.Init.get_config_fail");
auto build_index_task = std::make_shared<XBuildIndexTask>(nullptr, nullptr);
fiu_enable("BuildIndexPass.Run.empty_gpu_ids", 1, NULL, 0);
ASSERT_FALSE(build_index_pass.Run(build_index_task));
fiu_disable("BuildIndexPass.Run.empty_gpu_ids");
FaissFlatPass faiss_flat_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_flat_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
FaissIVFFlatPass faiss_ivf_flat_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_ivf_flat_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
FaissIVFPQPass faiss_ivf_pq_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_ivf_pq_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
auto file = std::make_shared<TableFileSchema>();
file->engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT;
auto search_task = std::make_shared<XSearchTask>(nullptr, file, nullptr);
ASSERT_FALSE(faiss_ivf_pq_pass.Run(search_task));
FaissIVFSQ8HPass faiss_ivf_q8h_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
faiss_ivf_q8h_pass.Init();
fiu_disable("check_config_gpu_search_threshold_fail");
auto search_task2 = std::make_shared<XSearchTask>(nullptr, file, nullptr);
ASSERT_FALSE(faiss_ivf_q8h_pass.Run(build_index_task));
ASSERT_FALSE(faiss_ivf_q8h_pass.Run(search_task2));
FaissIVFSQ8Pass faiss_ivf_q8_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_ivf_q8_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
FallbackPass fall_back_pass;
fall_back_pass.Init();
auto task = std::make_shared<XBuildIndexTask>(nullptr, nullptr);
ResMgrInst::GetInstance()->Add(std::make_shared<CpuResource>("name1", 1, true));
fall_back_pass.Run(task);
}
#endif
} // namespace scheduler
} // namespace milvus
core/unittest/scheduler/test_resource.cpp
浏览文件 @ 4dee7dfa
......@@ -132,13 +132,17 @@ class ResourceAdvanceTest : public testing::Test {
void
WaitLoader(uint64_t count) {
std::unique_lock<std::mutex> lock(load_mutex_);
cv_.wait(lock, [&] { return load_count_ == count; });
cv_.wait(lock, [&] {
return load_count_ == count;
});
}
void
WaitExecutor(uint64_t count) {
std::unique_lock<std::mutex> lock(exec_mutex_);
cv_.wait(lock, [&] { return exec_count_ == count; });
cv_.wait(lock, [&] {
return exec_count_ == count;
});
}
ResourcePtr disk_resource_;
......@@ -207,6 +211,12 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) {
for (uint64_t i = 0; i < NUM; ++i) {
ASSERT_EQ(tasks[i]->exec_count_, 1);
}
std::stringstream out;
out << *std::static_pointer_cast<CpuResource>(cpu_resource_);
out << *std::static_pointer_cast<GpuResource>(gpu_resource_);
out << *std::static_pointer_cast<DiskResource>(disk_resource_);
out << *std::static_pointer_cast<TestResource>(test_resource_);
}
TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) {
......@@ -263,6 +273,20 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
for (uint64_t i = 0; i < NUM; ++i) {
ASSERT_EQ(tasks[i]->exec_count_, 1);
}
test_resource_->TaskAvgCost();
std::cout << test_resource_->Dump() << " " << test_resource_->NumOfTaskToExec() << std::endl;
auto null_resource = ResourceFactory::Create("invalid", "invalid", 0);
ASSERT_EQ(null_resource, nullptr);
}
TEST(Connection_Test, CONNECTION_TEST) {
std::string connection_name = "cpu";
uint64_t speed = 982;
Connection connection(connection_name, speed);
ASSERT_EQ(connection_name, connection.name());
ASSERT_EQ(speed, connection.speed());
std::cout << connection.Dump() << std::endl;
}
} // namespace scheduler
......
core/unittest/scheduler/test_resource_mgr.cpp
浏览文件 @ 4dee7dfa
......@@ -57,6 +57,11 @@ TEST_F(ResourceMgrBaseTest, ADD) {
auto resource = std::make_shared<TestResource>("test", 0, true);
auto ret = empty_mgr_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
mgr1_->Start();
mgr1_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
mgr1_->Stop();
}
TEST_F(ResourceMgrBaseTest, ADD_DISK) {
......@@ -85,6 +90,11 @@ TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
TEST_F(ResourceMgrBaseTest, CLEAR) {
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Start();
mgr1_->Clear();
mgr1_->Stop();
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Clear();
ASSERT_EQ(mgr1_->GetNumOfResource(), 0);
}
......@@ -158,6 +168,10 @@ TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) {
ASSERT_FALSE(mgr1_->DumpTaskTables().empty());
}
TEST_F(ResourceMgrBaseTest, Start) {
empty_mgr_->Start();
}
/************ ResourceMgrAdvanceTest ************/
class ResourceMgrAdvanceTest : public testing::Test {
......@@ -184,7 +198,9 @@ class ResourceMgrAdvanceTest : public testing::Test {
TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
bool flag = false;
auto callback = [&](EventPtr event) { flag = true; };
auto callback = [&](EventPtr event) {
flag = true;
};
mgr1_->RegisterSubscriber(callback);
TableFileSchemaPtr dummy = nullptr;
disk_res->task_table().Put(
......
core/unittest/server/test_cache.cpp
浏览文件 @ 4dee7dfa
此差异已折叠。
core/unittest/server/test_rpc.cpp
浏览文件 @ 4dee7dfa
此差异已折叠。
core/unittest/server/test_util.cpp
浏览文件 @ 4dee7dfa
此差异已折叠。
core/unittest/server/utils.cpp
浏览文件 @ 4dee7dfa
......@@ -58,7 +58,7 @@ static const char* VALID_CONFIG_STR =
"engine_config:\n"
" use_blas_threshold: 20 \n"
"\n"
#ifdef MILVUS_GPU_VERSION
#ifdef MILVUS_GPU_VERSION
"gpu_resource_config:\n"
" enable: true # whether to enable GPU resources\n"
" cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer\n"
......@@ -66,7 +66,11 @@ static const char* VALID_CONFIG_STR =
" - gpu0\n"
" build_index_resources: # define the GPU devices used for index building, must be in format gpux\n"
" - gpu0\n"
#endif
#endif
"\n"
"sequence_config:\n"
" - seq1\n"
" - seq2\n"
"\n";
static const char* INVALID_CONFIG_STR = "*INVALID*";
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或