Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 68625cbf 编写于 作者: J Jin Hai 提交者： GitHub
浏览文件
操作

Fix 1724 (#1821) 

* #1724 Remove unused unittests
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* #1724 Remove unused unittests - part2
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* #1724 Remove unused unittests - part3
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* #1724 Remove unused unittests - part4
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* #1724 Remove unused unittests - part5
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* Change table_id to collection_id
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Change table to collection
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Fix format error
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* Change table_ to collection_
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Change table_id_ to collection_id_
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Fix format error
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* #1724 Change table_name to collection_name
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Reformat
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* Change TableFile to Segment
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* Change TableSchema to CollectionSchema
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* fix compile lint
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>
上级 6f858953
展开全部 隐藏空白更改
内联 并排
Showing with 2884 addition and 2832 deletion
core/src/config/Config.cpp
浏览文件 @ 68625cbf
......@@ -788,22 +788,22 @@ Config::CheckDBConfigPreloadTable(const std::string& value) {
std::unordered_set<std::string> table_set;
for (auto& table : tables) {
if (!ValidationUtil::ValidateTableName(table).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid table name: " + table);
for (auto& collection : tables) {
if (!ValidationUtil::ValidateCollectionName(collection).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid collection name: " + collection);
}
bool exist = false;
auto status = DBWrapper::DB()->HasNativeTable(table, exist);
auto status = DBWrapper::DB()->HasNativeTable(collection, exist);
if (!(status.ok() && exist)) {
return Status(SERVER_TABLE_NOT_EXIST, "Table " + table + " not exist");
return Status(SERVER_TABLE_NOT_EXIST, "Collection " + collection + " not exist");
}
table_set.insert(table);
table_set.insert(collection);
}
if (table_set.size() != tables.size()) {
std::string msg =
"Invalid preload tables. "
"Possible reason: db_config.preload_table contains duplicate table.";
"Possible reason: db_config.preload_table contains duplicate collection.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
......
core/src/db/DB.h
浏览文件 @ 68625cbf
......@@ -44,82 +44,82 @@ class DB {
Stop() = 0;
virtual Status
CreateTable(meta::TableSchema& table_schema_) = 0;
CreateTable(meta::CollectionSchema& table_schema_) = 0;
virtual Status
DropTable(const std::string& table_id) = 0;
DropTable(const std::string& collection_id) = 0;
virtual Status
DescribeTable(meta::TableSchema& table_schema_) = 0;
DescribeTable(meta::CollectionSchema& table_schema_) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not_) = 0;
HasTable(const std::string& collection_id, bool& has_or_not_) = 0;
virtual Status
HasNativeTable(const std::string& table_id, bool& has_or_not_) = 0;
HasNativeTable(const std::string& collection_id, bool& has_or_not_) = 0;
virtual Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
AllTables(std::vector<meta::CollectionSchema>& table_schema_array) = 0;
virtual Status
GetTableInfo(const std::string& table_id, TableInfo& table_info) = 0;
GetTableInfo(const std::string& collection_id, TableInfo& table_info) = 0;
virtual Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
GetTableRowCount(const std::string& collection_id, uint64_t& row_count) = 0;
virtual Status
PreloadTable(const std::string& table_id) = 0;
PreloadTable(const std::string& collection_id) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
UpdateTableFlag(const std::string& collection_id, int64_t flag) = 0;
virtual Status
CreatePartition(const std::string& table_id, const std::string& partition_name,
CreatePartition(const std::string& collection_id, const std::string& partition_name,
const std::string& partition_tag) = 0;
virtual Status
DropPartition(const std::string& partition_name) = 0;
virtual Status
DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0;
DropPartitionByTag(const std::string& collection_id, const std::string& partition_tag) = 0;
virtual Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) = 0;
ShowPartitions(const std::string& collection_id, std::vector<meta::CollectionSchema>& partition_schema_array) = 0;
virtual Status
InsertVectors(const std::string& table_id, const std::string& partition_tag, VectorsData& vectors) = 0;
InsertVectors(const std::string& collection_id, const std::string& partition_tag, VectorsData& vectors) = 0;
virtual Status
DeleteVector(const std::string& table_id, IDNumber vector_id) = 0;
DeleteVector(const std::string& collection_id, IDNumber vector_id) = 0;
virtual Status
DeleteVectors(const std::string& table_id, IDNumbers vector_ids) = 0;
DeleteVectors(const std::string& collection_id, IDNumbers vector_ids) = 0;
virtual Status
Flush(const std::string& table_id) = 0;
Flush(const std::string& collection_id) = 0;
virtual Status
Flush() = 0;
virtual Status
Compact(const std::string& table_id) = 0;
Compact(const std::string& collection_id) = 0;
virtual Status
GetVectorByID(const std::string& table_id, const IDNumber& vector_id, VectorsData& vector) = 0;
GetVectorByID(const std::string& collection_id, const IDNumber& vector_id, VectorsData& vector) = 0;
virtual Status
GetVectorIDs(const std::string& table_id, const std::string& segment_id, IDNumbers& vector_ids) = 0;
GetVectorIDs(const std::string& collection_id, const std::string& segment_id, IDNumbers& vector_ids) = 0;
// virtual Status
// Merge(const std::set<std::string>& table_ids) = 0;
virtual Status
QueryByID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
QueryByID(const std::shared_ptr<server::Context>& context, const std::string& collection_id,
const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) = 0;
virtual Status
Query(const std::shared_ptr<server::Context>& context, const std::string& table_id,
Query(const std::shared_ptr<server::Context>& context, const std::string& collection_id,
const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;
......@@ -132,13 +132,13 @@ class DB {
Size(uint64_t& result) = 0;
virtual Status
CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
CreateIndex(const std::string& collection_id, const TableIndex& index) = 0;
virtual Status
DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
DescribeIndex(const std::string& collection_id, TableIndex& index) = 0;
virtual Status
DropIndex(const std::string& table_id) = 0;
DropIndex(const std::string& collection_id) = 0;
virtual Status
DropAll() = 0;
......
core/src/db/DBImpl.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/src/db/DBImpl.h
浏览文件 @ 68625cbf
......@@ -55,91 +55,92 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
DropAll() override;
Status
CreateTable(meta::TableSchema& table_schema) override;
CreateTable(meta::CollectionSchema& table_schema) override;
Status
DropTable(const std::string& table_id) override;
DropTable(const std::string& collection_id) override;
Status
DescribeTable(meta::TableSchema& table_schema) override;
DescribeTable(meta::CollectionSchema& table_schema) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
HasTable(const std::string& collection_id, bool& has_or_not) override;
Status
HasNativeTable(const std::string& table_id, bool& has_or_not_) override;
HasNativeTable(const std::string& collection_id, bool& has_or_not_) override;
Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) override;
AllTables(std::vector<meta::CollectionSchema>& table_schema_array) override;
Status
GetTableInfo(const std::string& table_id, TableInfo& table_info) override;
GetTableInfo(const std::string& collection_id, TableInfo& table_info) override;
Status
PreloadTable(const std::string& table_id) override;
PreloadTable(const std::string& collection_id) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
UpdateTableFlag(const std::string& collection_id, int64_t flag) override;
Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) override;
GetTableRowCount(const std::string& collection_id, uint64_t& row_count) override;
Status
CreatePartition(const std::string& table_id, const std::string& partition_name,
CreatePartition(const std::string& collection_id, const std::string& partition_name,
const std::string& partition_tag) override;
Status
DropPartition(const std::string& partition_name) override;
Status
DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override;
DropPartitionByTag(const std::string& collection_id, const std::string& partition_tag) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) override;
ShowPartitions(const std::string& collection_id,
std::vector<meta::CollectionSchema>& partition_schema_array) override;
Status
InsertVectors(const std::string& table_id, const std::string& partition_tag, VectorsData& vectors) override;
InsertVectors(const std::string& collection_id, const std::string& partition_tag, VectorsData& vectors) override;
Status
DeleteVector(const std::string& table_id, IDNumber vector_id) override;
DeleteVector(const std::string& collection_id, IDNumber vector_id) override;
Status
DeleteVectors(const std::string& table_id, IDNumbers vector_ids) override;
DeleteVectors(const std::string& collection_id, IDNumbers vector_ids) override;
Status
Flush(const std::string& table_id) override;
Flush(const std::string& collection_id) override;
Status
Flush() override;
Status
Compact(const std::string& table_id) override;
Compact(const std::string& collection_id) override;
Status
GetVectorByID(const std::string& table_id, const IDNumber& vector_id, VectorsData& vector) override;
GetVectorByID(const std::string& collection_id, const IDNumber& vector_id, VectorsData& vector) override;
Status
GetVectorIDs(const std::string& table_id, const std::string& segment_id, IDNumbers& vector_ids) override;
GetVectorIDs(const std::string& collection_id, const std::string& segment_id, IDNumbers& vector_ids) override;
// Status
// Merge(const std::set<std::string>& table_ids) override;
Status
CreateIndex(const std::string& table_id, const TableIndex& index) override;
CreateIndex(const std::string& collection_id, const TableIndex& index) override;
Status
DescribeIndex(const std::string& table_id, TableIndex& index) override;
DescribeIndex(const std::string& collection_id, TableIndex& index) override;
Status
DropIndex(const std::string& table_id) override;
DropIndex(const std::string& collection_id) override;
Status
QueryByID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
QueryByID(const std::shared_ptr<server::Context>& context, const std::string& collection_id,
const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) override;
Status
Query(const std::shared_ptr<server::Context>& context, const std::string& table_id,
Query(const std::shared_ptr<server::Context>& context, const std::string& collection_id,
const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) override;
......@@ -160,13 +161,13 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
private:
Status
QueryAsync(const std::shared_ptr<server::Context>& context, const meta::TableFilesSchema& files, uint64_t k,
QueryAsync(const std::shared_ptr<server::Context>& context, const meta::SegmentsSchema& files, uint64_t k,
const milvus::json& extra_params, const VectorsData& vectors, ResultIds& result_ids,
ResultDistances& result_distances);
Status
GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, VectorsData& vector,
const meta::TableFilesSchema& files);
GetVectorByIdHelper(const std::string& collection_id, IDNumber vector_id, VectorsData& vector,
const meta::SegmentsSchema& files);
void
BackgroundTimerTask();
......@@ -184,10 +185,10 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
StartMergeTask();
Status
MergeFiles(const std::string& table_id, const meta::TableFilesSchema& files);
MergeFiles(const std::string& collection_id, const meta::SegmentsSchema& files);
Status
BackgroundMergeFiles(const std::string& table_id);
BackgroundMergeFiles(const std::string& collection_id);
void
BackgroundMerge(std::set<std::string> table_ids);
......@@ -199,8 +200,8 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
BackgroundBuildIndex();
Status
CompactFile(const std::string& table_id, const meta::TableFileSchema& file,
meta::TableFilesSchema& files_to_update);
CompactFile(const std::string& collection_id, const meta::SegmentSchema& file,
meta::SegmentsSchema& files_to_update);
/*
Status
......@@ -208,33 +209,33 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
*/
Status
GetFilesToBuildIndex(const std::string& table_id, const std::vector<int>& file_types,
meta::TableFilesSchema& files);
GetFilesToBuildIndex(const std::string& collection_id, const std::vector<int>& file_types,
meta::SegmentsSchema& files);
Status
GetFilesToSearch(const std::string& table_id, meta::TableFilesSchema& files);
GetFilesToSearch(const std::string& collection_id, meta::SegmentsSchema& files);
Status
GetPartitionByTag(const std::string& table_id, const std::string& partition_tag, std::string& partition_name);
GetPartitionByTag(const std::string& collection_id, const std::string& partition_tag, std::string& partition_name);
Status
GetPartitionsByTags(const std::string& table_id, const std::vector<std::string>& partition_tags,
GetPartitionsByTags(const std::string& collection_id, const std::vector<std::string>& partition_tags,
std::set<std::string>& partition_name_array);
Status
DropTableRecursively(const std::string& table_id);
DropTableRecursively(const std::string& collection_id);
Status
UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index);
UpdateTableIndexRecursively(const std::string& collection_id, const TableIndex& index);
Status
WaitTableIndexRecursively(const std::string& table_id, const TableIndex& index);
WaitTableIndexRecursively(const std::string& collection_id, const TableIndex& index);
Status
DropTableIndexRecursively(const std::string& table_id);
DropTableIndexRecursively(const std::string& collection_id);
Status
GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count);
GetTableRowCountRecursively(const std::string& collection_id, uint64_t& row_count);
Status
ExecWalRecord(const wal::MXLogRecord& record);
......
core/src/db/IndexFailedChecker.cpp
浏览文件 @ 68625cbf
......@@ -20,17 +20,17 @@ namespace engine {
constexpr uint64_t INDEX_FAILED_RETRY_TIME = 1;
Status
IndexFailedChecker::CleanFailedIndexFileOfTable(const std::string& table_id) {
IndexFailedChecker::CleanFailedIndexFileOfTable(const std::string& collection_id) {
std::lock_guard<std::mutex> lck(mutex_);
index_failed_files_.erase(table_id); // rebuild failed index files for this table
index_failed_files_.erase(collection_id); // rebuild failed index files for this collection
return Status::OK();
}
Status
IndexFailedChecker::GetErrMsgForTable(const std::string& table_id, std::string& err_msg) {
IndexFailedChecker::GetErrMsgForTable(const std::string& collection_id, std::string& err_msg) {
std::lock_guard<std::mutex> lck(mutex_);
auto iter = index_failed_files_.find(table_id);
auto iter = index_failed_files_.find(collection_id);
if (iter != index_failed_files_.end()) {
err_msg = iter->second.begin()->second[0];
}
......@@ -39,14 +39,14 @@ IndexFailedChecker::GetErrMsgForTable(const std::string& table_id, std::string&
}
Status
IndexFailedChecker::MarkFailedIndexFile(const meta::TableFileSchema& file, const std::string& err_msg) {
IndexFailedChecker::MarkFailedIndexFile(const meta::SegmentSchema& file, const std::string& err_msg) {
std::lock_guard<std::mutex> lck(mutex_);
auto iter = index_failed_files_.find(file.table_id_);
auto iter = index_failed_files_.find(file.collection_id_);
if (iter == index_failed_files_.end()) {
File2ErrArray failed_files;
failed_files.insert(std::make_pair(file.file_id_, std::vector<std::string>(1, err_msg)));
index_failed_files_.insert(std::make_pair(file.table_id_, failed_files));
index_failed_files_.insert(std::make_pair(file.collection_id_, failed_files));
} else {
auto it_failed_files = iter->second.find(file.file_id_);
if (it_failed_files != iter->second.end()) {
......@@ -60,14 +60,14 @@ IndexFailedChecker::MarkFailedIndexFile(const meta::TableFileSchema& file, const
}
Status
IndexFailedChecker::MarkSucceedIndexFile(const meta::TableFileSchema& file) {
IndexFailedChecker::MarkSucceedIndexFile(const meta::SegmentSchema& file) {
std::lock_guard<std::mutex> lck(mutex_);
auto iter = index_failed_files_.find(file.table_id_);
auto iter = index_failed_files_.find(file.collection_id_);
if (iter != index_failed_files_.end()) {
iter->second.erase(file.file_id_);
if (iter->second.empty()) {
index_failed_files_.erase(file.table_id_);
index_failed_files_.erase(file.collection_id_);
}
}
......@@ -75,14 +75,14 @@ IndexFailedChecker::MarkSucceedIndexFile(const meta::TableFileSchema& file) {
}
Status
IndexFailedChecker::IgnoreFailedIndexFiles(meta::TableFilesSchema& table_files) {
IndexFailedChecker::IgnoreFailedIndexFiles(meta::SegmentsSchema& table_files) {
std::lock_guard<std::mutex> lck(mutex_);
// there could be some failed files belong to different table.
// there could be some failed files belong to different collection.
// some files may has failed for several times, no need to build index for these files.
// thus we can avoid dead circle for build index operation
for (auto it_file = table_files.begin(); it_file != table_files.end();) {
auto it_failed_files = index_failed_files_.find((*it_file).table_id_);
auto it_failed_files = index_failed_files_.find((*it_file).collection_id_);
if (it_failed_files != index_failed_files_.end()) {
auto it_failed_file = it_failed_files->second.find((*it_file).file_id_);
if (it_failed_file != it_failed_files->second.end()) {
......
core/src/db/IndexFailedChecker.h
浏览文件 @ 68625cbf
......@@ -25,23 +25,23 @@ namespace engine {
class IndexFailedChecker {
public:
Status
CleanFailedIndexFileOfTable(const std::string& table_id);
CleanFailedIndexFileOfTable(const std::string& collection_id);
Status
GetErrMsgForTable(const std::string& table_id, std::string& err_msg);
GetErrMsgForTable(const std::string& collection_id, std::string& err_msg);
Status
MarkFailedIndexFile(const meta::TableFileSchema& file, const std::string& err_msg);
MarkFailedIndexFile(const meta::SegmentSchema& file, const std::string& err_msg);
Status
MarkSucceedIndexFile(const meta::TableFileSchema& file);
MarkSucceedIndexFile(const meta::SegmentSchema& file);
Status
IgnoreFailedIndexFiles(meta::TableFilesSchema& table_files);
IgnoreFailedIndexFiles(meta::SegmentsSchema& table_files);
private:
std::mutex mutex_;
Table2FileErr index_failed_files_; // table id mapping to (file id mapping to failed times)
Table2FileErr index_failed_files_; // collection id mapping to (file id mapping to failed times)
};
} // namespace engine
......
core/src/db/OngoingFileChecker.cpp
浏览文件 @ 68625cbf
......@@ -24,13 +24,13 @@ OngoingFileChecker::GetInstance() {
}
Status
OngoingFileChecker::MarkOngoingFile(const meta::TableFileSchema& table_file) {
OngoingFileChecker::MarkOngoingFile(const meta::SegmentSchema& table_file) {
std::lock_guard<std::mutex> lck(mutex_);
return MarkOngoingFileNoLock(table_file);
}
Status
OngoingFileChecker::MarkOngoingFiles(const meta::TableFilesSchema& table_files) {
OngoingFileChecker::MarkOngoingFiles(const meta::SegmentsSchema& table_files) {
std::lock_guard<std::mutex> lck(mutex_);
for (auto& table_file : table_files) {
......@@ -41,13 +41,13 @@ OngoingFileChecker::MarkOngoingFiles(const meta::TableFilesSchema& table_files)
}
Status
OngoingFileChecker::UnmarkOngoingFile(const meta::TableFileSchema& table_file) {
OngoingFileChecker::UnmarkOngoingFile(const meta::SegmentSchema& table_file) {
std::lock_guard<std::mutex> lck(mutex_);
return UnmarkOngoingFileNoLock(table_file);
}
Status
OngoingFileChecker::UnmarkOngoingFiles(const meta::TableFilesSchema& table_files) {
OngoingFileChecker::UnmarkOngoingFiles(const meta::SegmentsSchema& table_files) {
std::lock_guard<std::mutex> lck(mutex_);
for (auto& table_file : table_files) {
......@@ -58,10 +58,10 @@ OngoingFileChecker::UnmarkOngoingFiles(const meta::TableFilesSchema& table_files
}
bool
OngoingFileChecker::IsIgnored(const meta::TableFileSchema& schema) {
OngoingFileChecker::IsIgnored(const meta::SegmentSchema& schema) {
std::lock_guard<std::mutex> lck(mutex_);
auto iter = ongoing_files_.find(schema.table_id_);
auto iter = ongoing_files_.find(schema.collection_id_);
if (iter == ongoing_files_.end()) {
return false;
} else {
......@@ -75,16 +75,16 @@ OngoingFileChecker::IsIgnored(const meta::TableFileSchema& schema) {
}
Status
OngoingFileChecker::MarkOngoingFileNoLock(const meta::TableFileSchema& table_file) {
if (table_file.table_id_.empty() || table_file.file_id_.empty()) {
return Status(DB_ERROR, "Invalid table files");
OngoingFileChecker::MarkOngoingFileNoLock(const meta::SegmentSchema& table_file) {
if (table_file.collection_id_.empty() || table_file.file_id_.empty()) {
return Status(DB_ERROR, "Invalid collection files");
}
auto iter = ongoing_files_.find(table_file.table_id_);
auto iter = ongoing_files_.find(table_file.collection_id_);
if (iter == ongoing_files_.end()) {
File2RefCount files_refcount;
files_refcount.insert(std::make_pair(table_file.file_id_, 1));
ongoing_files_.insert(std::make_pair(table_file.table_id_, files_refcount));
ongoing_files_.insert(std::make_pair(table_file.collection_id_, files_refcount));
} else {
auto it_file = iter->second.find(table_file.file_id_);
if (it_file == iter->second.end()) {
......@@ -95,18 +95,18 @@ OngoingFileChecker::MarkOngoingFileNoLock(const meta::TableFileSchema& table_fil
}
ENGINE_LOG_DEBUG << "Mark ongoing file:" << table_file.file_id_
<< " refcount:" << ongoing_files_[table_file.table_id_][table_file.file_id_];
<< " refcount:" << ongoing_files_[table_file.collection_id_][table_file.file_id_];
return Status::OK();
}
Status
OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_file) {
if (table_file.table_id_.empty() || table_file.file_id_.empty()) {
return Status(DB_ERROR, "Invalid table files");
OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::SegmentSchema& table_file) {
if (table_file.collection_id_.empty() || table_file.file_id_.empty()) {
return Status(DB_ERROR, "Invalid collection files");
}
auto iter = ongoing_files_.find(table_file.table_id_);
auto iter = ongoing_files_.find(table_file.collection_id_);
if (iter != ongoing_files_.end()) {
auto it_file = iter->second.find(table_file.file_id_);
if (it_file != iter->second.end()) {
......@@ -117,7 +117,7 @@ OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_f
if (it_file->second <= 0) {
iter->second.erase(table_file.file_id_);
if (iter->second.empty()) {
ongoing_files_.erase(table_file.table_id_);
ongoing_files_.erase(table_file.collection_id_);
}
}
}
......
core/src/db/OngoingFileChecker.h
浏览文件 @ 68625cbf
......@@ -29,30 +29,30 @@ class OngoingFileChecker {
GetInstance();
Status
MarkOngoingFile(const meta::TableFileSchema& table_file);
MarkOngoingFile(const meta::SegmentSchema& table_file);
Status
MarkOngoingFiles(const meta::TableFilesSchema& table_files);
MarkOngoingFiles(const meta::SegmentsSchema& table_files);
Status
UnmarkOngoingFile(const meta::TableFileSchema& table_file);
UnmarkOngoingFile(const meta::SegmentSchema& table_file);
Status
UnmarkOngoingFiles(const meta::TableFilesSchema& table_files);
UnmarkOngoingFiles(const meta::SegmentsSchema& table_files);
bool
IsIgnored(const meta::TableFileSchema& schema);
IsIgnored(const meta::SegmentSchema& schema);
private:
Status
MarkOngoingFileNoLock(const meta::TableFileSchema& table_file);
MarkOngoingFileNoLock(const meta::SegmentSchema& table_file);
Status
UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_file);
UnmarkOngoingFileNoLock(const meta::SegmentSchema& table_file);
private:
std::mutex mutex_;
Table2FileRef ongoing_files_; // table id mapping to (file id mapping to ongoing ref-count)
Table2FileRef ongoing_files_; // collection id mapping to (file id mapping to ongoing ref-count)
};
} // namespace engine
......
core/src/db/Utils.cpp
浏览文件 @ 68625cbf
......@@ -36,19 +36,19 @@ uint64_t index_file_counter = 0;
std::mutex index_file_counter_mutex;
static std::string
ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_;
ConstructParentFolder(const std::string& db_path, const meta::SegmentSchema& table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.collection_id_;
std::string partition_path = table_path + "/" + table_file.segment_id_;
return partition_path;
}
static std::string
GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) {
GetTableFileParentFolder(const DBMetaOptions& options, const meta::SegmentSchema& table_file) {
uint64_t path_count = options.slave_paths_.size() + 1;
std::string target_path = options.path_;
uint64_t index = 0;
if (meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
if (meta::SegmentSchema::NEW_INDEX == table_file.file_type_) {
// index file is large file and to be persisted permanently
// we need to distribute index files to each db_path averagely
// round robin according to a file counter
......@@ -79,9 +79,9 @@ GetMicroSecTimeStamp() {
}
Status
CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
CreateTablePath(const DBMetaOptions& options, const std::string& collection_id) {
std::string db_path = options.path_;
std::string table_path = db_path + TABLES_FOLDER + table_id;
std::string table_path = db_path + TABLES_FOLDER + collection_id;
auto status = server::CommonUtil::CreateDirectory(table_path);
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
......@@ -89,7 +89,7 @@ CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
}
for (auto& path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
table_path = path + TABLES_FOLDER + collection_id;
status = server::CommonUtil::CreateDirectory(table_path);
fiu_do_on("CreateTablePath.creat_slave_path", status = Status(DB_INVALID_PATH, ""));
if (!status.ok()) {
......@@ -102,18 +102,18 @@ CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
}
Status
DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
DeleteTablePath(const DBMetaOptions& options, const std::string& collection_id, bool force) {
std::vector<std::string> paths = options.slave_paths_;
paths.push_back(options.path_);
for (auto& path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id;
std::string table_path = path + TABLES_FOLDER + collection_id;
if (force) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
ENGINE_LOG_DEBUG << "Remove collection folder: " << table_path;
} else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
ENGINE_LOG_DEBUG << "Remove collection folder: " << table_path;
}
}
......@@ -122,7 +122,7 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool
config.GetStorageConfigS3Enable(s3_enable);
if (s3_enable) {
std::string table_path = options.path_ + TABLES_FOLDER + table_id;
std::string table_path = options.path_ + TABLES_FOLDER + collection_id;
auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
Status stat = storage_inst.DeleteObjects(table_path);
......@@ -135,7 +135,7 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool
}
Status
CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
CreateTableFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
......@@ -151,7 +151,7 @@ CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_f
}
Status
GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
GetTableFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_;
......@@ -179,23 +179,23 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file
}
}
std::string msg = "Table file doesn't exist: " + file_path;
std::string msg = "Collection file doesn't exist: " + file_path;
if (table_file.file_size_ > 0) { // no need to pop error for empty file
ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for table: " << table_file.table_id_;
ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for collection: " << table_file.collection_id_;
}
return Status(DB_ERROR, msg);
}
Status
DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
DeleteTableFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
utils::GetTableFilePath(options, table_file);
boost::filesystem::remove(table_file.location_);
return Status::OK();
}
Status
DeleteSegment(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
DeleteSegment(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
utils::GetTableFilePath(options, table_file);
std::string segment_dir;
GetParentPath(table_file.location_, segment_dir);
......
core/src/db/Utils.h
浏览文件 @ 68625cbf
......@@ -26,18 +26,18 @@ int64_t
GetMicroSecTimeStamp();
Status
CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
CreateTablePath(const DBMetaOptions& options, const std::string& collection_id);
Status
DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
DeleteTablePath(const DBMetaOptions& options, const std::string& collection_id, bool force = true);
Status
CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
CreateTableFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file);
Status
GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
GetTableFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file);
Status
DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
DeleteTableFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file);
Status
DeleteSegment(const DBMetaOptions& options, meta::TableFileSchema& table_file);
DeleteSegment(const DBMetaOptions& options, meta::SegmentSchema& table_file);
Status
GetParentPath(const std::string& path, std::string& parent_path);
......
core/src/db/insert/MemManager.h
浏览文件 @ 68625cbf
......@@ -24,21 +24,21 @@ namespace engine {
class MemManager {
public:
virtual Status
InsertVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
InsertVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
const float* vectors, uint64_t lsn, std::set<std::string>& flushed_tables) = 0;
virtual Status
InsertVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
InsertVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
const uint8_t* vectors, uint64_t lsn, std::set<std::string>& flushed_tables) = 0;
virtual Status
DeleteVector(const std::string& table_id, IDNumber vector_id, uint64_t lsn) = 0;
DeleteVector(const std::string& collection_id, IDNumber vector_id, uint64_t lsn) = 0;
virtual Status
DeleteVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, uint64_t lsn) = 0;
DeleteVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, uint64_t lsn) = 0;
virtual Status
Flush(const std::string& table_id, bool apply_delete = true) = 0;
Flush(const std::string& collection_id, bool apply_delete = true) = 0;
virtual Status
Flush(std::set<std::string>& table_ids, bool apply_delete = true) = 0;
......@@ -47,7 +47,7 @@ class MemManager {
// Serialize(std::set<std::string>& table_ids) = 0;
virtual Status
EraseMemVector(const std::string& table_id) = 0;
EraseMemVector(const std::string& collection_id) = 0;
virtual size_t
GetCurrentMutableMem() = 0;
......
core/src/db/insert/MemManagerImpl.cpp
浏览文件 @ 68625cbf
......@@ -21,18 +21,18 @@ namespace milvus {
namespace engine {
MemTablePtr
MemManagerImpl::GetMemByTable(const std::string& table_id) {
auto memIt = mem_id_map_.find(table_id);
MemManagerImpl::GetMemByTable(const std::string& collection_id) {
auto memIt = mem_id_map_.find(collection_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
}
mem_id_map_[table_id] = std::make_shared<MemTable>(table_id, meta_, options_);
return mem_id_map_[table_id];
mem_id_map_[collection_id] = std::make_shared<MemTable>(collection_id, meta_, options_);
return mem_id_map_[collection_id];
}
Status
MemManagerImpl::InsertVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
MemManagerImpl::InsertVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
const float* vectors, uint64_t lsn, std::set<std::string>& flushed_tables) {
flushed_tables.clear();
if (GetCurrentMem() > options_.insert_buffer_size_) {
......@@ -54,11 +54,11 @@ MemManagerImpl::InsertVectors(const std::string& table_id, int64_t length, const
std::unique_lock<std::mutex> lock(mutex_);
return InsertVectorsNoLock(table_id, source, lsn);
return InsertVectorsNoLock(collection_id, source, lsn);
}
Status
MemManagerImpl::InsertVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
MemManagerImpl::InsertVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
const uint8_t* vectors, uint64_t lsn, std::set<std::string>& flushed_tables) {
flushed_tables.clear();
if (GetCurrentMem() > options_.insert_buffer_size_) {
......@@ -80,12 +80,12 @@ MemManagerImpl::InsertVectors(const std::string& table_id, int64_t length, const
std::unique_lock<std::mutex> lock(mutex_);
return InsertVectorsNoLock(table_id, source, lsn);
return InsertVectorsNoLock(collection_id, source, lsn);
}
Status
MemManagerImpl::InsertVectorsNoLock(const std::string& table_id, const VectorSourcePtr& source, uint64_t lsn) {
MemTablePtr mem = GetMemByTable(table_id);
MemManagerImpl::InsertVectorsNoLock(const std::string& collection_id, const VectorSourcePtr& source, uint64_t lsn) {
MemTablePtr mem = GetMemByTable(collection_id);
mem->SetLSN(lsn);
auto status = mem->Add(source);
......@@ -93,18 +93,19 @@ MemManagerImpl::InsertVectorsNoLock(const std::string& table_id, const VectorSou
}
Status
MemManagerImpl::DeleteVector(const std::string& table_id, IDNumber vector_id, uint64_t lsn) {
MemManagerImpl::DeleteVector(const std::string& collection_id, IDNumber vector_id, uint64_t lsn) {
std::unique_lock<std::mutex> lock(mutex_);
MemTablePtr mem = GetMemByTable(table_id);
MemTablePtr mem = GetMemByTable(collection_id);
mem->SetLSN(lsn);
auto status = mem->Delete(vector_id);
return status;
}
Status
MemManagerImpl::DeleteVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, uint64_t lsn) {
MemManagerImpl::DeleteVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids,
uint64_t lsn) {
std::unique_lock<std::mutex> lock(mutex_);
MemTablePtr mem = GetMemByTable(table_id);
MemTablePtr mem = GetMemByTable(collection_id);
mem->SetLSN(lsn);
IDNumbers ids;
......@@ -128,8 +129,8 @@ MemManagerImpl::DeleteVectors(const std::string& table_id, int64_t length, const
}
Status
MemManagerImpl::Flush(const std::string& table_id, bool apply_delete) {
ToImmutable(table_id);
MemManagerImpl::Flush(const std::string& collection_id, bool apply_delete) {
ToImmutable(collection_id);
// TODO: There is actually only one memTable in the immutable list
MemList temp_immutable_list;
{
......@@ -140,13 +141,13 @@ MemManagerImpl::Flush(const std::string& table_id, bool apply_delete) {
std::unique_lock<std::mutex> lock(serialization_mtx_);
auto max_lsn = GetMaxLSN(temp_immutable_list);
for (auto& mem : temp_immutable_list) {
ENGINE_LOG_DEBUG << "Flushing table: " << mem->GetTableId();
ENGINE_LOG_DEBUG << "Flushing collection: " << mem->GetTableId();
auto status = mem->Serialize(max_lsn, apply_delete);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Flush table " << mem->GetTableId() << " failed";
ENGINE_LOG_ERROR << "Flush collection " << mem->GetTableId() << " failed";
return status;
}
ENGINE_LOG_DEBUG << "Flushed table: " << mem->GetTableId();
ENGINE_LOG_DEBUG << "Flushed collection: " << mem->GetTableId();
}
return Status::OK();
......@@ -166,14 +167,14 @@ MemManagerImpl::Flush(std::set<std::string>& table_ids, bool apply_delete) {
table_ids.clear();
auto max_lsn = GetMaxLSN(temp_immutable_list);
for (auto& mem : temp_immutable_list) {
ENGINE_LOG_DEBUG << "Flushing table: " << mem->GetTableId();
ENGINE_LOG_DEBUG << "Flushing collection: " << mem->GetTableId();
auto status = mem->Serialize(max_lsn, apply_delete);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Flush table " << mem->GetTableId() << " failed";
ENGINE_LOG_ERROR << "Flush collection " << mem->GetTableId() << " failed";
return status;
}
table_ids.insert(mem->GetTableId());
ENGINE_LOG_DEBUG << "Flushed table: " << mem->GetTableId();
ENGINE_LOG_DEBUG << "Flushed collection: " << mem->GetTableId();
}
meta_->SetGlobalLastLSN(max_lsn);
......@@ -182,15 +183,15 @@ MemManagerImpl::Flush(std::set<std::string>& table_ids, bool apply_delete) {
}
Status
MemManagerImpl::ToImmutable(const std::string& table_id) {
MemManagerImpl::ToImmutable(const std::string& collection_id) {
std::unique_lock<std::mutex> lock(mutex_);
auto memIt = mem_id_map_.find(table_id);
auto memIt = mem_id_map_.find(collection_id);
if (memIt != mem_id_map_.end()) {
if (!memIt->second->Empty()) {
immu_mem_list_.push_back(memIt->second);
mem_id_map_.erase(memIt);
}
// std::string err_msg = "Could not find table = " + table_id + " to flush";
// std::string err_msg = "Could not find collection = " + collection_id + " to flush";
// ENGINE_LOG_ERROR << err_msg;
// return Status(DB_NOT_FOUND, err_msg);
}
......@@ -204,7 +205,7 @@ MemManagerImpl::ToImmutable() {
MemIdMap temp_map;
for (auto& kv : mem_id_map_) {
if (kv.second->Empty()) {
// empty table without any deletes, no need to serialize
// empty collection without any deletes, no need to serialize
temp_map.insert(kv);
} else {
immu_mem_list_.push_back(kv.second);
......@@ -216,17 +217,17 @@ MemManagerImpl::ToImmutable() {
}
Status
MemManagerImpl::EraseMemVector(const std::string& table_id) {
MemManagerImpl::EraseMemVector(const std::string& collection_id) {
{ // erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id);
mem_id_map_.erase(collection_id);
}
{ // erase MemVector from serialize cache
std::unique_lock<std::mutex> lock(serialization_mtx_);
MemList temp_list;
for (auto& mem : immu_mem_list_) {
if (mem->GetTableId() != table_id) {
if (mem->GetTableId() != collection_id) {
temp_list.push_back(mem);
}
}
......@@ -265,9 +266,9 @@ MemManagerImpl::GetCurrentMem() {
uint64_t
MemManagerImpl::GetMaxLSN(const MemList& tables) {
uint64_t max_lsn = 0;
for (auto& table : tables) {
auto cur_lsn = table->GetLSN();
if (table->GetLSN() > max_lsn) {
for (auto& collection : tables) {
auto cur_lsn = collection->GetLSN();
if (collection->GetLSN() > max_lsn) {
max_lsn = cur_lsn;
}
}
......
core/src/db/insert/MemManagerImpl.h
浏览文件 @ 68625cbf
......@@ -41,21 +41,21 @@ class MemManagerImpl : public MemManager, public server::CacheConfigHandler {
}
Status
InsertVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
InsertVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
const float* vectors, uint64_t lsn, std::set<std::string>& flushed_tables) override;
Status
InsertVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
InsertVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, int64_t dim,
const uint8_t* vectors, uint64_t lsn, std::set<std::string>& flushed_tables) override;
Status
DeleteVector(const std::string& table_id, IDNumber vector_id, uint64_t lsn) override;
DeleteVector(const std::string& collection_id, IDNumber vector_id, uint64_t lsn) override;
Status
DeleteVectors(const std::string& table_id, int64_t length, const IDNumber* vector_ids, uint64_t lsn) override;
DeleteVectors(const std::string& collection_id, int64_t length, const IDNumber* vector_ids, uint64_t lsn) override;
Status
Flush(const std::string& table_id, bool apply_delete = true) override;
Flush(const std::string& collection_id, bool apply_delete = true) override;
Status
Flush(std::set<std::string>& table_ids, bool apply_delete = true) override;
......@@ -64,7 +64,7 @@ class MemManagerImpl : public MemManager, public server::CacheConfigHandler {
// Serialize(std::set<std::string>& table_ids) override;
Status
EraseMemVector(const std::string& table_id) override;
EraseMemVector(const std::string& collection_id) override;
size_t
GetCurrentMutableMem() override;
......@@ -81,16 +81,16 @@ class MemManagerImpl : public MemManager, public server::CacheConfigHandler {
private:
MemTablePtr
GetMemByTable(const std::string& table_id);
GetMemByTable(const std::string& collection_id);
Status
InsertVectorsNoLock(const std::string& table_id, const VectorSourcePtr& source, uint64_t lsn);
InsertVectorsNoLock(const std::string& collection_id, const VectorSourcePtr& source, uint64_t lsn);
Status
ToImmutable();
Status
ToImmutable(const std::string& table_id);
ToImmutable(const std::string& collection_id);
uint64_t
GetMaxLSN(const MemList& tables);
......
core/src/db/insert/MemTable.cpp
浏览文件 @ 68625cbf
......@@ -26,8 +26,8 @@
namespace milvus {
namespace engine {
MemTable::MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options)
: table_id_(table_id), meta_(meta), options_(options) {
MemTable::MemTable(const std::string& collection_id, const meta::MetaPtr& meta, const DBOptions& options)
: collection_id_(collection_id), meta_(meta), options_(options) {
SetIdentity("MemTable");
AddCacheInsertDataListener();
}
......@@ -42,7 +42,7 @@ MemTable::Add(const VectorSourcePtr& source) {
Status status;
if (mem_table_file_list_.empty() || current_mem_table_file->IsFull()) {
MemTableFilePtr new_mem_table_file = std::make_shared<MemTableFile>(table_id_, meta_, options_);
MemTableFilePtr new_mem_table_file = std::make_shared<MemTableFile>(collection_id_, meta_, options_);
status = new_mem_table_file->Add(source);
if (status.ok()) {
mem_table_file_list_.emplace_back(new_mem_table_file);
......@@ -62,7 +62,7 @@ MemTable::Add(const VectorSourcePtr& source) {
Status
MemTable::Delete(segment::doc_id_t doc_id) {
// Locate which table file the doc id lands in
// Locate which collection file the doc id lands in
for (auto& table_file : mem_table_file_list_) {
table_file->Delete(doc_id);
}
......@@ -74,7 +74,7 @@ MemTable::Delete(segment::doc_id_t doc_id) {
Status
MemTable::Delete(const std::vector<segment::doc_id_t>& doc_ids) {
// Locate which table file the doc id lands in
// Locate which collection file the doc id lands in
for (auto& table_file : mem_table_file_list_) {
table_file->Delete(doc_ids);
}
......@@ -122,7 +122,7 @@ MemTable::Serialize(uint64_t wal_lsn, bool apply_delete) {
}
// Update flush lsn
auto status = meta_->UpdateTableFlushLSN(table_id_, wal_lsn);
auto status = meta_->UpdateTableFlushLSN(collection_id_, wal_lsn);
if (!status.ok()) {
std::string err_msg = "Failed to write flush lsn to meta: " + status.ToString();
ENGINE_LOG_ERROR << err_msg;
......@@ -131,7 +131,7 @@ MemTable::Serialize(uint64_t wal_lsn, bool apply_delete) {
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff = end - start;
ENGINE_LOG_DEBUG << "Finished flushing for table " << table_id_ << " in " << diff.count() << " s";
ENGINE_LOG_DEBUG << "Finished flushing for collection " << collection_id_ << " in " << diff.count() << " s";
return Status::OK();
}
......@@ -143,7 +143,7 @@ MemTable::Empty() {
const std::string&
MemTable::GetTableId() const {
return table_id_;
return collection_id_;
}
size_t
......@@ -159,7 +159,7 @@ MemTable::GetCurrentMem() {
Status
MemTable::ApplyDeletes() {
// Applying deletes to other segments on disk and their corresponding cache:
// For each segment in table:
// For each segment in collection:
// Load its bloom filter
// For each id in delete list:
// If present, add the uid to segment's uid list
......@@ -173,16 +173,16 @@ MemTable::ApplyDeletes() {
// Serialize segment's deletedDoc TODO(zhiru): append directly to previous file for now, may have duplicates
// Serialize bloom filter
ENGINE_LOG_DEBUG << "Applying " << doc_ids_to_delete_.size() << " deletes in table: " << table_id_;
ENGINE_LOG_DEBUG << "Applying " << doc_ids_to_delete_.size() << " deletes in collection: " << collection_id_;
auto start_total = std::chrono::high_resolution_clock::now();
// auto start = std::chrono::high_resolution_clock::now();
std::vector<int> file_types{meta::TableFileSchema::FILE_TYPE::RAW, meta::TableFileSchema::FILE_TYPE::TO_INDEX,
meta::TableFileSchema::FILE_TYPE::BACKUP};
meta::TableFilesSchema table_files;
auto status = meta_->FilesByType(table_id_, file_types, table_files);
std::vector<int> file_types{meta::SegmentSchema::FILE_TYPE::RAW, meta::SegmentSchema::FILE_TYPE::TO_INDEX,
meta::SegmentSchema::FILE_TYPE::BACKUP};
meta::SegmentsSchema table_files;
auto status = meta_->FilesByType(collection_id_, file_types, table_files);
if (!status.ok()) {
std::string err_msg = "Failed to apply deletes: " + status.ToString();
ENGINE_LOG_ERROR << err_msg;
......@@ -209,7 +209,7 @@ MemTable::ApplyDeletes() {
}
}
meta::TableFilesSchema files_to_check;
meta::SegmentsSchema files_to_check;
for (auto& kv : ids_to_check_map) {
files_to_check.emplace_back(table_files[kv.first]);
}
......@@ -222,7 +222,7 @@ MemTable::ApplyDeletes() {
std::chrono::duration<double> diff0 = time0 - start_total;
ENGINE_LOG_DEBUG << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff0.count() << " s";
meta::TableFilesSchema table_files_to_update;
meta::SegmentsSchema table_files_to_update;
for (auto& kv : ids_to_check_map) {
auto& table_file = table_files[kv.first];
......@@ -235,7 +235,7 @@ MemTable::ApplyDeletes() {
segment::SegmentReader segment_reader(segment_dir);
auto& segment_id = table_file.segment_id_;
meta::TableFilesSchema segment_files;
meta::SegmentsSchema segment_files;
status = meta_->GetTableFilesBySegmentId(segment_id, segment_files);
if (!status.ok()) {
break;
......@@ -351,10 +351,10 @@ MemTable::ApplyDeletes() {
ENGINE_LOG_DEBUG << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff5.count()
<< " s";
// Update table file row count
// Update collection file row count
for (auto& file : segment_files) {
if (file.file_type_ == meta::TableFileSchema::RAW || file.file_type_ == meta::TableFileSchema::TO_INDEX ||
file.file_type_ == meta::TableFileSchema::INDEX || file.file_type_ == meta::TableFileSchema::BACKUP) {
if (file.file_type_ == meta::SegmentSchema::RAW || file.file_type_ == meta::SegmentSchema::TO_INDEX ||
file.file_type_ == meta::SegmentSchema::INDEX || file.file_type_ == meta::SegmentSchema::BACKUP) {
file.row_count_ -= delete_count;
table_files_to_update.emplace_back(file);
}
......@@ -362,8 +362,8 @@ MemTable::ApplyDeletes() {
auto time7 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff6 = time7 - time6;
diff6 = time6 - time5;
ENGINE_LOG_DEBUG << "Update table file row count in vector of segment: " << table_file.segment_id_ << " in "
<< diff6.count() << " s";
ENGINE_LOG_DEBUG << "Update collection file row count in vector of segment: " << table_file.segment_id_
<< " in " << diff6.count() << " s";
}
auto time7 = std::chrono::high_resolution_clock::now();
......@@ -380,9 +380,9 @@ MemTable::ApplyDeletes() {
auto end_total = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff7 = end_total - time7;
ENGINE_LOG_DEBUG << "Update deletes to meta in table " << table_id_ << " in " << diff7.count() << " s";
ENGINE_LOG_DEBUG << "Update deletes to meta in collection " << collection_id_ << " in " << diff7.count() << " s";
std::chrono::duration<double> diff_total = end_total - start_total;
ENGINE_LOG_DEBUG << "Finished applying deletes in table " << table_id_ << " in " << diff_total.count() << " s";
ENGINE_LOG_DEBUG << "Finished deletes in collection " << collection_id_ << " in " << diff_total.count() << " s";
OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_check);
......
core/src/db/insert/MemTable.h
浏览文件 @ 68625cbf
......@@ -30,7 +30,7 @@ class MemTable : public server::CacheConfigHandler {
public:
using MemTableFileList = std::vector<MemTableFilePtr>;
MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options);
MemTable(const std::string& collection_id, const meta::MetaPtr& meta, const DBOptions& options);
Status
Add(const VectorSourcePtr& source);
......@@ -74,7 +74,7 @@ class MemTable : public server::CacheConfigHandler {
ApplyDeletes();
private:
const std::string table_id_;
const std::string collection_id_;
MemTableFileList mem_table_file_list_;
......
core/src/db/insert/MemTableFile.cpp
浏览文件 @ 68625cbf
......@@ -28,8 +28,8 @@
namespace milvus {
namespace engine {
MemTableFile::MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options)
: table_id_(table_id), meta_(meta), options_(options) {
MemTableFile::MemTableFile(const std::string& collection_id, const meta::MetaPtr& meta, const DBOptions& options)
: collection_id_(collection_id), meta_(meta), options_(options) {
current_mem_ = 0;
auto status = CreateTableFile();
if (status.ok()) {
......@@ -47,8 +47,8 @@ MemTableFile::MemTableFile(const std::string& table_id, const meta::MetaPtr& met
Status
MemTableFile::CreateTableFile() {
meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = table_id_;
meta::SegmentSchema table_file_schema;
table_file_schema.collection_id_ = collection_id_;
auto status = meta_->CreateTableFile(table_file_schema);
if (status.ok()) {
table_file_schema_ = table_file_schema;
......@@ -64,9 +64,9 @@ MemTableFile::Add(const VectorSourcePtr& source) {
if (table_file_schema_.dimension_ <= 0) {
std::string err_msg =
"MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) +
", table_id = " + table_file_schema_.table_id_;
", collection_id = " + table_file_schema_.collection_id_;
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, "Not able to create table file");
return Status(DB_ERROR, "Not able to create collection file");
}
size_t single_vector_mem_size = source->SingleVectorSize(table_file_schema_.dimension_);
......@@ -162,11 +162,11 @@ MemTableFile::Serialize(uint64_t wal_lsn) {
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to serialize segment: " << table_file_schema_.segment_id_;
/* Can't mark it as to_delete because data is stored in this mem table file. Any further flush
* will try to serialize the same mem table file and it won't be able to find the directory
* to write to or update the associated table file in meta.
/* Can't mark it as to_delete because data is stored in this mem collection file. Any further flush
* will try to serialize the same mem collection file and it won't be able to find the directory
* to write to or update the associated collection file in meta.
*
table_file_schema_.file_type_ = meta::TableFileSchema::TO_DELETE;
table_file_schema_.file_type_ = meta::SegmentSchema::TO_DELETE;
meta_->UpdateTableFile(table_file_schema_);
ENGINE_LOG_DEBUG << "Failed to serialize segment, mark file: " << table_file_schema_.file_id_
<< " to to_delete";
......@@ -186,19 +186,19 @@ MemTableFile::Serialize(uint64_t wal_lsn) {
// else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP &&
table_file_schema_.engine_type_ != (int)EngineType::FAISS_BIN_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX
: meta::TableFileSchema::RAW;
table_file_schema_.file_type_ =
(size >= table_file_schema_.index_file_size_) ? meta::SegmentSchema::TO_INDEX : meta::SegmentSchema::RAW;
} else {
table_file_schema_.file_type_ = meta::TableFileSchema::RAW;
table_file_schema_.file_type_ = meta::SegmentSchema::RAW;
}
// Set table file's flush_lsn so WAL can roll back and delete garbage files which can be obtained from
// Set collection file's flush_lsn so WAL can roll back and delete garbage files which can be obtained from
// GetTableFilesByFlushLSN() in meta.
table_file_schema_.flush_lsn_ = wal_lsn;
status = meta_->UpdateTableFile(table_file_schema_);
ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::SegmentSchema::RAW) ? "raw" : "to_index")
<< " file " << table_file_schema_.file_id_ << " of size " << size << " bytes, lsn = " << wal_lsn;
// TODO(zhiru): cache
......
core/src/db/insert/MemTableFile.h
浏览文件 @ 68625cbf
......@@ -28,7 +28,7 @@ namespace engine {
class MemTableFile : public server::CacheConfigHandler {
public:
MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options);
MemTableFile(const std::string& collection_id, const meta::MetaPtr& meta, const DBOptions& options);
~MemTableFile() = default;
......@@ -66,8 +66,8 @@ class MemTableFile : public server::CacheConfigHandler {
CreateTableFile();
private:
const std::string table_id_;
meta::TableFileSchema table_file_schema_;
const std::string collection_id_;
meta::SegmentSchema table_file_schema_;
meta::MetaPtr meta_;
DBOptions options_;
size_t current_mem_;
......
core/src/db/insert/VectorSource.cpp
浏览文件 @ 68625cbf
......@@ -28,7 +28,7 @@ VectorSource::VectorSource(VectorsData vectors) : vectors_(std::move(vectors)) {
Status
VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,
const meta::TableFileSchema& table_file_schema, const size_t& num_vectors_to_add,
const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add,
size_t& num_vectors_added) {
uint64_t n = vectors_.vector_count_;
server::CollectAddMetrics metrics(n, table_file_schema.dimension_);
......
core/src/db/insert/VectorSource.h
浏览文件 @ 68625cbf
......@@ -30,7 +30,7 @@ class VectorSource {
Status
Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,
const meta::TableFileSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added);
const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added);
size_t
GetNumVectorsAdded();
......
core/src/db/meta/Meta.h
浏览文件 @ 68625cbf
......@@ -35,7 +35,7 @@ class Meta {
class CleanUpFilter {
public:
virtual bool
IsIgnored(const TableFileSchema& schema) = 0;
IsIgnored(const SegmentSchema& schema) = 0;
};
*/
......@@ -43,92 +43,92 @@ class Meta {
virtual ~Meta() = default;
virtual Status
CreateTable(TableSchema& table_schema) = 0;
CreateTable(CollectionSchema& table_schema) = 0;
virtual Status
DescribeTable(TableSchema& table_schema) = 0;
DescribeTable(CollectionSchema& table_schema) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not) = 0;
HasTable(const std::string& collection_id, bool& has_or_not) = 0;
virtual Status
AllTables(std::vector<TableSchema>& table_schema_array) = 0;
AllTables(std::vector<CollectionSchema>& table_schema_array) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
UpdateTableFlag(const std::string& collection_id, int64_t flag) = 0;
virtual Status
UpdateTableFlushLSN(const std::string& table_id, uint64_t flush_lsn) = 0;
UpdateTableFlushLSN(const std::string& collection_id, uint64_t flush_lsn) = 0;
virtual Status
GetTableFlushLSN(const std::string& table_id, uint64_t& flush_lsn) = 0;
GetTableFlushLSN(const std::string& collection_id, uint64_t& flush_lsn) = 0;
virtual Status
GetTableFilesByFlushLSN(uint64_t flush_lsn, TableFilesSchema& table_files) = 0;
GetTableFilesByFlushLSN(uint64_t flush_lsn, SegmentsSchema& table_files) = 0;
virtual Status
DropTable(const std::string& table_id) = 0;
DropTable(const std::string& collection_id) = 0;
virtual Status
DeleteTableFiles(const std::string& table_id) = 0;
DeleteTableFiles(const std::string& collection_id) = 0;
virtual Status
CreateTableFile(TableFileSchema& file_schema) = 0;
CreateTableFile(SegmentSchema& file_schema) = 0;
virtual Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) = 0;
GetTableFiles(const std::string& collection_id, const std::vector<size_t>& ids, SegmentsSchema& table_files) = 0;
virtual Status
GetTableFilesBySegmentId(const std::string& segment_id, TableFilesSchema& table_files) = 0;
GetTableFilesBySegmentId(const std::string& segment_id, SegmentsSchema& table_files) = 0;
virtual Status
UpdateTableFile(TableFileSchema& file_schema) = 0;
UpdateTableFile(SegmentSchema& file_schema) = 0;
virtual Status
UpdateTableFiles(TableFilesSchema& files) = 0;
UpdateTableFiles(SegmentsSchema& files) = 0;
virtual Status
UpdateTableFilesRowCount(TableFilesSchema& files) = 0;
UpdateTableFilesRowCount(SegmentsSchema& files) = 0;
virtual Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
UpdateTableIndex(const std::string& collection_id, const TableIndex& index) = 0;
virtual Status
UpdateTableFilesToIndex(const std::string& table_id) = 0;
UpdateTableFilesToIndex(const std::string& collection_id) = 0;
virtual Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
DescribeTableIndex(const std::string& collection_id, TableIndex& index) = 0;
virtual Status
DropTableIndex(const std::string& table_id) = 0;
DropTableIndex(const std::string& collection_id) = 0;
virtual Status
CreatePartition(const std::string& table_name, const std::string& partition_name, const std::string& tag,
CreatePartition(const std::string& collection_name, const std::string& partition_name, const std::string& tag,
uint64_t lsn) = 0;
virtual Status
DropPartition(const std::string& partition_name) = 0;
virtual Status
ShowPartitions(const std::string& table_name, std::vector<meta::TableSchema>& partition_schema_array) = 0;
ShowPartitions(const std::string& collection_name, std::vector<meta::CollectionSchema>& partition_schema_array) = 0;
virtual Status
GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0;
GetPartitionName(const std::string& collection_name, const std::string& tag, std::string& partition_name) = 0;
virtual Status
FilesToSearch(const std::string& table_id, TableFilesSchema& files) = 0;
FilesToSearch(const std::string& collection_id, SegmentsSchema& files) = 0;
virtual Status
FilesToMerge(const std::string& table_id, TableFilesSchema& files) = 0;
FilesToMerge(const std::string& collection_id, SegmentsSchema& files) = 0;
virtual Status
FilesToIndex(TableFilesSchema&) = 0;
FilesToIndex(SegmentsSchema&) = 0;
virtual Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types, TableFilesSchema& files) = 0;
FilesByType(const std::string& collection_id, const std::vector<int>& file_types, SegmentsSchema& files) = 0;
virtual Status
FilesByID(const std::vector<size_t>& ids, TableFilesSchema& files) = 0;
FilesByID(const std::vector<size_t>& ids, SegmentsSchema& files) = 0;
virtual Status
Size(uint64_t& result) = 0;
......@@ -146,7 +146,7 @@ class Meta {
DropAll() = 0;
virtual Status
Count(const std::string& table_id, uint64_t& result) = 0;
Count(const std::string& collection_id, uint64_t& result) = 0;
virtual Status
SetGlobalLastLSN(uint64_t lsn) = 0;
......
core/src/db/meta/MetaTypes.h
浏览文件 @ 68625cbf
......@@ -40,14 +40,14 @@ struct EnvironmentSchema {
uint64_t global_lsn_ = 0;
}; // EnvironmentSchema
struct TableSchema {
struct CollectionSchema {
typedef enum {
NORMAL,
TO_DELETE,
} TABLE_STATE;
size_t id_ = 0;
std::string table_id_;
std::string collection_id_;
int32_t state_ = (int)NORMAL;
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
......@@ -60,9 +60,9 @@ struct TableSchema {
std::string partition_tag_;
std::string version_ = CURRENT_VERSION;
uint64_t flush_lsn_ = 0;
}; // TableSchema
}; // CollectionSchema
struct TableFileSchema {
struct SegmentSchema {
typedef enum {
NEW,
RAW,
......@@ -75,7 +75,7 @@ struct TableFileSchema {
} FILE_TYPE;
size_t id_ = 0;
std::string table_id_;
std::string collection_id_;
std::string segment_id_;
std::string file_id_;
int32_t file_type_ = NEW;
......@@ -92,10 +92,10 @@ struct TableFileSchema {
std::string index_params_; // not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; // not persist to meta
uint64_t flush_lsn_ = 0;
}; // TableFileSchema
}; // SegmentSchema
using TableFileSchemaPtr = std::shared_ptr<meta::TableFileSchema>;
using TableFilesSchema = std::vector<TableFileSchema>;
using SegmentSchemaPtr = std::shared_ptr<meta::SegmentSchema>;
using SegmentsSchema = std::vector<SegmentSchema>;
} // namespace meta
} // namespace engine
......
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/src/db/meta/MySQLMetaImpl.h
浏览文件 @ 68625cbf
......@@ -32,92 +32,94 @@ class MySQLMetaImpl : public Meta {
~MySQLMetaImpl();
Status
CreateTable(TableSchema& table_schema) override;
CreateTable(CollectionSchema& table_schema) override;
Status
DescribeTable(TableSchema& table_schema) override;
DescribeTable(CollectionSchema& table_schema) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
HasTable(const std::string& collection_id, bool& has_or_not) override;
Status
AllTables(std::vector<TableSchema>& table_schema_array) override;
AllTables(std::vector<CollectionSchema>& table_schema_array) override;
Status
DropTable(const std::string& table_id) override;
DropTable(const std::string& collection_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
DeleteTableFiles(const std::string& collection_id) override;
Status
CreateTableFile(TableFileSchema& file_schema) override;
CreateTableFile(SegmentSchema& file_schema) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
GetTableFiles(const std::string& collection_id, const std::vector<size_t>& ids,
SegmentsSchema& table_files) override;
Status
GetTableFilesBySegmentId(const std::string& segment_id, TableFilesSchema& table_files) override;
GetTableFilesBySegmentId(const std::string& segment_id, SegmentsSchema& table_files) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
UpdateTableIndex(const std::string& collection_id, const TableIndex& index) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
UpdateTableFlag(const std::string& collection_id, int64_t flag) override;
Status
UpdateTableFlushLSN(const std::string& table_id, uint64_t flush_lsn) override;
UpdateTableFlushLSN(const std::string& collection_id, uint64_t flush_lsn) override;
Status
GetTableFlushLSN(const std::string& table_id, uint64_t& flush_lsn) override;
GetTableFlushLSN(const std::string& collection_id, uint64_t& flush_lsn) override;
Status
GetTableFilesByFlushLSN(uint64_t flush_lsn, TableFilesSchema& table_files) override;
GetTableFilesByFlushLSN(uint64_t flush_lsn, SegmentsSchema& table_files) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
UpdateTableFile(SegmentSchema& file_schema) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
UpdateTableFilesToIndex(const std::string& collection_id) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
UpdateTableFiles(SegmentsSchema& files) override;
Status
UpdateTableFilesRowCount(TableFilesSchema& files) override;
UpdateTableFilesRowCount(SegmentsSchema& files) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
DescribeTableIndex(const std::string& collection_id, TableIndex& index) override;
Status
DropTableIndex(const std::string& table_id) override;
DropTableIndex(const std::string& collection_id) override;
Status
CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag,
CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag,
uint64_t lsn) override;
Status
DropPartition(const std::string& partition_name) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) override;
ShowPartitions(const std::string& collection_id,
std::vector<meta::CollectionSchema>& partition_schema_array) override;
Status
GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
GetPartitionName(const std::string& collection_id, const std::string& tag, std::string& partition_name) override;
Status
FilesToSearch(const std::string& table_id, TableFilesSchema& files) override;
FilesToSearch(const std::string& collection_id, SegmentsSchema& files) override;
Status
FilesToMerge(const std::string& table_id, TableFilesSchema& files) override;
FilesToMerge(const std::string& collection_id, SegmentsSchema& files) override;
Status
FilesToIndex(TableFilesSchema&) override;
FilesToIndex(SegmentsSchema&) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types, TableFilesSchema& files) override;
FilesByType(const std::string& collection_id, const std::vector<int>& file_types, SegmentsSchema& files) override;
Status
FilesByID(const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
FilesByID(const std::vector<size_t>& ids, SegmentsSchema& table_files) override;
Status
Archive() override;
......@@ -135,7 +137,7 @@ class MySQLMetaImpl : public Meta {
DropAll() override;
Status
Count(const std::string& table_id, uint64_t& result) override;
Count(const std::string& collection_id, uint64_t& result) override;
Status
SetGlobalLastLSN(uint64_t lsn) override;
......@@ -147,7 +149,7 @@ class MySQLMetaImpl : public Meta {
Status
NextFileId(std::string& file_id);
Status
NextTableId(std::string& table_id);
NextTableId(std::string& collection_id);
Status
DiscardFiles(int64_t to_discard_size);
......
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/src/db/meta/SqliteMetaImpl.h
浏览文件 @ 68625cbf
......@@ -31,92 +31,94 @@ class SqliteMetaImpl : public Meta {
~SqliteMetaImpl();
Status
CreateTable(TableSchema& table_schema) override;
CreateTable(CollectionSchema& table_schema) override;
Status
DescribeTable(TableSchema& table_schema) override;
DescribeTable(CollectionSchema& table_schema) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
HasTable(const std::string& collection_id, bool& has_or_not) override;
Status
AllTables(std::vector<TableSchema>& table_schema_array) override;
AllTables(std::vector<CollectionSchema>& table_schema_array) override;
Status
DropTable(const std::string& table_id) override;
DropTable(const std::string& collection_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
DeleteTableFiles(const std::string& collection_id) override;
Status
CreateTableFile(TableFileSchema& file_schema) override;
CreateTableFile(SegmentSchema& file_schema) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
GetTableFiles(const std::string& collection_id, const std::vector<size_t>& ids,
SegmentsSchema& table_files) override;
Status
GetTableFilesBySegmentId(const std::string& segment_id, TableFilesSchema& table_files) override;
GetTableFilesBySegmentId(const std::string& segment_id, SegmentsSchema& table_files) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
UpdateTableIndex(const std::string& collection_id, const TableIndex& index) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
UpdateTableFlag(const std::string& collection_id, int64_t flag) override;
Status
UpdateTableFlushLSN(const std::string& table_id, uint64_t flush_lsn) override;
UpdateTableFlushLSN(const std::string& collection_id, uint64_t flush_lsn) override;
Status
GetTableFlushLSN(const std::string& table_id, uint64_t& flush_lsn) override;
GetTableFlushLSN(const std::string& collection_id, uint64_t& flush_lsn) override;
Status
GetTableFilesByFlushLSN(uint64_t flush_lsn, TableFilesSchema& table_files) override;
GetTableFilesByFlushLSN(uint64_t flush_lsn, SegmentsSchema& table_files) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
UpdateTableFile(SegmentSchema& file_schema) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
UpdateTableFilesToIndex(const std::string& collection_id) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
UpdateTableFiles(SegmentsSchema& files) override;
Status
UpdateTableFilesRowCount(TableFilesSchema& files) override;
UpdateTableFilesRowCount(SegmentsSchema& files) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
DescribeTableIndex(const std::string& collection_id, TableIndex& index) override;
Status
DropTableIndex(const std::string& table_id) override;
DropTableIndex(const std::string& collection_id) override;
Status
CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag,
CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag,
uint64_t lsn) override;
Status
DropPartition(const std::string& partition_name) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) override;
ShowPartitions(const std::string& collection_id,
std::vector<meta::CollectionSchema>& partition_schema_array) override;
Status
GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
GetPartitionName(const std::string& collection_id, const std::string& tag, std::string& partition_name) override;
Status
FilesToSearch(const std::string& table_id, TableFilesSchema& files) override;
FilesToSearch(const std::string& collection_id, SegmentsSchema& files) override;
Status
FilesToMerge(const std::string& table_id, TableFilesSchema& files) override;
FilesToMerge(const std::string& collection_id, SegmentsSchema& files) override;
Status
FilesToIndex(TableFilesSchema&) override;
FilesToIndex(SegmentsSchema&) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types, TableFilesSchema& files) override;
FilesByType(const std::string& collection_id, const std::vector<int>& file_types, SegmentsSchema& files) override;
Status
FilesByID(const std::vector<size_t>& ids, TableFilesSchema& files) override;
FilesByID(const std::vector<size_t>& ids, SegmentsSchema& files) override;
Status
Size(uint64_t& result) override;
......@@ -134,7 +136,7 @@ class SqliteMetaImpl : public Meta {
DropAll() override;
Status
Count(const std::string& table_id, uint64_t& result) override;
Count(const std::string& collection_id, uint64_t& result) override;
Status
SetGlobalLastLSN(uint64_t lsn) override;
......@@ -146,7 +148,7 @@ class SqliteMetaImpl : public Meta {
Status
NextFileId(std::string& file_id);
Status
NextTableId(std::string& table_id);
NextTableId(std::string& collection_id);
Status
DiscardFiles(int64_t to_discard_size);
......
core/src/db/wal/WalBuffer.cpp
浏览文件 @ 68625cbf
......@@ -185,7 +185,7 @@ MXLogBuffer::SurplusSpace() {
uint32_t
MXLogBuffer::RecordSize(const MXLogRecord& record) {
return SizeOfMXLogRecordHeader + (uint32_t)record.table_id.size() + (uint32_t)record.partition_tag.size() +
return SizeOfMXLogRecordHeader + (uint32_t)record.collection_id.size() + (uint32_t)record.partition_tag.size() +
record.length * (uint32_t)sizeof(IDNumber) + record.data_size;
}
......@@ -218,7 +218,7 @@ MXLogBuffer::Append(MXLogRecord& record) {
MXLogRecordHeader head;
BuildLsn(mxlog_buffer_writer_.file_no, mxlog_buffer_writer_.buf_offset + (uint32_t)record_size, head.mxl_lsn);
head.mxl_type = (uint8_t)record.type;
head.table_id_size = (uint16_t)record.table_id.size();
head.table_id_size = (uint16_t)record.collection_id.size();
head.partition_tag_size = (uint16_t)record.partition_tag.size();
head.vector_num = record.length;
head.data_size = record.data_size;
......@@ -226,9 +226,9 @@ MXLogBuffer::Append(MXLogRecord& record) {
memcpy(current_write_buf + current_write_offset, &head, SizeOfMXLogRecordHeader);
current_write_offset += SizeOfMXLogRecordHeader;
if (!record.table_id.empty()) {
memcpy(current_write_buf + current_write_offset, record.table_id.data(), record.table_id.size());
current_write_offset += record.table_id.size();
if (!record.collection_id.empty()) {
memcpy(current_write_buf + current_write_offset, record.collection_id.data(), record.collection_id.size());
current_write_offset += record.collection_id.size();
}
if (!record.partition_tag.empty()) {
......@@ -307,10 +307,10 @@ MXLogBuffer::Next(const uint64_t last_applied_lsn, MXLogRecord& record) {
current_read_offset += SizeOfMXLogRecordHeader;
if (head->table_id_size != 0) {
record.table_id.assign(current_read_buf + current_read_offset, head->table_id_size);
record.collection_id.assign(current_read_buf + current_read_offset, head->table_id_size);
current_read_offset += head->table_id_size;
} else {
record.table_id = "";
record.collection_id = "";
}
if (head->partition_tag_size != 0) {
......
core/src/db/wal/WalDefinations.h
浏览文件 @ 68625cbf
......@@ -22,7 +22,7 @@ namespace milvus {
namespace engine {
namespace wal {
using TableSchemaPtr = std::shared_ptr<milvus::engine::meta::TableSchema>;
using TableSchemaPtr = std::shared_ptr<milvus::engine::meta::CollectionSchema>;
using TableMetaPtr = std::shared_ptr<std::unordered_map<std::string, TableSchemaPtr> >;
#define UNIT_MB (1024 * 1024)
......@@ -33,7 +33,7 @@ enum class MXLogType { InsertBinary, InsertVector, Delete, Update, Flush, None }
struct MXLogRecord {
uint64_t lsn;
MXLogType type;
std::string table_id;
std::string collection_id;
std::string partition_tag;
uint32_t length;
const IDNumber* ids;
......
core/src/db/wal/WalManager.cpp
浏览文件 @ 68625cbf
......@@ -61,7 +61,7 @@ WalManager::Init(const meta::MetaPtr& meta) {
if (meta != nullptr) {
meta->GetGlobalLastLSN(recovery_start);
std::vector<meta::TableSchema> table_schema_array;
std::vector<meta::CollectionSchema> table_schema_array;
auto status = meta->AllTables(table_schema_array);
if (!status.ok()) {
return WAL_META_ERROR;
......@@ -89,7 +89,7 @@ WalManager::Init(const meta::MetaPtr& meta) {
for (auto& schema : table_schema_array) {
TableLsn tb_lsn = {schema.flush_lsn_, applied_lsn};
tables_[schema.table_id_] = tb_lsn;
tables_[schema.collection_id_] = tb_lsn;
}
}
}
......@@ -140,7 +140,7 @@ WalManager::GetNextRecovery(MXLogRecord& record) {
// background thread has not started.
// so, needn't lock here.
auto it = tables_.find(record.table_id);
auto it = tables_.find(record.collection_id);
if (it != tables_.end()) {
if (it->second.flush_lsn < record.lsn) {
break;
......@@ -162,11 +162,11 @@ WalManager::GetNextRecord(MXLogRecord& record) {
if (p_buffer_->GetReadLsn() >= flush_info_.lsn_) {
// can exec flush requirement
record.type = MXLogType::Flush;
record.table_id = flush_info_.table_id_;
record.collection_id = flush_info_.collection_id_;
record.lsn = flush_info_.lsn_;
flush_info_.Clear();
WAL_LOG_INFO << "record flush table " << record.table_id << " lsn " << record.lsn;
WAL_LOG_INFO << "record flush collection " << record.collection_id << " lsn " << record.lsn;
return true;
}
}
......@@ -187,7 +187,7 @@ WalManager::GetNextRecord(MXLogRecord& record) {
}
std::lock_guard<std::mutex> lck(mutex_);
auto it = tables_.find(record.table_id);
auto it = tables_.find(record.collection_id);
if (it != tables_.end()) {
if (it->second.flush_lsn < record.lsn) {
break;
......@@ -195,41 +195,42 @@ WalManager::GetNextRecord(MXLogRecord& record) {
}
}
WAL_LOG_INFO << "record type " << (int32_t)record.type << " table " << record.table_id << " lsn " << record.lsn;
WAL_LOG_INFO << "record type " << (int32_t)record.type << " collection " << record.collection_id << " lsn "
<< record.lsn;
return error_code;
}
uint64_t
WalManager::CreateTable(const std::string& table_id) {
WAL_LOG_INFO << "create table " << table_id << " " << last_applied_lsn_;
WalManager::CreateTable(const std::string& collection_id) {
WAL_LOG_INFO << "create collection " << collection_id << " " << last_applied_lsn_;
std::lock_guard<std::mutex> lck(mutex_);
uint64_t applied_lsn = last_applied_lsn_;
tables_[table_id] = {applied_lsn, applied_lsn};
tables_[collection_id] = {applied_lsn, applied_lsn};
return applied_lsn;
}
void
WalManager::DropTable(const std::string& table_id) {
WAL_LOG_INFO << "drop table " << table_id;
WalManager::DropTable(const std::string& collection_id) {
WAL_LOG_INFO << "drop collection " << collection_id;
std::lock_guard<std::mutex> lck(mutex_);
tables_.erase(table_id);
tables_.erase(collection_id);
}
void
WalManager::TableFlushed(const std::string& table_id, uint64_t lsn) {
WalManager::TableFlushed(const std::string& collection_id, uint64_t lsn) {
std::unique_lock<std::mutex> lck(mutex_);
auto it = tables_.find(table_id);
auto it = tables_.find(collection_id);
if (it != tables_.end()) {
it->second.flush_lsn = lsn;
}
lck.unlock();
WAL_LOG_INFO << table_id << " is flushed by lsn " << lsn;
WAL_LOG_INFO << collection_id << " is flushed by lsn " << lsn;
}
template <typename T>
bool
WalManager::Insert(const std::string& table_id, const std::string& partition_tag, const IDNumbers& vector_ids,
WalManager::Insert(const std::string& collection_id, const std::string& partition_tag, const IDNumbers& vector_ids,
const std::vector<T>& vectors) {
MXLogType log_type;
if (std::is_same<T, float>::value) {
......@@ -247,11 +248,11 @@ WalManager::Insert(const std::string& table_id, const std::string& partition_tag
}
size_t dim = vectors.size() / vector_num;
size_t unit_size = dim * sizeof(T) + sizeof(IDNumber);
size_t head_size = SizeOfMXLogRecordHeader + table_id.length() + partition_tag.length();
size_t head_size = SizeOfMXLogRecordHeader + collection_id.length() + partition_tag.length();
MXLogRecord record;
record.type = log_type;
record.table_id = table_id;
record.collection_id = collection_id;
record.partition_tag = partition_tag;
uint64_t new_lsn = 0;
......@@ -283,19 +284,19 @@ WalManager::Insert(const std::string& table_id, const std::string& partition_tag
std::unique_lock<std::mutex> lck(mutex_);
last_applied_lsn_ = new_lsn;
auto it = tables_.find(table_id);
auto it = tables_.find(collection_id);
if (it != tables_.end()) {
it->second.wal_lsn = new_lsn;
}
lck.unlock();
WAL_LOG_INFO << table_id << " insert in part " << partition_tag << " with lsn " << new_lsn;
WAL_LOG_INFO << collection_id << " insert in part " << partition_tag << " with lsn " << new_lsn;
return p_meta_handler_->SetMXLogInternalMeta(new_lsn);
}
bool
WalManager::DeleteById(const std::string& table_id, const IDNumbers& vector_ids) {
WalManager::DeleteById(const std::string& collection_id, const IDNumbers& vector_ids) {
size_t vector_num = vector_ids.size();
if (vector_num == 0) {
WAL_LOG_ERROR << "The ids is empty.";
......@@ -303,11 +304,11 @@ WalManager::DeleteById(const std::string& table_id, const IDNumbers& vector_ids)
}
size_t unit_size = sizeof(IDNumber);
size_t head_size = SizeOfMXLogRecordHeader + table_id.length();
size_t head_size = SizeOfMXLogRecordHeader + collection_id.length();
MXLogRecord record;
record.type = MXLogType::Delete;
record.table_id = table_id;
record.collection_id = collection_id;
record.partition_tag = "";
uint64_t new_lsn = 0;
......@@ -335,26 +336,26 @@ WalManager::DeleteById(const std::string& table_id, const IDNumbers& vector_ids)
std::unique_lock<std::mutex> lck(mutex_);
last_applied_lsn_ = new_lsn;
auto it = tables_.find(table_id);
auto it = tables_.find(collection_id);
if (it != tables_.end()) {
it->second.wal_lsn = new_lsn;
}
lck.unlock();
WAL_LOG_INFO << table_id << " delete rows by id, lsn " << new_lsn;
WAL_LOG_INFO << collection_id << " delete rows by id, lsn " << new_lsn;
return p_meta_handler_->SetMXLogInternalMeta(new_lsn);
}
uint64_t
WalManager::Flush(const std::string& table_id) {
WalManager::Flush(const std::string& collection_id) {
std::lock_guard<std::mutex> lck(mutex_);
// At most one flush requirement is waiting at any time.
// Otherwise, flush_info_ should be modified to a list.
__glibcxx_assert(!flush_info_.IsValid());
uint64_t lsn = 0;
if (table_id.empty()) {
if (collection_id.empty()) {
// flush all tables
for (auto& it : tables_) {
if (it.second.wal_lsn > it.second.flush_lsn) {
......@@ -364,8 +365,8 @@ WalManager::Flush(const std::string& table_id) {
}
} else {
// flush one table
auto it = tables_.find(table_id);
// flush one collection
auto it = tables_.find(collection_id);
if (it != tables_.end()) {
if (it->second.wal_lsn > it->second.flush_lsn) {
lsn = it->second.wal_lsn;
......@@ -374,11 +375,11 @@ WalManager::Flush(const std::string& table_id) {
}
if (lsn != 0) {
flush_info_.table_id_ = table_id;
flush_info_.collection_id_ = collection_id;
flush_info_.lsn_ = lsn;
}
WAL_LOG_INFO << table_id << " want to be flush, lsn " << lsn;
WAL_LOG_INFO << collection_id << " want to be flush, lsn " << lsn;
return lsn;
}
......@@ -391,12 +392,12 @@ WalManager::RemoveOldFiles(uint64_t flushed_lsn) {
}
template bool
WalManager::Insert<float>(const std::string& table_id, const std::string& partition_tag, const IDNumbers& vector_ids,
const std::vector<float>& vectors);
WalManager::Insert<float>(const std::string& collection_id, const std::string& partition_tag,
const IDNumbers& vector_ids, const std::vector<float>& vectors);
template bool
WalManager::Insert<uint8_t>(const std::string& table_id, const std::string& partition_tag, const IDNumbers& vector_ids,
const std::vector<uint8_t>& vectors);
WalManager::Insert<uint8_t>(const std::string& collection_id, const std::string& partition_tag,
const IDNumbers& vector_ids, const std::vector<uint8_t>& vectors);
} // namespace wal
} // namespace engine
......
core/src/db/wal/WalManager.h
浏览文件 @ 68625cbf
......@@ -57,57 +57,57 @@ class WalManager {
GetNextRecord(MXLogRecord& record);
/*
* Create table
* @param table_id: table id
* Create collection
* @param collection_id: collection id
* @retval lsn
*/
uint64_t
CreateTable(const std::string& table_id);
CreateTable(const std::string& collection_id);
/*
* Drop table
* @param table_id: table id
* Drop collection
* @param collection_id: collection id
* @retval none
*/
void
DropTable(const std::string& table_id);
DropTable(const std::string& collection_id);
/*
* Table is flushed
* @param table_id: table id
* Collection is flushed
* @param collection_id: collection id
* @param lsn: flushed lsn
*/
void
TableFlushed(const std::string& table_id, uint64_t lsn);
TableFlushed(const std::string& collection_id, uint64_t lsn);
/*
* Insert
* @param table_id: table id
* @param table_id: partition tag
* @param collection_id: collection id
* @param collection_id: partition tag
* @param vector_ids: vector ids
* @param vectors: vectors
*/
template <typename T>
bool
Insert(const std::string& table_id, const std::string& partition_tag, const IDNumbers& vector_ids,
Insert(const std::string& collection_id, const std::string& partition_tag, const IDNumbers& vector_ids,
const std::vector<T>& vectors);
/*
* Insert
* @param table_id: table id
* @param collection_id: collection id
* @param vector_ids: vector ids
*/
bool
DeleteById(const std::string& table_id, const IDNumbers& vector_ids);
DeleteById(const std::string& collection_id, const IDNumbers& vector_ids);
/*
* Get flush lsn
* @param table_id: table id (empty means all tables)
* @param collection_id: collection id (empty means all tables)
* @retval if there is something not flushed, return lsn;
* else, return 0
*/
uint64_t
Flush(const std::string& table_id = "");
Flush(const std::string& collection_id = "");
void
RemoveOldFiles(uint64_t flushed_lsn);
......@@ -131,7 +131,7 @@ class WalManager {
// if multi-thread call Flush(), use list
struct FlushInfo {
std::string table_id_;
std::string collection_id_;
uint64_t lsn_ = 0;
bool
......@@ -147,12 +147,12 @@ class WalManager {
};
extern template bool
WalManager::Insert<float>(const std::string& table_id, const std::string& partition_tag, const IDNumbers& vector_ids,
const std::vector<float>& vectors);
WalManager::Insert<float>(const std::string& collection_id, const std::string& partition_tag,
const IDNumbers& vector_ids, const std::vector<float>& vectors);
extern template bool
WalManager::Insert<uint8_t>(const std::string& table_id, const std::string& partition_tag, const IDNumbers& vector_ids,
const std::vector<uint8_t>& vectors);
WalManager::Insert<uint8_t>(const std::string& collection_id, const std::string& partition_tag,
const IDNumbers& vector_ids, const std::vector<uint8_t>& vectors);
} // namespace wal
} // namespace engine
......
core/src/index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.cpp
浏览文件 @ 68625cbf
......@@ -23,7 +23,7 @@ AdapterMgr::GetAdapter(const IndexType type) {
RegisterAdapter();
try {
return table_.at(type)();
return collection_.at(type)();
} catch (...) {
KNOWHERE_THROW_MSG("Can not find this type of confadapter");
}
......
core/src/index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.h
浏览文件 @ 68625cbf
......@@ -26,7 +26,7 @@ class AdapterMgr {
template <typename T>
struct register_t {
explicit register_t(const IndexType type) {
AdapterMgr::GetInstance().table_[type] = ([] { return std::make_shared<T>(); });
AdapterMgr::GetInstance().collection_[type] = ([] { return std::make_shared<T>(); });
}
};
......@@ -44,7 +44,7 @@ class AdapterMgr {
protected:
bool init_ = false;
std::unordered_map<IndexType, std::function<ConfAdapterPtr()>> table_;
std::unordered_map<IndexType, std::function<ConfAdapterPtr()>> collection_;
};
} // namespace knowhere
......
core/src/metrics/Metrics.h
浏览文件 @ 68625cbf
......@@ -186,11 +186,11 @@ class CollectDurationMetrics : CollectMetricsBase {
~CollectDurationMetrics() {
auto total_time = TimeFromBegine();
switch (index_type_) {
case engine::meta::TableFileSchema::RAW: {
case engine::meta::SegmentSchema::RAW: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
case engine::meta::TableFileSchema::TO_INDEX: {
case engine::meta::SegmentSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
......@@ -214,11 +214,11 @@ class CollectSearchTaskMetrics : CollectMetricsBase {
~CollectSearchTaskMetrics() {
auto total_time = TimeFromBegine();
switch (index_type_) {
case engine::meta::TableFileSchema::RAW: {
case engine::meta::SegmentSchema::RAW: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
case engine::meta::TableFileSchema::TO_INDEX: {
case engine::meta::SegmentSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
......
core/src/metrics/prometheus/PrometheusMetrics.h
浏览文件 @ 68625cbf
......@@ -440,7 +440,7 @@ class PrometheusMetrics : public MetricsBase {
prometheus::Histogram& all_build_index_duration_seconds_histogram_ =
all_build_index_duration_seconds_.Add({}, BucketBoundaries{2e6, 4e6, 6e6, 8e6, 1e7});
// record duration of merging mem table
// record duration of merging mem collection
prometheus::Family<prometheus::Histogram>& mem_table_merge_duration_seconds_ =
prometheus::BuildHistogram()
.Name("mem_table_merge_duration_microseconds")
......
core/src/scheduler/Definition.h
浏览文件 @ 68625cbf
......@@ -27,8 +27,8 @@
namespace milvus {
namespace scheduler {
using TableFileSchemaPtr = engine::meta::TableFileSchemaPtr;
using TableFileSchema = engine::meta::TableFileSchema;
using SegmentSchemaPtr = engine::meta::SegmentSchemaPtr;
using SegmentSchema = engine::meta::SegmentSchema;
using ExecutionEnginePtr = engine::ExecutionEnginePtr;
using EngineFactory = engine::EngineFactory;
......
core/src/scheduler/TaskTable.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/src/scheduler/TaskTable.h
浏览文件 @ 68625cbf
此差异已折叠。
core/src/scheduler/job/BuildIndexJob.cpp
浏览文件 @ 68625cbf
......@@ -25,7 +25,7 @@ BuildIndexJob::BuildIndexJob(engine::meta::MetaPtr meta_ptr, engine::DBOptions o
}
bool
BuildIndexJob::AddToIndexFiles(const engine::meta::TableFileSchemaPtr& to_index_file) {
BuildIndexJob::AddToIndexFiles(const engine::meta::SegmentSchemaPtr& to_index_file) {
std::unique_lock<std::mutex> lock(mutex_);
if (to_index_file == nullptr || to_index_files_.find(to_index_file->id_) != to_index_files_.end()) {
return false;
......
core/src/scheduler/job/BuildIndexJob.h
浏览文件 @ 68625cbf
......@@ -29,10 +29,10 @@
namespace milvus {
namespace scheduler {
using engine::meta::TableFileSchemaPtr;
using engine::meta::SegmentSchemaPtr;
using Id2ToIndexMap = std::unordered_map<size_t, TableFileSchemaPtr>;
using Id2ToTableFileMap = std::unordered_map<size_t, TableFileSchema>;
using Id2ToIndexMap = std::unordered_map<size_t, SegmentSchemaPtr>;
using Id2ToTableFileMap = std::unordered_map<size_t, SegmentSchema>;
class BuildIndexJob : public Job, public server::CacheConfigHandler {
public:
......@@ -42,7 +42,7 @@ class BuildIndexJob : public Job, public server::CacheConfigHandler {
public:
bool
AddToIndexFiles(const TableFileSchemaPtr& to_index_file);
AddToIndexFiles(const SegmentSchemaPtr& to_index_file);
void
WaitBuildIndexFinish();
......
core/src/scheduler/job/DeleteJob.cpp
浏览文件 @ 68625cbf
......@@ -16,9 +16,9 @@
namespace milvus {
namespace scheduler {
DeleteJob::DeleteJob(std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource)
DeleteJob::DeleteJob(std::string collection_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource)
: Job(JobType::DELETE),
table_id_(std::move(table_id)),
collection_id_(std::move(collection_id)),
meta_ptr_(std::move(meta_ptr)),
num_resource_(num_resource) {
}
......@@ -27,7 +27,7 @@ void
DeleteJob::WaitAndDelete() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] { return done_resource == num_resource_; });
meta_ptr_->DeleteTableFiles(table_id_);
meta_ptr_->DeleteTableFiles(collection_id_);
}
void
......@@ -42,7 +42,7 @@ DeleteJob::ResourceDone() {
json
DeleteJob::Dump() const {
json ret{
{"table_id", table_id_},
{"collection_id", collection_id_},
{"number_of_resource", num_resource_},
{"number_of_done", done_resource},
};
......
core/src/scheduler/job/DeleteJob.h
浏览文件 @ 68625cbf
此差异已折叠。
core/src/scheduler/job/SearchJob.h
浏览文件 @ 68625cbf
此差异已折叠。
core/src/scheduler/task/TestTask.h
浏览文件 @ 68625cbf
此差异已折叠。
core/src/server/DBWrapper.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/src/utils/ValidationUtil.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/src/utils/ValidationUtil.h
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_db.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_db_mysql.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_delete.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_mem.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_meta.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_misc.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/db/test_wal.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/server/test_rpc.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/server/test_util.cpp
浏览文件 @ 68625cbf
此差异已折叠。
core/unittest/server/test_web.cpp
浏览文件 @ 68625cbf
此差异已折叠。
sdk/grpc/ClientProxy.cpp
浏览文件 @ 68625cbf
此差异已折叠。
sdk/grpc/GrpcClient.cpp
浏览文件 @ 68625cbf
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或