Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
qq_35024513
milvus
提交
9d5ddb28
milvus
项目概览
qq_35024513
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
14
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
9d5ddb28
编写于
9月 03, 2020
作者:
C
cqy123456
提交者:
GitHub
9月 03, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
#3576 optimize code structure (#3581)
Signed-off-by:
N
cqy
<
yaya645@126.com
>
上级
bf5bdf25
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
49 addition
and
50 deletion
+49
-50
core/src/db/Types.h
core/src/db/Types.h
+3
-0
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+14
-17
core/src/index/archive/KnowhereResource.cpp
core/src/index/archive/KnowhereResource.cpp
+32
-0
core/src/server/DBWrapper.cpp
core/src/server/DBWrapper.cpp
+0
-33
未找到文件。
core/src/db/Types.h
浏览文件 @
9d5ddb28
...
...
@@ -60,6 +60,9 @@ using VectorDistances = std::vector<VectorDistance>;
using
ResultIds
=
std
::
vector
<
faiss
::
Index
::
idx_t
>
;
using
ResultDistances
=
std
::
vector
<
faiss
::
Index
::
distance_t
>
;
using
ConCurrentBitset
=
faiss
::
ConcurrentBitset
;
using
ConCurrentBitsetPtr
=
faiss
::
ConcurrentBitsetPtr
;
///////////////////////////////////////////////////////////////////////////////////////////////////
enum
class
DataType
{
NONE
=
0
,
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
9d5ddb28
...
...
@@ -300,9 +300,9 @@ ExecutionEngineImpl::VecSearch(milvus::engine::ExecutionEngineContext& context,
Status
ExecutionEngineImpl
::
Search
(
ExecutionEngineContext
&
context
)
{
try
{
faiss
::
Conc
urrentBitsetPtr
bitset
;
ConC
urrentBitsetPtr
bitset
;
std
::
string
vector_placeholder
;
faiss
::
Conc
urrentBitsetPtr
list
;
ConC
urrentBitsetPtr
list
;
SegmentPtr
segment_ptr
;
segment_reader_
->
GetSegment
(
segment_ptr
);
...
...
@@ -360,13 +360,12 @@ ExecutionEngineImpl::Search(ExecutionEngineContext& context) {
}
Status
ExecutionEngineImpl
::
ExecBinaryQuery
(
const
milvus
::
query
::
GeneralQueryPtr
&
general_query
,
faiss
::
ConcurrentBitsetPtr
&
bitset
,
ExecutionEngineImpl
::
ExecBinaryQuery
(
const
milvus
::
query
::
GeneralQueryPtr
&
general_query
,
ConCurrentBitsetPtr
&
bitset
,
std
::
unordered_map
<
std
::
string
,
DataType
>&
attr_type
,
std
::
string
&
vector_placeholder
)
{
Status
status
=
Status
::
OK
();
if
(
general_query
->
leaf
==
nullptr
)
{
faiss
::
Conc
urrentBitsetPtr
left_bitset
,
right_bitset
;
ConC
urrentBitsetPtr
left_bitset
,
right_bitset
;
if
(
general_query
->
bin
->
left_query
!=
nullptr
)
{
status
=
ExecBinaryQuery
(
general_query
->
bin
->
left_query
,
left_bitset
,
attr_type
,
vector_placeholder
);
if
(
!
status
.
ok
())
{
...
...
@@ -412,16 +411,16 @@ ExecutionEngineImpl::ExecBinaryQuery(const milvus::query::GeneralQueryPtr& gener
return
status
;
}
else
{
if
(
general_query
->
leaf
->
term_query
!=
nullptr
)
{
bitset
=
std
::
make_shared
<
faiss
::
Conc
urrentBitset
>
(
entity_count_
);
bitset
=
std
::
make_shared
<
ConC
urrentBitset
>
(
entity_count_
);
STATUS_CHECK
(
ProcessTermQuery
(
bitset
,
general_query
->
leaf
->
term_query
,
attr_type
));
}
if
(
general_query
->
leaf
->
range_query
!=
nullptr
)
{
bitset
=
std
::
make_shared
<
faiss
::
Conc
urrentBitset
>
(
entity_count_
);
bitset
=
std
::
make_shared
<
ConC
urrentBitset
>
(
entity_count_
);
STATUS_CHECK
(
ProcessRangeQuery
(
attr_type
,
bitset
,
general_query
->
leaf
->
range_query
));
}
if
(
!
general_query
->
leaf
->
vector_placeholder
.
empty
())
{
// skip vector query
bitset
=
std
::
make_shared
<
faiss
::
Conc
urrentBitset
>
(
entity_count_
,
255
);
bitset
=
std
::
make_shared
<
ConC
urrentBitset
>
(
entity_count_
,
255
);
vector_placeholder
=
general_query
->
leaf
->
vector_placeholder
;
}
}
...
...
@@ -430,8 +429,7 @@ ExecutionEngineImpl::ExecBinaryQuery(const milvus::query::GeneralQueryPtr& gener
template
<
typename
T
>
Status
ProcessIndexedTermQuery
(
faiss
::
ConcurrentBitsetPtr
&
bitset
,
knowhere
::
IndexPtr
&
index_ptr
,
milvus
::
json
&
term_values_json
)
{
ProcessIndexedTermQuery
(
ConCurrentBitsetPtr
&
bitset
,
knowhere
::
IndexPtr
&
index_ptr
,
milvus
::
json
&
term_values_json
)
{
try
{
auto
T_index
=
std
::
dynamic_pointer_cast
<
knowhere
::
StructuredIndexSort
<
T
>>
(
index_ptr
);
if
(
not
T_index
)
{
...
...
@@ -453,7 +451,7 @@ ProcessIndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, knowhere::IndexPtr&
}
Status
ExecutionEngineImpl
::
IndexedTermQuery
(
faiss
::
Conc
urrentBitsetPtr
&
bitset
,
const
std
::
string
&
field_name
,
ExecutionEngineImpl
::
IndexedTermQuery
(
ConC
urrentBitsetPtr
&
bitset
,
const
std
::
string
&
field_name
,
const
DataType
&
data_type
,
milvus
::
json
&
term_values_json
)
{
SegmentPtr
segment_ptr
;
segment_reader_
->
GetSegment
(
segment_ptr
);
...
...
@@ -493,7 +491,7 @@ ExecutionEngineImpl::IndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, const
}
Status
ExecutionEngineImpl
::
ProcessTermQuery
(
faiss
::
Conc
urrentBitsetPtr
&
bitset
,
const
query
::
TermQueryPtr
&
term_query
,
ExecutionEngineImpl
::
ProcessTermQuery
(
ConC
urrentBitsetPtr
&
bitset
,
const
query
::
TermQueryPtr
&
term_query
,
std
::
unordered_map
<
std
::
string
,
DataType
>&
attr_type
)
{
try
{
auto
term_query_json
=
term_query
->
json_obj
;
...
...
@@ -520,8 +518,7 @@ ExecutionEngineImpl::ProcessTermQuery(faiss::ConcurrentBitsetPtr& bitset, const
template
<
typename
T
>
Status
ProcessIndexedRangeQuery
(
faiss
::
ConcurrentBitsetPtr
&
bitset
,
knowhere
::
IndexPtr
&
index_ptr
,
milvus
::
json
&
range_values_json
)
{
ProcessIndexedRangeQuery
(
ConCurrentBitsetPtr
&
bitset
,
knowhere
::
IndexPtr
&
index_ptr
,
milvus
::
json
&
range_values_json
)
{
try
{
auto
T_index
=
std
::
dynamic_pointer_cast
<
knowhere
::
StructuredIndexSort
<
T
>>
(
index_ptr
);
...
...
@@ -543,7 +540,7 @@ ProcessIndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, knowhere::IndexPtr&
}
Status
ExecutionEngineImpl
::
IndexedRangeQuery
(
faiss
::
Conc
urrentBitsetPtr
&
bitset
,
const
DataType
&
data_type
,
ExecutionEngineImpl
::
IndexedRangeQuery
(
ConC
urrentBitsetPtr
&
bitset
,
const
DataType
&
data_type
,
knowhere
::
IndexPtr
&
index_ptr
,
milvus
::
json
&
range_values_json
)
{
auto
status
=
Status
::
OK
();
switch
(
data_type
)
{
...
...
@@ -579,7 +576,7 @@ ExecutionEngineImpl::IndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, const
Status
ExecutionEngineImpl
::
ProcessRangeQuery
(
const
std
::
unordered_map
<
std
::
string
,
DataType
>&
attr_type
,
faiss
::
Conc
urrentBitsetPtr
&
bitset
,
const
query
::
RangeQueryPtr
&
range_query
)
{
ConC
urrentBitsetPtr
&
bitset
,
const
query
::
RangeQueryPtr
&
range_query
)
{
SegmentPtr
segment_ptr
;
segment_reader_
->
GetSegment
(
segment_ptr
);
try
{
...
...
@@ -809,7 +806,7 @@ ExecutionEngineImpl::BuildKnowhereIndex(const std::string& field_name, const Col
LOG_ENGINE_DEBUG_
<<
"Index config: "
<<
conf
.
dump
();
std
::
vector
<
idx_t
>
uids
;
faiss
::
Conc
urrentBitsetPtr
blacklist
;
ConC
urrentBitsetPtr
blacklist
;
knowhere
::
DatasetPtr
dataset
;
if
(
from_index
)
{
dataset
=
...
...
core/src/index/archive/KnowhereResource.cpp
浏览文件 @
9d5ddb28
...
...
@@ -13,10 +13,13 @@
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#endif
#include <faiss/Clustering.h>
#include <faiss/utils/distances.h>
#include "config/ServerConfig.h"
#include "faiss/FaissHook.h"
#include "scheduler/Utils.h"
#include "utils/ConfigUtils.h"
#include "utils/Error.h"
#include "utils/Log.h"
...
...
@@ -60,6 +63,35 @@ KnowhereResource::Initialize() {
return
Status
(
KNOWHERE_UNEXPECTED_ERROR
,
"FAISS hook fail, CPU not supported!"
);
}
// engine config
int64_t
omp_thread
=
config
.
engine
.
omp_thread_num
();
if
(
omp_thread
>
0
)
{
omp_set_num_threads
(
omp_thread
);
LOG_SERVER_DEBUG_
<<
"Specify openmp thread number: "
<<
omp_thread
;
}
else
{
int64_t
sys_thread_cnt
=
8
;
if
(
milvus
::
server
::
GetSystemAvailableThreads
(
sys_thread_cnt
))
{
omp_thread
=
static_cast
<
int32_t
>
(
ceil
(
sys_thread_cnt
*
0.5
));
omp_set_num_threads
(
omp_thread
);
}
}
// init faiss global variable
int64_t
use_blas_threshold
=
config
.
engine
.
use_blas_threshold
();
faiss
::
distance_compute_blas_threshold
=
use_blas_threshold
;
int64_t
clustering_type
=
config
.
engine
.
clustering_type
();
switch
(
clustering_type
)
{
case
ClusteringType
::
K_MEANS
:
default:
faiss
::
clustering_type
=
faiss
::
ClusteringType
::
K_MEANS
;
break
;
case
ClusteringType
::
K_MEANS_PLUS_PLUS
:
faiss
::
clustering_type
=
faiss
::
ClusteringType
::
K_MEANS_PLUS_PLUS
;
break
;
}
#ifdef MILVUS_GPU_VERSION
bool
enable_gpu
=
config
.
gpu
.
enable
();
fiu_do_on
(
"KnowhereResource.Initialize.disable_gpu"
,
enable_gpu
=
false
);
...
...
core/src/server/DBWrapper.cpp
浏览文件 @
9d5ddb28
...
...
@@ -16,14 +16,10 @@
#include <string>
#include <vector>
#include <faiss/Clustering.h>
#include <faiss/utils/distances.h>
#include "config/ServerConfig.h"
#include "db/DBFactory.h"
#include "db/snapshot/OperationExecutor.h"
#include "utils/CommonUtil.h"
#include "utils/ConfigUtils.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
...
...
@@ -66,35 +62,6 @@ DBWrapper::StartService() {
opt
.
transcript_enable_
=
config
.
transcript
.
enable
();
opt
.
replay_script_path_
=
config
.
transcript
.
replay
();
// engine config
int64_t
omp_thread
=
config
.
engine
.
omp_thread_num
();
if
(
omp_thread
>
0
)
{
omp_set_num_threads
(
omp_thread
);
LOG_SERVER_DEBUG_
<<
"Specify openmp thread number: "
<<
omp_thread
;
}
else
{
int64_t
sys_thread_cnt
=
8
;
if
(
GetSystemAvailableThreads
(
sys_thread_cnt
))
{
omp_thread
=
static_cast
<
int32_t
>
(
ceil
(
sys_thread_cnt
*
0.5
));
omp_set_num_threads
(
omp_thread
);
}
}
// init faiss global variable
int64_t
use_blas_threshold
=
config
.
engine
.
use_blas_threshold
();
faiss
::
distance_compute_blas_threshold
=
use_blas_threshold
;
int64_t
clustering_type
=
config
.
engine
.
clustering_type
();
switch
(
clustering_type
)
{
case
ClusteringType
::
K_MEANS
:
default:
faiss
::
clustering_type
=
faiss
::
ClusteringType
::
K_MEANS
;
break
;
case
ClusteringType
::
K_MEANS_PLUS_PLUS
:
faiss
::
clustering_type
=
faiss
::
ClusteringType
::
K_MEANS_PLUS_PLUS
;
break
;
}
// create db root folder
s
=
CommonUtil
::
CreateDirectory
(
opt
.
meta_
.
path_
);
if
(
!
s
.
ok
())
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录