Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
6cb65a3a
M
milvus
项目概览
milvus
/
milvus
10 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
6cb65a3a
编写于
8月 20, 2020
作者:
C
Cai Yudong
提交者:
GitHub
8月 20, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix clang-tidy warnings (#3344)
Signed-off-by:
N
yudong.cai
<
yudong.cai@zilliz.com
>
上级
fd7c2554
变更
35
隐藏空白更改
内联
并排
Showing
35 changed file
with
94 addition
and
132 deletion
+94
-132
core/src/index/knowhere/knowhere/common/Timer.cpp
core/src/index/knowhere/knowhere/common/Timer.cpp
+1
-14
core/src/index/knowhere/knowhere/index/Index.h
core/src/index/knowhere/knowhere/index/Index.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
...ndex/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
+29
-37
core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h
...c/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h
...x/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h
+2
-2
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h
...dex/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.h
...rc/index/knowhere/knowhere/index/vector_index/IndexHNSW.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h
...c/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h
...src/index/knowhere/knowhere/index/vector_index/IndexIVF.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.h
...src/index/knowhere/knowhere/index/vector_index/IndexNSG.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp
...x/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.h
...dex/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.h
...index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.h
...index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h
...c/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h
+1
-1
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
...here/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
+1
-3
core/src/index/unittest/test_annoy.cpp
core/src/index/unittest/test_annoy.cpp
+1
-1
core/src/index/unittest/test_binaryidmap.cpp
core/src/index/unittest/test_binaryidmap.cpp
+3
-3
core/src/index/unittest/test_binaryivf.cpp
core/src/index/unittest/test_binaryivf.cpp
+2
-2
core/src/index/unittest/test_customized_index.cpp
core/src/index/unittest/test_customized_index.cpp
+1
-1
core/src/index/unittest/test_hnsw.cpp
core/src/index/unittest/test_hnsw.cpp
+2
-2
core/src/index/unittest/test_idmap.cpp
core/src/index/unittest/test_idmap.cpp
+5
-5
core/src/index/unittest/test_ivf.cpp
core/src/index/unittest/test_ivf.cpp
+2
-2
core/src/index/unittest/test_rhnsw_flat.cpp
core/src/index/unittest/test_rhnsw_flat.cpp
+2
-2
core/src/index/unittest/test_rhnsw_pq.cpp
core/src/index/unittest/test_rhnsw_pq.cpp
+2
-2
core/src/index/unittest/test_rhnsw_sq8.cpp
core/src/index/unittest/test_rhnsw_sq8.cpp
+2
-2
core/src/main.cpp
core/src/main.cpp
+4
-4
core/src/scheduler/Scheduler.cpp
core/src/scheduler/Scheduler.cpp
+0
-4
core/src/scheduler/Scheduler.h
core/src/scheduler/Scheduler.h
+1
-2
core/src/segment/SegmentWriter.cpp
core/src/segment/SegmentWriter.cpp
+1
-1
core/src/utils/ConfigUtils.cpp
core/src/utils/ConfigUtils.cpp
+3
-3
core/src/utils/SignalHandler.cpp
core/src/utils/SignalHandler.cpp
+0
-1
core/src/utils/Status.cpp
core/src/utils/Status.cpp
+5
-8
core/src/utils/Status.h
core/src/utils/Status.h
+4
-4
core/src/utils/TimeRecorder.cpp
core/src/utils/TimeRecorder.cpp
+9
-15
未找到文件。
core/src/index/knowhere/knowhere/common/Timer.cpp
浏览文件 @
6cb65a3a
...
...
@@ -42,22 +42,9 @@ TimeRecorder::PrintTimeRecord(const std::string& msg, double span) {
str_log
+=
")"
;
switch
(
log_level_
)
{
case
0
:
{
case
0
:
std
::
cout
<<
str_log
<<
std
::
endl
;
break
;
}
case
1
:
{
LOG_KNOWHERE_DEBUG_
<<
str_log
;
break
;
}
// case 2: {
// LOG_KNOWHERE_TRACE_ << str_log;
// break;
// }
// case 3: {
// LOG_KNOWHERE_WARNING_ << str_log;
// break;
// }
default:
LOG_KNOWHERE_DEBUG_
<<
str_log
;
break
;
...
...
core/src/index/knowhere/knowhere/index/Index.h
浏览文件 @
6cb65a3a
...
...
@@ -23,7 +23,7 @@ namespace knowhere {
class
Index
:
public
milvus
::
cache
::
DataObj
{
public:
virtual
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
=
0
;
Serialize
(
const
Config
&
config
)
=
0
;
virtual
void
Load
(
const
BinarySet
&
)
=
0
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
浏览文件 @
6cb65a3a
...
...
@@ -24,12 +24,15 @@
namespace
milvus
{
namespace
knowhere
{
#define DEFAULT_MAX_DIM 32768
#define DEFAULT_MIN_DIM 1
#define DEFAULT_MAX_K 16384
#define DEFAULT_MIN_K 1
#define DEFAULT_MIN_ROWS 1 // minimum size for build index
#define DEFAULT_MAX_ROWS 50000000
static
const
int64_t
MIN_NLIST
=
1
;
static
const
int64_t
MAX_NLIST
=
1LL
<<
20
;
static
const
int64_t
MIN_NPROBE
=
1
;
static
const
int64_t
MAX_NPROBE
=
MAX_NLIST
;
static
const
int64_t
DEFAULT_MIN_DIM
=
1
;
static
const
int64_t
DEFAULT_MAX_DIM
=
32768
;
static
const
int64_t
DEFAULT_MIN_ROWS
=
1
;
// minimum size for build index
static
const
int64_t
DEFAULT_MAX_ROWS
=
50000000
;
static
const
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
L2
,
knowhere
::
Metric
::
IP
};
#define CheckIntByRange(key, min, max) \
if (!oricfg.contains(key) || !oricfg[key].is_number_integer() || oricfg[key].get<int64_t>() > max || \
...
...
@@ -59,7 +62,6 @@ namespace knowhere {
bool
ConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
L2
,
knowhere
::
Metric
::
IP
};
CheckIntByRange
(
knowhere
::
meta
::
DIM
,
DEFAULT_MIN_DIM
,
DEFAULT_MAX_DIM
);
CheckStrByValues
(
knowhere
::
Metric
::
TYPE
,
METRICS
);
return
true
;
...
...
@@ -67,6 +69,8 @@ ConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
bool
ConfAdapter
::
CheckSearch
(
Config
&
oricfg
,
const
IndexType
type
,
const
IndexMode
mode
)
{
const
int64_t
DEFAULT_MIN_K
=
1
;
const
int64_t
DEFAULT_MAX_K
=
16384
;
CheckIntByRange
(
knowhere
::
meta
::
TOPK
,
DEFAULT_MIN_K
-
1
,
DEFAULT_MAX_K
);
return
true
;
}
...
...
@@ -85,9 +89,6 @@ MatchNlist(int64_t size, int64_t nlist) {
bool
IVFConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
int64_t
MAX_NLIST
=
999999
;
static
int64_t
MIN_NLIST
=
1
;
CheckIntByRange
(
knowhere
::
IndexParams
::
nlist
,
MIN_NLIST
,
MAX_NLIST
);
CheckIntByRange
(
knowhere
::
meta
::
ROWS
,
DEFAULT_MIN_ROWS
,
DEFAULT_MAX_ROWS
);
...
...
@@ -109,14 +110,11 @@ IVFConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
bool
IVFConfAdapter
::
CheckSearch
(
Config
&
oricfg
,
const
IndexType
type
,
const
IndexMode
mode
)
{
static
int64_t
MIN_NPROBE
=
1
;
static
int64_t
MAX_NPROBE
=
999999
;
// todo(linxj): [1, nlist]
if
(
mode
==
IndexMode
::
MODE_GPU
)
{
#ifdef MILVUS_GPU_VERSION
CheckIntByRange
(
knowhere
::
IndexParams
::
nprobe
,
MIN_NPROBE
,
faiss
::
gpu
::
getMaxKSelection
());
#endif
}
else
{
#endif
CheckIntByRange
(
knowhere
::
IndexParams
::
nprobe
,
MIN_NPROBE
,
MAX_NPROBE
);
}
...
...
@@ -125,7 +123,7 @@ IVFConfAdapter::CheckSearch(Config& oricfg, const IndexType type, const IndexMod
bool
IVFSQConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
int64_t
DEFAULT_NBITS
=
8
;
const
int64_t
DEFAULT_NBITS
=
8
;
oricfg
[
knowhere
::
IndexParams
::
nbits
]
=
DEFAULT_NBITS
;
return
IVFConfAdapter
::
CheckTrain
(
oricfg
,
mode
);
...
...
@@ -133,10 +131,7 @@ IVFSQConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
bool
IVFPQConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
int64_t
DEFAULT_NBITS
=
8
;
static
int64_t
MAX_NLIST
=
999999
;
static
int64_t
MIN_NLIST
=
1
;
static
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
L2
,
knowhere
::
Metric
::
IP
};
const
int64_t
DEFAULT_NBITS
=
8
;
oricfg
[
knowhere
::
IndexParams
::
nbits
]
=
DEFAULT_NBITS
;
...
...
@@ -174,8 +169,8 @@ IVFPQConfAdapter::GetValidMList(int64_t dimension, std::vector<int64_t>& resset)
* Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supported with
* no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads.
*/
static
std
::
vector
<
int64_t
>
support_dim_per_subquantizer
{
32
,
28
,
24
,
20
,
16
,
12
,
10
,
8
,
6
,
4
,
3
,
2
,
1
};
static
std
::
vector
<
int64_t
>
support_subquantizer
{
96
,
64
,
56
,
48
,
40
,
32
,
28
,
24
,
20
,
16
,
12
,
8
,
4
,
3
,
2
,
1
};
static
const
std
::
vector
<
int64_t
>
support_dim_per_subquantizer
{
32
,
28
,
24
,
20
,
16
,
12
,
10
,
8
,
6
,
4
,
3
,
2
,
1
};
static
const
std
::
vector
<
int64_t
>
support_subquantizer
{
96
,
64
,
56
,
48
,
40
,
32
,
28
,
24
,
20
,
16
,
12
,
8
,
4
,
3
,
2
,
1
};
for
(
const
auto
&
dimperquantizer
:
support_dim_per_subquantizer
)
{
if
(
!
(
dimension
%
dimperquantizer
))
{
...
...
@@ -190,15 +185,14 @@ IVFPQConfAdapter::GetValidMList(int64_t dimension, std::vector<int64_t>& resset)
bool
NSGConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
int64_t
MIN_KNNG
=
5
;
static
int64_t
MAX_KNNG
=
300
;
static
int64_t
MIN_SEARCH_LENGTH
=
10
;
static
int64_t
MAX_SEARCH_LENGTH
=
300
;
static
int64_t
MIN_OUT_DEGREE
=
5
;
static
int64_t
MAX_OUT_DEGREE
=
300
;
static
int64_t
MIN_CANDIDATE_POOL_SIZE
=
50
;
static
int64_t
MAX_CANDIDATE_POOL_SIZE
=
1000
;
static
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
L2
,
knowhere
::
Metric
::
IP
};
const
int64_t
MIN_KNNG
=
5
;
const
int64_t
MAX_KNNG
=
300
;
const
int64_t
MIN_SEARCH_LENGTH
=
10
;
const
int64_t
MAX_SEARCH_LENGTH
=
300
;
const
int64_t
MIN_OUT_DEGREE
=
5
;
const
int64_t
MAX_OUT_DEGREE
=
300
;
const
int64_t
MIN_CANDIDATE_POOL_SIZE
=
50
;
const
int64_t
MAX_CANDIDATE_POOL_SIZE
=
1000
;
CheckStrByValues
(
knowhere
::
Metric
::
TYPE
,
METRICS
);
CheckIntByRange
(
knowhere
::
meta
::
ROWS
,
DEFAULT_MIN_ROWS
,
DEFAULT_MAX_ROWS
);
...
...
@@ -325,9 +319,9 @@ RHNSWSQConfAdapter::CheckSearch(Config& oricfg, const IndexType type, const Inde
bool
BinIDMAPConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
HAMMING
,
knowhere
::
Metric
::
JACCARD
,
knowhere
::
Metric
::
TANIMOTO
,
knowhere
::
Metric
::
SUBSTRUCTURE
,
knowhere
::
Metric
::
SUPERSTRUCTURE
};
static
const
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
HAMMING
,
knowhere
::
Metric
::
JACCARD
,
knowhere
::
Metric
::
TANIMOTO
,
knowhere
::
Metric
::
SUBSTRUCTURE
,
knowhere
::
Metric
::
SUPERSTRUCTURE
};
CheckIntByRange
(
knowhere
::
meta
::
DIM
,
DEFAULT_MIN_DIM
,
DEFAULT_MAX_DIM
);
CheckStrByValues
(
knowhere
::
Metric
::
TYPE
,
METRICS
);
...
...
@@ -337,10 +331,8 @@ BinIDMAPConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
bool
BinIVFConfAdapter
::
CheckTrain
(
Config
&
oricfg
,
const
IndexMode
mode
)
{
static
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
HAMMING
,
knowhere
::
Metric
::
JACCARD
,
knowhere
::
Metric
::
TANIMOTO
};
static
int64_t
MAX_NLIST
=
999999
;
static
int64_t
MIN_NLIST
=
1
;
static
const
std
::
vector
<
std
::
string
>
METRICS
{
knowhere
::
Metric
::
HAMMING
,
knowhere
::
Metric
::
JACCARD
,
knowhere
::
Metric
::
TANIMOTO
};
CheckIntByRange
(
knowhere
::
meta
::
ROWS
,
DEFAULT_MIN_ROWS
,
DEFAULT_MAX_ROWS
);
CheckIntByRange
(
knowhere
::
meta
::
DIM
,
DEFAULT_MIN_DIM
,
DEFAULT_MAX_DIM
);
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h
浏览文件 @
6cb65a3a
...
...
@@ -30,7 +30,7 @@ class IndexAnnoy : public VecIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
Load
(
const
BinarySet
&
index_binary
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h
浏览文件 @
6cb65a3a
...
...
@@ -33,10 +33,10 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
)
override
;
void
Load
(
const
BinarySet
&
index_binary
)
override
;
Load
(
const
BinarySet
&
)
override
;
void
Train
(
const
DatasetPtr
&
,
const
Config
&
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h
浏览文件 @
6cb65a3a
...
...
@@ -36,7 +36,7 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
BuildAll
(
const
DatasetPtr
&
dataset_ptr
,
const
Config
&
config
)
override
{
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.h
浏览文件 @
6cb65a3a
...
...
@@ -29,7 +29,7 @@ class IndexHNSW : public VecIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
Load
(
const
BinarySet
&
index_binary
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h
浏览文件 @
6cb65a3a
...
...
@@ -31,7 +31,7 @@ class IDMAP : public VecIndex, public FaissBaseIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
)
override
;
void
Load
(
const
BinarySet
&
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h
浏览文件 @
6cb65a3a
...
...
@@ -36,7 +36,7 @@ class IVF : public VecIndex, public FaissBaseIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
)
override
;
void
Load
(
const
BinarySet
&
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.h
浏览文件 @
6cb65a3a
...
...
@@ -35,7 +35,7 @@ class NSG : public VecIndex {
}
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
)
override
;
void
Load
(
const
BinarySet
&
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp
浏览文件 @
6cb65a3a
...
...
@@ -89,7 +89,7 @@ IndexRHNSWFlat::Train(const DatasetPtr& dataset_ptr, const Config& config) {
auto
idx
=
new
faiss
::
IndexRHNSWFlat
(
int
(
dim
),
config
[
IndexParams
::
M
],
metric_type
);
idx
->
hnsw
.
efConstruction
=
config
[
IndexParams
::
efConstruction
];
index_
=
std
::
shared_ptr
<
faiss
::
Index
>
(
idx
);
index_
->
train
(
rows
,
(
float
*
)
p_data
);
index_
->
train
(
rows
,
reinterpret_cast
<
const
float
*>
(
p_data
)
);
}
catch
(
std
::
exception
&
e
)
{
KNOWHERE_THROW_MSG
(
e
.
what
());
}
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.h
浏览文件 @
6cb65a3a
...
...
@@ -35,7 +35,7 @@ class IndexRHNSWFlat : public IndexRHNSW {
IndexRHNSWFlat
(
int
d
,
int
M
,
MetricType
metric
=
Metric
::
L2
);
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
Load
(
const
BinarySet
&
index_binary
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.h
浏览文件 @
6cb65a3a
...
...
@@ -34,7 +34,7 @@ class IndexRHNSWPQ : public IndexRHNSW {
IndexRHNSWPQ
(
int
d
,
int
pq_m
,
int
M
);
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
Load
(
const
BinarySet
&
index_binary
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.h
浏览文件 @
6cb65a3a
...
...
@@ -34,7 +34,7 @@ class IndexRHNSWSQ : public IndexRHNSW {
IndexRHNSWSQ
(
int
d
,
faiss
::
QuantizerType
qtype
,
int
M
,
MetricType
metric
=
Metric
::
L2
);
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
Load
(
const
BinarySet
&
index_binary
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h
浏览文件 @
6cb65a3a
...
...
@@ -28,7 +28,7 @@ class CPUSPTAGRNG : public VecIndex {
public:
BinarySet
Serialize
(
const
Config
&
config
=
Config
()
)
override
;
Serialize
(
const
Config
&
config
)
override
;
void
Load
(
const
BinarySet
&
index_array
)
override
;
...
...
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
浏览文件 @
6cb65a3a
...
...
@@ -67,9 +67,7 @@ IVFSQHybrid::CopyGpuToCpu(const Config& config) {
faiss
::
Index
*
host_index
=
faiss
::
gpu
::
index_gpu_to_cpu
(
device_index
);
if
(
auto
*
ivf_index
=
dynamic_cast
<
faiss
::
IndexIVF
*>
(
host_index
))
{
if
(
ivf_index
!=
nullptr
)
{
ivf_index
->
to_readonly
();
}
ivf_index
->
to_readonly
();
ivf_index
->
backup_quantizer
();
}
...
...
core/src/index/unittest/test_annoy.cpp
浏览文件 @
6cb65a3a
...
...
@@ -170,7 +170,7 @@ TEST_P(AnnoyTest, annoy_serialize) {
{
// serialize index
index_
->
BuildAll
(
base_dataset
,
conf
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
milvus
::
knowhere
::
Config
()
);
auto
bin_data
=
binaryset
.
GetByName
(
"annoy_index_data"
);
std
::
string
filename1
=
"/tmp/annoy_test_data_serialize.bin"
;
...
...
core/src/index/unittest/test_binaryidmap.cpp
浏览文件 @
6cb65a3a
...
...
@@ -51,7 +51,7 @@ TEST_P(BinaryIDMAPTest, binaryidmap_basic) {
// null faiss index
{
ASSERT_ANY_THROW
(
index_
->
Serialize
());
ASSERT_ANY_THROW
(
index_
->
Serialize
(
conf
));
ASSERT_ANY_THROW
(
index_
->
Query
(
query_dataset
,
conf
));
ASSERT_ANY_THROW
(
index_
->
Add
(
nullptr
,
conf
));
ASSERT_ANY_THROW
(
index_
->
AddWithoutIds
(
nullptr
,
conf
));
...
...
@@ -67,7 +67,7 @@ TEST_P(BinaryIDMAPTest, binaryidmap_basic) {
AssertAnns
(
result
,
nq
,
k
);
// PrintResult(result, nq, k);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
new_index
=
std
::
make_shared
<
milvus
::
knowhere
::
BinaryIDMAP
>
();
new_index
->
Load
(
binaryset
);
auto
result2
=
new_index
->
Query
(
query_dataset
,
conf
);
...
...
@@ -112,7 +112,7 @@ TEST_P(BinaryIDMAPTest, binaryidmap_serialize) {
// PrintResult(re_result, nq, k);
EXPECT_EQ
(
index_
->
Count
(),
nb
);
EXPECT_EQ
(
index_
->
Dim
(),
dim
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
bin
=
binaryset
.
GetByName
(
"BinaryIVF"
);
std
::
string
filename
=
"/tmp/bianryidmap_test_serialize.bin"
;
...
...
core/src/index/unittest/test_binaryivf.cpp
浏览文件 @
6cb65a3a
...
...
@@ -62,7 +62,7 @@ TEST_P(BinaryIVFTest, binaryivf_basic) {
// null faiss index
{
ASSERT_ANY_THROW
(
index_
->
Serialize
());
ASSERT_ANY_THROW
(
index_
->
Serialize
(
conf
));
ASSERT_ANY_THROW
(
index_
->
Query
(
query_dataset
,
conf
));
ASSERT_ANY_THROW
(
index_
->
Add
(
nullptr
,
conf
));
ASSERT_ANY_THROW
(
index_
->
AddWithoutIds
(
nullptr
,
conf
));
...
...
@@ -131,7 +131,7 @@ TEST_P(BinaryIVFTest, binaryivf_serialize) {
index_
->
BuildAll
(
base_dataset
,
conf
);
// index_->set_index_model(model);
// index_->Add(base_dataset, conf);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
bin
=
binaryset
.
GetByName
(
"BinaryIVF"
);
std
::
string
filename
=
"/tmp/binaryivf_test_serialize.bin"
;
...
...
core/src/index/unittest/test_customized_index.cpp
浏览文件 @
6cb65a3a
...
...
@@ -58,7 +58,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
EXPECT_EQ
(
index_
->
Count
(),
nb
);
EXPECT_EQ
(
index_
->
Dim
(),
dim
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
{
// copy cpu to gpu
auto
cpu_idx
=
std
::
make_shared
<
milvus
::
knowhere
::
IVFSQHybrid
>
(
DEVICEID
);
...
...
core/src/index/unittest/test_hnsw.cpp
浏览文件 @
6cb65a3a
...
...
@@ -66,7 +66,7 @@ TEST_P(HNSWTest, HNSW_basic) {
EXPECT_EQ
(
index_
->
Dim
(),
dim
);
// Serialize and Load before Query
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
();
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
(
conf
);
int64_t
dim
=
base_dataset
->
Get
<
int64_t
>
(
milvus
::
knowhere
::
meta
::
DIM
);
int64_t
rows
=
base_dataset
->
Get
<
int64_t
>
(
milvus
::
knowhere
::
meta
::
ROWS
);
...
...
@@ -96,7 +96,7 @@ TEST_P(HNSWTest, HNSW_delete) {
}
// Serialize and Load before Query
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
();
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
(
conf
);
int64_t
dim
=
base_dataset
->
Get
<
int64_t
>
(
milvus
::
knowhere
::
meta
::
DIM
);
int64_t
rows
=
base_dataset
->
Get
<
int64_t
>
(
milvus
::
knowhere
::
meta
::
ROWS
);
...
...
core/src/index/unittest/test_idmap.cpp
浏览文件 @
6cb65a3a
...
...
@@ -72,7 +72,7 @@ TEST_P(IDMAPTest, idmap_basic) {
// null faiss index
{
ASSERT_ANY_THROW
(
index_
->
Serialize
());
ASSERT_ANY_THROW
(
index_
->
Serialize
(
conf
));
ASSERT_ANY_THROW
(
index_
->
Query
(
query_dataset
,
conf
));
ASSERT_ANY_THROW
(
index_
->
Add
(
nullptr
,
conf
));
ASSERT_ANY_THROW
(
index_
->
AddWithoutIds
(
nullptr
,
conf
));
...
...
@@ -95,7 +95,7 @@ TEST_P(IDMAPTest, idmap_basic) {
#endif
}
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
new_index
=
std
::
make_shared
<
milvus
::
knowhere
::
IDMAP
>
();
new_index
->
Load
(
binaryset
);
auto
result2
=
new_index
->
Query
(
query_dataset
,
conf
);
...
...
@@ -158,7 +158,7 @@ TEST_P(IDMAPTest, idmap_serialize) {
// PrintResult(re_result, nq, k);
EXPECT_EQ
(
index_
->
Count
(),
nb
);
EXPECT_EQ
(
index_
->
Dim
(),
dim
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
bin
=
binaryset
.
GetByName
(
"IVF"
);
std
::
string
filename
=
"/tmp/idmap_test_serialize.bin"
;
...
...
@@ -216,10 +216,10 @@ TEST_P(IDMAPTest, idmap_copy) {
fiu_init
(
0
);
fiu_enable
(
"GPUIDMP.SerializeImpl.throw_exception"
,
1
,
nullptr
,
0
);
ASSERT_ANY_THROW
(
clone_index
->
Serialize
());
ASSERT_ANY_THROW
(
clone_index
->
Serialize
(
conf
));
fiu_disable
(
"GPUIDMP.SerializeImpl.throw_exception"
);
auto
binary
=
clone_index
->
Serialize
();
auto
binary
=
clone_index
->
Serialize
(
conf
);
clone_index
->
Load
(
binary
);
auto
new_result
=
clone_index
->
Query
(
query_dataset
,
conf
);
AssertAnns
(
new_result
,
nq
,
k
);
...
...
core/src/index/unittest/test_ivf.cpp
浏览文件 @
6cb65a3a
...
...
@@ -196,7 +196,7 @@ TEST_P(IVFTest, ivf_serialize) {
// serialize index
index_
->
Train
(
base_dataset
,
conf_
);
index_
->
Add
(
base_dataset
,
conf_
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf_
);
auto
bin
=
binaryset
.
GetByName
(
"IVF"
);
std
::
string
filename
=
"/tmp/ivf_test_serialize.bin"
;
...
...
@@ -340,7 +340,7 @@ TEST_P(IVFTest, invalid_gpu_source) {
fiu_init
(
0
);
fiu_enable
(
"GPUIVF.SerializeImpl.throw_exception"
,
1
,
nullptr
,
0
);
ASSERT_ANY_THROW
(
index_
->
Serialize
());
ASSERT_ANY_THROW
(
index_
->
Serialize
(
conf_
));
fiu_disable
(
"GPUIVF.SerializeImpl.throw_exception"
);
fiu_enable
(
"GPUIVF.search_impl.invald_index"
,
1
,
nullptr
,
0
);
...
...
core/src/index/unittest/test_rhnsw_flat.cpp
浏览文件 @
6cb65a3a
...
...
@@ -56,7 +56,7 @@ TEST_P(RHNSWFlatTest, HNSW_basic) {
AssertAnns
(
result1
,
nq
,
k
);
// Serialize and Load before Query
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
();
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
(
conf
);
auto
tmp_index
=
std
::
make_shared
<
milvus
::
knowhere
::
IndexRHNSWFlat
>
();
...
...
@@ -122,7 +122,7 @@ TEST_P(RHNSWFlatTest, HNSW_serialize) {
{
index_
->
Train
(
base_dataset
,
conf
);
index_
->
Add
(
base_dataset
,
conf
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
std
::
string
index_type
=
index_
->
index_type
();
std
::
string
idx_name
=
index_type
+
"_Index"
;
std
::
string
dat_name
=
index_type
+
"_Data"
;
...
...
core/src/index/unittest/test_rhnsw_pq.cpp
浏览文件 @
6cb65a3a
...
...
@@ -53,7 +53,7 @@ TEST_P(RHNSWPQTest, HNSW_basic) {
EXPECT_EQ
(
index_
->
Dim
(),
dim
);
// Serialize and Load before Query
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
();
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
(
conf
);
auto
result1
=
index_
->
Query
(
query_dataset
,
conf
);
// AssertAnns(result1, nq, k);
...
...
@@ -121,7 +121,7 @@ TEST_P(RHNSWPQTest, HNSW_serialize) {
{
index_
->
Train
(
base_dataset
,
conf
);
index_
->
Add
(
base_dataset
,
conf
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
bin_idx
=
binaryset
.
GetByName
(
index_
->
index_type
()
+
"_Index"
);
auto
bin_dat
=
binaryset
.
GetByName
(
index_
->
index_type
()
+
"_Data"
);
...
...
core/src/index/unittest/test_rhnsw_sq8.cpp
浏览文件 @
6cb65a3a
...
...
@@ -54,7 +54,7 @@ TEST_P(RHNSWSQ8Test, HNSW_basic) {
EXPECT_EQ
(
index_
->
Dim
(),
dim
);
// Serialize and Load before Query
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
();
milvus
::
knowhere
::
BinarySet
bs
=
index_
->
Serialize
(
conf
);
auto
result1
=
index_
->
Query
(
query_dataset
,
conf
);
AssertAnns
(
result1
,
nq
,
k
);
...
...
@@ -122,7 +122,7 @@ TEST_P(RHNSWSQ8Test, HNSW_serialize) {
{
index_
->
Train
(
base_dataset
,
conf
);
index_
->
Add
(
base_dataset
,
conf
);
auto
binaryset
=
index_
->
Serialize
();
auto
binaryset
=
index_
->
Serialize
(
conf
);
auto
bin_idx
=
binaryset
.
GetByName
(
index_
->
index_type
()
+
"_Index"
);
auto
bin_dat
=
binaryset
.
GetByName
(
index_
->
index_type
()
+
"_Data"
);
...
...
core/src/main.cpp
浏览文件 @
6cb65a3a
...
...
@@ -39,10 +39,10 @@ print_help(const std::string& app_name) {
void
print_banner
()
{
std
::
cout
<<
std
::
endl
;
std
::
cout
<<
" __ _________ _ ____ ______
"
<<
std
::
endl
;
std
::
cout
<<
" / |/ / _/ /| | / / / / / __/
"
<<
std
::
endl
;
std
::
cout
<<
" / /|_/ // // /_| |/ / /_/ /
\\
\\
"
<<
std
::
endl
;
std
::
cout
<<
" /_/ /_/___/____/___/
\\
____/___/
"
<<
std
::
endl
;
std
::
cout
<<
R"( __ _________ _ ____ ______ )
"
<<
std
::
endl
;
std
::
cout
<<
R"( / |/ / _/ /| | / / / / / __/ )
"
<<
std
::
endl
;
std
::
cout
<<
R"( / /|_/ // // /_| |/ / /_/ /\ \ )
"
<<
std
::
endl
;
std
::
cout
<<
R"( /_/ /_/___/____/___/\____/___/ )
"
<<
std
::
endl
;
std
::
cout
<<
std
::
endl
;
std
::
cout
<<
"Welcome to use Milvus!"
<<
std
::
endl
;
std
::
cout
<<
"Milvus "
<<
BUILD_TYPE
<<
" version: v"
<<
MILVUS_VERSION
<<
", built at "
<<
BUILD_TIME
<<
", with "
...
...
core/src/scheduler/Scheduler.cpp
浏览文件 @
6cb65a3a
...
...
@@ -32,10 +32,6 @@ Scheduler::Scheduler(ResourceMgrPtr res_mgr) : running_(false), res_mgr_(std::mo
std
::
bind
(
&
Scheduler
::
OnFinishTask
,
this
,
std
::
placeholders
::
_1
)));
}
Scheduler
::~
Scheduler
()
{
res_mgr_
=
nullptr
;
}
void
Scheduler
::
Start
()
{
running_
=
true
;
...
...
core/src/scheduler/Scheduler.h
浏览文件 @
6cb65a3a
...
...
@@ -29,8 +29,7 @@ namespace scheduler {
class
Scheduler
:
public
interface
::
dumpable
{
public:
explicit
Scheduler
(
ResourceMgrPtr
res_mgr
);
~
Scheduler
();
virtual
~
Scheduler
()
=
default
;
Scheduler
(
const
Scheduler
&
)
=
delete
;
Scheduler
(
Scheduler
&&
)
=
delete
;
...
...
core/src/segment/SegmentWriter.cpp
浏览文件 @
6cb65a3a
...
...
@@ -179,7 +179,7 @@ SegmentWriter::WriteBloomFilter() {
segment
::
IdBloomFilterPtr
bloom_filter_ptr
;
ss_codec
.
GetIdBloomFilterFormat
()
->
Create
(
fs_ptr_
,
file_path
,
bloom_filter_ptr
);
int64_t
*
uids
=
(
int64_t
*
)
(
uid_data
->
data_
.
data
());
auto
uids
=
reinterpret_cast
<
int64_t
*>
(
uid_data
->
data_
.
data
());
int64_t
row_count
=
segment_ptr_
->
GetRowCount
();
for
(
int64_t
i
=
0
;
i
<
row_count
;
i
++
)
{
bloom_filter_ptr
->
Add
(
uids
[
i
]);
...
...
core/src/utils/ConfigUtils.cpp
浏览文件 @
6cb65a3a
...
...
@@ -245,8 +245,8 @@ ValidateDbURI(const std::string& uri) {
std
::
string
hostRegex
=
"(.*)"
;
std
::
string
portRegex
=
"(.*)"
;
std
::
string
dbNameRegex
=
"(.*)"
;
std
::
string
uriRegexStr
=
dialectRegex
+
"
\\
:
\\
/
\\
/"
+
usernameRegex
+
"
\\
:"
+
passwordRegex
+
"
\\
@"
+
hostRegex
+
"
\\
:"
+
portRegex
+
"
\\
/
"
+
dbNameRegex
;
std
::
string
uriRegexStr
=
dialectRegex
+
R"(\:\/\/)"
+
usernameRegex
+
R"(\:)"
+
passwordRegex
+
R"(\@)"
+
hostRegex
+
R"(\:)"
+
portRegex
+
R"(\/)
"
+
dbNameRegex
;
std
::
regex
uriRegex
(
uriRegexStr
);
std
::
smatch
pieces_match
;
...
...
@@ -296,7 +296,7 @@ ValidateStoragePath(const std::string& path) {
// and path must start with '/'.
// examples below are invalid
// '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
std
::
string
path_pattern
=
"^
\\
/(
\\
w+-?
\\
/?)+$
"
;
std
::
string
path_pattern
=
R"(^\/(\w+-?\/?)+$)
"
;
std
::
regex
regex
(
path_pattern
);
return
std
::
regex_match
(
path
,
regex
)
?
Status
::
OK
()
:
Status
(
SERVER_INVALID_ARGUMENT
,
"Invalid file path"
);
...
...
core/src/utils/SignalHandler.cpp
浏览文件 @
6cb65a3a
...
...
@@ -13,7 +13,6 @@
#include "utils/Log.h"
#include <execinfo.h>
#include <signal.h>
#include <string>
namespace
milvus
{
...
...
core/src/utils/Status.cpp
浏览文件 @
6cb65a3a
...
...
@@ -30,27 +30,24 @@ Status::Status(StatusCode code, const std::string& msg) {
state_
=
result
;
}
Status
::
Status
()
:
state_
(
nullptr
)
{
}
Status
::~
Status
()
{
delete
state_
;
}
Status
::
Status
(
const
Status
&
s
)
:
state_
(
nullptr
)
{
Status
::
Status
(
const
Status
&
s
)
{
CopyFrom
(
s
);
}
Status
::
Status
(
Status
&&
s
)
{
MoveFrom
(
s
);
}
Status
&
Status
::
operator
=
(
const
Status
&
s
)
{
CopyFrom
(
s
);
return
*
this
;
}
Status
::
Status
(
Status
&&
s
)
:
state_
(
nullptr
)
{
MoveFrom
(
s
);
}
Status
&
Status
::
operator
=
(
Status
&&
s
)
{
MoveFrom
(
s
);
...
...
core/src/utils/Status.h
浏览文件 @
6cb65a3a
...
...
@@ -31,16 +31,16 @@ using StatusCode = ErrorCode;
class
Status
{
public:
Status
(
StatusCode
code
,
const
std
::
string
&
msg
);
Status
();
~
Status
();
Status
()
=
default
;
virtual
~
Status
();
Status
(
const
Status
&
s
);
Status
(
Status
&&
s
);
Status
&
operator
=
(
const
Status
&
s
);
Status
(
Status
&&
s
);
Status
&
operator
=
(
Status
&&
s
);
...
...
core/src/utils/TimeRecorder.cpp
浏览文件 @
6cb65a3a
...
...
@@ -31,42 +31,36 @@ TimeRecorder::GetTimeSpanStr(double span) {
void
TimeRecorder
::
PrintTimeRecord
(
const
std
::
string
&
msg
,
double
span
)
{
std
::
string
str_log
;
if
(
!
header_
.
empty
())
if
(
!
header_
.
empty
())
{
str_log
+=
header_
+
": "
;
}
str_log
+=
msg
;
str_log
+=
" ("
;
str_log
+=
TimeRecorder
::
GetTimeSpanStr
(
span
);
str_log
+=
")"
;
switch
(
log_level_
)
{
case
0
:
{
case
0
:
LOG_SERVER_TRACE_
<<
str_log
;
break
;
}
case
1
:
{
case
1
:
LOG_SERVER_DEBUG_
<<
str_log
;
break
;
}
case
2
:
{
case
2
:
LOG_SERVER_INFO_
<<
str_log
;
break
;
}
case
3
:
{
case
3
:
LOG_SERVER_WARNING_
<<
str_log
;
break
;
}
case
4
:
{
case
4
:
LOG_SERVER_ERROR_
<<
str_log
;
break
;
}
case
5
:
{
case
5
:
LOG_SERVER_FATAL_
<<
str_log
;
break
;
}
default:
{
default:
LOG_SERVER_INFO_
<<
str_log
;
break
;
}
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录