mobilenetssd load_persistables报错
Created by: ellinyang
相关环境:训练GPU v100, paddle1.4.1 重载环境:GPU v100, paddle1.5.0 背景:训练时只使用save_persistables保存为单个文件,现想重载权重文件进行评估和预测,但一直失败。
训练时使用
model_ckpt_name = 'params_pass_%05d.ckpt' % (epoch)
fluid.io.save_persistables(self.exe, self.checkpoint_dir,
main_program=self.test_program,
filename=model_ckpt_name)
加载模型时,报错
fluid.io.load_persistables(exe, model_dir,test_prog, filename="params_pass_00065.ckpt")
错误日志如下:
Traceback (most recent call last):
File "infer.py", line 339, in <module>
save_dir=args.save_dir)
File "infer.py", line 90, in infer
fluid.io.load_persistables(exe, model_dir,test_prog, filename="params_pass_00065.ckpt")
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/io.py", line 742, in load_persistables
filename=filename)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/io.py", line 608, in load_vars
filename=filename)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/io.py", line 645, in load_vars
executor.run(load_prog)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 650, in run
use_program_cache=use_program_cache)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 748, in _run
exe.run(program.desc, scope, 0, True, True, fetch_var_name)
paddle.fluid.core_avx.EnforceNotMet: Invoke operator load_combine error.
Python Callstacks:
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/framework.py", line 1748, in append_op
attrs=kwargs.get("attrs", None))
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/io.py", line 644, in load_vars
attrs={'file_path': os.path.join(load_dirname, filename)})
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/io.py", line 608, in load_vars
filename=filename)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/io.py", line 742, in load_persistables
filename=filename)
File "infer.py", line 90, in infer
fluid.io.load_persistables(exe, model_dir,test_prog, filename="params_pass_00065.ckpt")
File "infer.py", line 339, in <module>
save_dir=args.save_dir)
C++ Callstacks:
Cannot parse tensor desc at [/paddle/paddle/fluid/framework/tensor_util.cu:462]
PaddlePaddle Call Stacks:
0 0x7f85567706a0p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 352
1 0x7f8556770a19p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 137
2 0x7f855872b703p paddle::framework::TensorFromStream(std::istream&, paddle::framework::Tensor*, paddle::platform::DeviceContext const&) + 1315
3 0x7f855831a8d0p paddle::framework::DeserializeFromStream(std::istream&, paddle::framework::LoDTensor*, paddle::platform::DeviceContext const&) + 576
4 0x7f8557451a30p paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, float>::LoadParamsFromBuffer(paddle::framework::ExecutionContext const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, std::istream*, bool, std::vector<std::string, std::allocator<std::string> > const&) const + 352
5 0x7f85574523aep paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 798
6 0x7f8557452823p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, int>, paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, signed char>, paddle::operators::LoadCombineOpKernel<paddle::platform::CUDADeviceContext, long> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
7 0x7f85586cd907p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 375
8 0x7f85586cdce1p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529
9 0x7f85586cb2dcp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332
10 0x7f85568fc38ep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 382
11 0x7f85568ff42fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocator<std::string> > const&, bool) + 143
12 0x7f8556761b2dp
13 0x7f85567a35c6p
14 0x4c5326p PyEval_EvalFrameEx + 37958
15 0x4b9b66p PyEval_EvalCodeEx + 774
16 0x4c1f56p PyEval_EvalFrameEx + 24694
17 0x4b9b66p PyEval_EvalCodeEx + 774
18 0x4c17c6p PyEval_EvalFrameEx + 22758
19 0x4b9b66p PyEval_EvalCodeEx + 774
20 0x4c17c6p PyEval_EvalFrameEx + 22758
21 0x4b9b66p PyEval_EvalCodeEx + 774
22 0x4c17c6p PyEval_EvalFrameEx + 22758
23 0x4b9b66p PyEval_EvalCodeEx + 774
24 0x4c17c6p PyEval_EvalFrameEx + 22758
25 0x4b9b66p PyEval_EvalCodeEx + 774
26 0x4c1f56p PyEval_EvalFrameEx + 24694
27 0x4b9b66p PyEval_EvalCodeEx + 774
28 0x4eb69fp
29 0x4e58f2p PyRun_FileExFlags + 130
30 0x4e41a6p PyRun_SimpleFileExFlags + 390
31 0x4938cep Py_Main + 1358
32 0x7f86a1d78830p __libc_start_main + 240
33 0x493299p _start + 41