提交 e81982bc 编写于 作者: G gineshidalgo99

Exceptions in threads automatically handled

上级 4d70fdf5
......@@ -322,6 +322,7 @@ OpenPose Library - Release Notes
30. Added pre-processing capabilities to Wrapper (WorkerType::PreProcessing), which will be run right after the image has been read.
31. Removed boost::shared_ptr and caffe::Blob dependencies from the headers. No 3rdparty dependencies left on headers (except dim3 for CUDA).
32. Added Array `poseNetOutput` to Datum so that user can introduce his custom network output.
33. OpenPose will never provoke a core dumped or crash. Exceptions in threads (`errorWorker()` instead of `error()`) lead to stopping the threads and reporting the error from the main thread, while exceptions in destructors (`errorDestructor()` instead of `error()`) are reported with std::cerr but not thrown as std::exceptions.
2. Functions or parameters renamed:
1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368.
2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary.
......
......@@ -111,7 +111,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -68,7 +68,7 @@ namespace op
const T* cpu_data() const;
void set_cpu_data(T* data);
// const int* gpu_shape() const; // Disabled because it produces compiler errors for some users
const int* gpu_shape() const;
const T* gpu_data() const;
void set_gpu_data(T* data);
const T* cpu_diff() const;
......
......@@ -143,7 +143,7 @@ namespace op
try
{
for (auto& tWorker : mTWorkers)
tWorker->initializationOnThread();
tWorker->initializationOnThreadNoException();
}
catch (const std::exception& e)
{
......
......@@ -96,7 +96,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -235,6 +235,8 @@ namespace op
for (auto& thread : mThreads)
thread->stopAndJoin();
log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
checkWorkerErrors();
log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
}
catch (const std::exception& e)
{
......@@ -396,6 +398,9 @@ namespace op
{
if (!mThreadWorkerQueues.empty())
{
// This avoids extra std::cout if errors occur on different threads
setMainThread();
// Check threads
checkAndCreateEmptyThreads();
......
......@@ -13,7 +13,7 @@ namespace op
virtual ~Worker();
virtual void initializationOnThread() = 0;
void initializationOnThreadNoException();
bool checkAndWork(TDatums& tDatums);
......@@ -35,6 +35,8 @@ namespace op
}
protected:
virtual void initializationOnThread() = 0;
virtual void work(TDatums& tDatums) = 0;
private:
......@@ -62,12 +64,35 @@ namespace op
{
}
template<typename TDatums>
void Worker<TDatums>::initializationOnThreadNoException()
{
try
{
this->initializationOnThread();
}
catch (const std::exception& e)
{
this->stop();
errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
template<typename TDatums>
bool Worker<TDatums>::checkAndWork(TDatums& tDatums)
{
if (mIsRunning)
work(tDatums);
return mIsRunning;
try
{
if (mIsRunning)
work(tDatums);
return mIsRunning;
}
catch (const std::exception& e)
{
this->stop();
errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
return false;
}
}
COMPILE_TEMPLATE_DATUM(Worker);
......
......@@ -12,10 +12,7 @@ namespace op
public:
virtual ~WorkerConsumer();
inline void work(TDatums& tDatums)
{
workConsumer(tDatums);
}
void work(TDatums& tDatums);
protected:
virtual void workConsumer(const TDatums& tDatums) = 0;
......@@ -34,6 +31,20 @@ namespace op
{
}
template<typename TDatums>
void WorkerConsumer<TDatums>::work(TDatums& tDatums)
{
try
{
workConsumer(tDatums);
}
catch (const std::exception& e)
{
this->stop();
errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
COMPILE_TEMPLATE_DATUM(WorkerConsumer);
}
......
......@@ -12,10 +12,7 @@ namespace op
public:
virtual ~WorkerProducer();
inline void work(TDatums& tDatums)
{
tDatums = std::move(workProducer());
}
void work(TDatums& tDatums);
protected:
virtual TDatums workProducer() = 0;
......@@ -34,6 +31,20 @@ namespace op
{
}
template<typename TDatums>
void WorkerProducer<TDatums>::work(TDatums& tDatums)
{
try
{
tDatums = std::move(workProducer());
}
catch (const std::exception& e)
{
this->stop();
errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
COMPILE_TEMPLATE_DATUM(WorkerProducer);
}
......
#ifndef OPENPOSE_UTILITIES_ERROR_AND_LOG_HPP
#define OPENPOSE_UTILITIES_ERROR_AND_LOG_HPP
#include <atomic>
#include <mutex>
#include <sstream> // std::stringstream
#include <string>
#include <vector>
......@@ -11,6 +9,14 @@
namespace op
{
OP_API void setMainThread();
OP_API std::string getThreadId();
OP_API bool getIfInMainThreadOrEmpty();
OP_API bool getIfNotInMainThreadOrEmpty();
template<typename T>
std::string tToString(const T& message)
{
......@@ -21,27 +27,65 @@ namespace op
return oss.str();
}
/**
* Differences between different kind of errrors:
* - error() is a normal error in the code.
* - errorWorker() is an error that occurred on a thread. Therefore, the machine will stop the threads, go back
* to the main thread, and then throw the error.
* - errorDestructor() is an error that occurred on a destructor. Exception on destructors provokes core dumped,
* so we simply output an error message via std::cerr.
*/
// Error managment - How to use:
// error(message, __LINE__, __FUNCTION__, __FILE__);
OP_API void error(const std::string& message, const int line = -1, const std::string& function = "",
const std::string& file = "");
OP_API void error(
const std::string& message, const int line = -1, const std::string& function = "",
const std::string& file = "");
template<typename T>
inline void error(const T& message, const int line = -1, const std::string& function = "",
const std::string& file = "")
inline void error(
const T& message, const int line = -1, const std::string& function = "", const std::string& file = "")
{
error(tToString(message), line, function, file);
}
// Worker error managment
OP_API void checkWorkerErrors();
OP_API void errorWorker(
const std::string& message, const int line = -1, const std::string& function = "",
const std::string& file = "");
template<typename T>
inline void errorWorker(
const T& message, const int line = -1, const std::string& function = "", const std::string& file = "")
{
errorWorker(tToString(message), line, function, file);
}
// Destructor error managment
OP_API void errorDestructor(
const std::string& message, const int line = -1, const std::string& function = "",
const std::string& file = "");
template<typename T>
inline void errorDestructor(
const T& message, const int line = -1, const std::string& function = "", const std::string& file = "")
{
errorDestructor(tToString(message), line, function, file);
}
// Printing info - How to use:
// It will print info if desiredPriority >= sPriorityThreshold
// log(message, desiredPriority, __LINE__, __FUNCTION__, __FILE__);
OP_API void log(const std::string& message, const Priority priority = Priority::Max, const int line = -1,
const std::string& function = "", const std::string& file = "");
OP_API void log(
const std::string& message, const Priority priority = Priority::Max, const int line = -1,
const std::string& function = "", const std::string& file = "");
template<typename T>
inline void log(const T& message, const Priority priority = Priority::Max, const int line = -1,
const std::string& function = "", const std::string& file = "")
inline void log(
const T& message, const Priority priority = Priority::Max, const int line = -1,
const std::string& function = "", const std::string& file = "")
{
log(tToString(message), priority, line, function, file);
}
......@@ -50,8 +94,9 @@ namespace op
// It will print info if desiredPriority >= sPriorityThreshold
// dLog(message, desiredPriority, __LINE__, __FUNCTION__, __FILE__);
template<typename T>
inline void dLog(const T& message, const Priority priority = Priority::Max, const int line = -1,
const std::string& function = "", const std::string& file = "")
inline void dLog(
const T& message, const Priority priority = Priority::Max, const int line = -1,
const std::string& function = "", const std::string& file = "")
{
#ifndef NDEBUG
log(message, priority, line, function, file);
......
......@@ -279,7 +279,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -578,32 +578,34 @@ namespace op
}
}
// template<typename T>
// const int* ArrayCpuGpu<T>::gpu_shape() const
// {
// try
// {
// #ifdef USE_CAFFE
// return spImpl->pCaffeBlobT->gpu_shape();
// #else
// return nullptr;
// #endif
// }
// catch (const std::exception& e)
// {
// error(e.what(), __LINE__, __FUNCTION__, __FILE__);
// return nullptr;
// }
// }
template<typename T>
const int* ArrayCpuGpu<T>::gpu_shape() const
{
try
{
#if defined(USE_CAFFE) && defined(USE_CUDA)
return spImpl->pCaffeBlobT->gpu_shape();
#else
error("Required `USE_CAFFE` and `USE_CUDA` flags enabled.", __LINE__, __FUNCTION__, __FILE__);
return nullptr;
#endif
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
return nullptr;
}
}
template<typename T>
const T* ArrayCpuGpu<T>::gpu_data() const
{
try
{
#ifdef USE_CAFFE
#if defined(USE_CAFFE) && defined(USE_CUDA)
return spImpl->pCaffeBlobT->gpu_data();
#else
error("Required `USE_CAFFE` and `USE_CUDA` flags enabled.", __LINE__, __FUNCTION__, __FILE__);
return nullptr;
#endif
}
......@@ -619,10 +621,11 @@ namespace op
{
try
{
#ifdef USE_CAFFE
#if defined(USE_CAFFE) && defined(USE_CUDA)
spImpl->pCaffeBlobT->set_gpu_data(data);
#else
UNUSED(data);
error("Required `USE_CAFFE` and `USE_CUDA` flags enabled.", __LINE__, __FUNCTION__, __FILE__);
#endif
}
catch (const std::exception& e)
......@@ -654,9 +657,10 @@ namespace op
{
try
{
#ifdef USE_CAFFE
#if defined(USE_CAFFE) && defined(USE_CUDA)
return spImpl->pCaffeBlobT->gpu_diff();
#else
error("Required `USE_CAFFE` and `USE_CUDA` flags enabled.", __LINE__, __FUNCTION__, __FILE__);
return nullptr;
#endif
}
......@@ -690,9 +694,10 @@ namespace op
{
try
{
#ifdef USE_CAFFE
#if defined(USE_CAFFE) && defined(USE_CUDA)
return spImpl->pCaffeBlobT->mutable_gpu_data();
#else
error("Required `USE_CAFFE` and `USE_CUDA` flags enabled.", __LINE__, __FUNCTION__, __FILE__);
return nullptr;
#endif
}
......@@ -726,9 +731,10 @@ namespace op
{
try
{
#ifdef USE_CAFFE
#if defined(USE_CAFFE) && defined(USE_CUDA)
return spImpl->pCaffeBlobT->mutable_gpu_diff();
#else
error("Required `USE_CAFFE` and `USE_CUDA` flags enabled.", __LINE__, __FUNCTION__, __FILE__);
return nullptr;
#endif
}
......
......@@ -51,7 +51,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -25,7 +25,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -94,7 +94,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -34,7 +34,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -57,7 +57,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -39,7 +39,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -149,7 +149,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -458,7 +458,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
#endif
}
......
......@@ -25,7 +25,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -28,16 +28,32 @@ namespace op
ImplNmsCaffe()
{
#if defined USE_CAFFE && defined USE_OPENCL
mKernelGpuPtr = nullptr;
mKernelCpuPtr = nullptr;
try
{
mKernelGpuPtr = nullptr;
mKernelCpuPtr = nullptr;
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
#endif
}
~ImplNmsCaffe()
{
#if defined USE_CAFFE && defined USE_OPENCL
if(mKernelGpuPtr != nullptr) clReleaseMemObject((cl_mem)mKernelGpuPtr);
if(mKernelCpuPtr != nullptr) delete mKernelCpuPtr;
try
{
if(mKernelGpuPtr != nullptr)
clReleaseMemObject((cl_mem)mKernelGpuPtr);
if(mKernelCpuPtr != nullptr)
delete mKernelCpuPtr;
}
catch (const std::exception& e)
{
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
#endif
}
};
......
......@@ -37,7 +37,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -32,7 +32,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
#include <atomic>
#include <mutex>
#include <opencv2/imgproc/imgproc.hpp> // cv::undistort, cv::initUndistortRectifyMap
#ifdef USE_FLIR_CAMERA
#include <Spinnaker.h>
......@@ -915,7 +917,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -51,7 +51,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
......@@ -62,7 +62,7 @@ namespace op
}
catch (const std::exception& e)
{
error(e.what(), __LINE__, __FUNCTION__, __FILE__);
errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
}
}
......
#include <atomic>
#include <mutex>
#include <ctime> // std::tm, std::time_t
#include <fstream> // std::ifstream, std::ofstream
#include <iostream> // std::cout, std::endl
......@@ -6,6 +8,11 @@
namespace op
{
std::atomic<bool> sThreadErrors;
std::mutex sMutex;
std::vector<std::string> sThreadErrorMessages;
std::string sMainThreadId;
#ifdef USE_UNITY_SUPPORT
namespace UnityDebugger
{
......@@ -64,6 +71,7 @@ namespace op
return false;
}
// Note: createFullMessage(message) = message
std::string createFullMessage(const std::string& message, const int line = -1, const std::string& function = "",
const std::string& file = "")
{
......@@ -135,8 +143,8 @@ namespace op
// Continue at the end of the file or delete it and re-write it (according to current file size)
const auto maxLogSize = 15 * 1024 * 1024; // 15 MB
std::ofstream loggingFile{fileToOpen,
(currentSizeBytes < maxLogSize ? std::ios_base::app : std::ios_base::trunc)};
std::ofstream loggingFile{
fileToOpen, (currentSizeBytes < maxLogSize ? std::ios_base::app : std::ios_base::trunc)};
// Message to write
loggingFile << getTime();
......@@ -147,21 +155,25 @@ namespace op
loggingFile.close();
}
// Public functions
void error(const std::string& message, const int line, const std::string& function, const std::string& file)
void errorAux(
const int errorMode, const std::string& message, const int line, const std::string& function,
const std::string& file)
{
const std::string errorInit = "\nError:\n";
// errorMode:
// 0: error
// 1: errorWorker
// 2: checkWorkerErrors
// 3: errorDestructor
const std::string errorInitBase = "\nError";
const std::string errorInit = errorInitBase + ":\n";
const std::string errorEnum = "- ";
// Compose error message
std::string errorMessageToPropagate;
std::string errorMessageToPrint;
// If first error
if (message.size() < errorInit.size() || message.substr(0, errorInit.size()) != errorInit)
if (message.size() < errorInitBase.size() || message.substr(0, errorInitBase.size()) != errorInitBase)
{
errorMessageToPrint = errorInit + createFullMessage(message) + "\n\nComing from:\n" + errorEnum
+ createFullMessage("", line, function, file);
......@@ -171,13 +183,32 @@ namespace op
else
{
errorMessageToPrint = errorEnum + createFullMessage("", line, function, file);
errorMessageToPropagate = createFullMessage(message.substr(0, message.size()-1)) + "\n"
+ errorMessageToPrint + "\n";
if (errorMode == 2)
{
const std::string errorThreadLine =
"[All threads closed and control returned to main thread]";
errorMessageToPrint = errorEnum + errorThreadLine + "\n" + errorMessageToPrint;
}
else if (errorMode == 3)
errorMessageToPrint += "\n" + errorEnum + "[Error occurred in a destructor, so no std::exception"
" has been thrown. Returning with exit status 0]";
errorMessageToPropagate = createFullMessage(message) + errorMessageToPrint + "\n";
if (errorMode == 1)
{
errorMessageToPropagate = errorInitBase + " occurred on a thread. OpenPose closed all its"
" threads and then propagated the error to the main thread:\n"
+ errorMessageToPropagate.substr(errorInit.size(), errorMessageToPropagate.size()-1);
}
if (errorMode == 2)
errorMessageToPrint = errorMessageToPropagate.substr(0, errorMessageToPropagate.size()-1);
}
// std::cerr
if (checkIfErrorHas(ErrorMode::StdCerr))
std::cerr << errorMessageToPrint << std::endl;
#ifdef NDEBUG
if (!getIfNotInMainThreadOrEmpty())
#endif
std::cerr << errorMessageToPrint << std::endl;
// File logging
if (checkIfErrorHas(ErrorMode::FileLogging))
......@@ -189,8 +220,76 @@ namespace op
#endif
// std::runtime_error
if (checkIfErrorHas(ErrorMode::StdRuntimeError))
throw std::runtime_error{errorMessageToPropagate};
if (errorMode == 1)
{
sThreadErrors = true;
std::lock_guard<std::mutex> lock{sMutex};
sThreadErrorMessages.emplace_back(errorMessageToPropagate);
}
else
if (checkIfErrorHas(ErrorMode::StdRuntimeError) && errorMode != 3)
throw std::runtime_error{errorMessageToPropagate};
}
// Public functions
void setMainThread()
{
std::lock_guard<std::mutex> lock{sMutex};
sMainThreadId = getThreadId();
}
std::string getThreadId()
{
std::stringstream threadId;
threadId << std::this_thread::get_id();
return threadId.str();
}
bool getIfInMainThreadOrEmpty()
{
std::lock_guard<std::mutex> lock{sMutex};
return (!sMainThreadId.empty() && sMainThreadId == getThreadId());
}
bool getIfNotInMainThreadOrEmpty()
{
std::lock_guard<std::mutex> lock{sMutex};
return (!sMainThreadId.empty() && sMainThreadId != getThreadId());
}
void error(const std::string& message, const int line, const std::string& function, const std::string& file)
{
errorAux(0, message, line, function, file);
}
void checkWorkerErrors()
{
if (sThreadErrors)
{
std::unique_lock<std::mutex> lock{sMutex};
std::string fullMessage = sThreadErrorMessages.at(0);
lock.unlock();
sThreadErrors = false; // Avoid infinity loop throwing the same error over and over.
errorAux(2, fullMessage, __LINE__, __FUNCTION__, __FILE__);
}
}
void errorWorker(const std::string& message, const int line, const std::string& function, const std::string& file)
{
// If we are 100% sure that we are in main thread, then normal error.
// Otherwise, worker error
errorAux((getIfInMainThreadOrEmpty() ? 0 : 1), message, line, function, file);
}
void errorDestructor(const std::string& message, const int line, const std::string& function, const std::string& file)
{
// If we are 100% sure that we are in main thread, then normal error.
// Otherwise, worker error
errorAux(3, message, line, function, file);
}
void log(const std::string& message, const Priority priority, const int line, const std::string& function,
......
......@@ -63,9 +63,7 @@ namespace op
std::string getKey(const int line, const std::string& function, const std::string& file)
{
std::stringstream threadId;
threadId << std::this_thread::get_id();
return file + function + std::to_string(line) + threadId.str();
return file + function + std::to_string(line) + getThreadId();
}
void printAveragedTimeMsCommon(const double timePast, const unsigned long long timeCounter, const int line,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册