OpenPose  1.0.0rc2
OpenPose: A Real-Time Multi-Person Key-Point Detection And Multi-Threading C++ Library
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
wrapper.hpp
Go to the documentation of this file.
1 #ifndef OPENPOSE_WRAPPER_WRAPPER_HPP
2 #define OPENPOSE_WRAPPER_WRAPPER_HPP
3 
12 
13 namespace op
14 {
30  template<typename TDatums,
31  typename TDatumsSP = std::shared_ptr<TDatums>,
32  typename TWorker = std::shared_ptr<Worker<TDatumsSP>>,
33  typename TQueue = Queue<TDatumsSP>>
34  class Wrapper
35  {
36  public:
44  explicit Wrapper(const ThreadManagerMode threadManagerMode = ThreadManagerMode::Synchronous);
45 
50  ~Wrapper();
51 
58  void disableMultiThreading();
59 
66  void setWorkerInput(const TWorker& worker, const bool workerOnNewThread = true);
67 
74  void setWorkerPostProcessing(const TWorker& worker, const bool workerOnNewThread = true);
75 
82  void setWorkerOutput(const TWorker& worker, const bool workerOnNewThread = true);
83 
84  // If output is not required, just use this function until the renderOutput argument. Keep the default values
85  // for the other parameters in order not to display/save any output.
86  void configure(const WrapperStructPose& wrapperStructPose,
87  // Producer: set producerSharedPtr=nullptr or use default WrapperStructInput{} to disable input
88  const WrapperStructInput& wrapperStructInput,
89  // Consumer (keep default values to disable any output)
90  const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{});
91 
92  // Similar to the previos configure, but it includes hand extraction and rendering
93  void configure(const WrapperStructPose& wrapperStructPose,
94  // Hand (use the default WrapperStructHand{} to disable any hand detector)
95  const WrapperStructHand& wrapperStructHand,
96  // Producer: set producerSharedPtr=nullptr or use default WrapperStructInput{} to disable input
97  const WrapperStructInput& wrapperStructInput,
98  // Consumer (keep default values to disable any output)
99  const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{});
100 
101  // Similar to the previos configure, but it includes hand extraction and rendering
102  void configure(const WrapperStructPose& wrapperStructPose,
103  // Face (use the default WrapperStructFace{} to disable any face detector)
104  const WrapperStructFace& wrapperStructFace,
105  // Producer: set producerSharedPtr=nullptr or use default WrapperStructInput{} to disable input
106  const WrapperStructInput& wrapperStructInput,
107  // Consumer (keep default values to disable any output)
108  const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{});
109 
110  // Similar to the previos configure, but it includes hand extraction and rendering
111  void configure(const WrapperStructPose& wrapperStructPose = WrapperStructPose{},
112  // Face (use the default WrapperStructFace{} to disable any face detector)
113  const WrapperStructFace& wrapperStructFace = WrapperStructFace{},
114  // Hand (use the default WrapperStructHand{} to disable any hand detector)
115  const WrapperStructHand& wrapperStructHand = WrapperStructHand{},
116  // Hand (use the default WrapperStructExtra{} to disable any hand detector)
117  const WrapperStructExtra& wrapperStructExtra = WrapperStructExtra{},
118  // Producer: set producerSharedPtr=nullptr or use default WrapperStructInput{} to disable input
119  const WrapperStructInput& wrapperStructInput = WrapperStructInput{},
120  // Consumer (keep default values to disable any output)
121  const WrapperStructOutput& wrapperStructOutput = WrapperStructOutput{});
122 
128  void exec();
129 
139  void start();
140 
145  void stop();
146 
152  bool isRunning() const;
153 
161  bool tryEmplace(TDatumsSP& tDatums);
162 
171  bool waitAndEmplace(TDatumsSP& tDatums);
172 
179  bool tryPush(const TDatumsSP& tDatums);
180 
187  bool waitAndPush(const TDatumsSP& tDatums);
188 
196  bool tryPop(TDatumsSP& tDatums);
197 
206  bool waitAndPop(TDatumsSP& tDatums);
207 
208  private:
209  const ThreadManagerMode mThreadManagerMode;
210  const std::shared_ptr<std::pair<std::atomic<bool>, std::atomic<int>>> spVideoSeek;
211  bool mConfigured;
212  ThreadManager<TDatumsSP> mThreadManager;
213  bool mUserInputWsOnNewThread;
214  bool mUserPostProcessingWsOnNewThread;
215  bool mUserOutputWsOnNewThread;
216  unsigned long long mThreadId;
217  bool mMultiThreadEnabled;
218  // Workers
219  std::vector<TWorker> mUserInputWs;
220  TWorker wDatumProducer;
221  TWorker spWScaleAndSizeExtractor;
222  TWorker spWCvMatToOpInput;
223  TWorker spWCvMatToOpOutput;
224  std::vector<std::vector<TWorker>> spWPoseExtractors;
225  std::vector<std::vector<TWorker>> spWPoseTriangulations;
226  std::vector<std::vector<TWorker>> spWJointAngleEstimations;
227  std::vector<TWorker> mPostProcessingWs;
228  std::vector<TWorker> mUserPostProcessingWs;
229  std::vector<TWorker> mOutputWs;
230  TWorker spWGui;
231  std::vector<TWorker> mUserOutputWs;
232 
239  void reset();
240 
247  void configureThreadManager();
248 
256  unsigned long long threadIdPP();
257 
258  DELETE_COPY(Wrapper);
259  };
260 }
261 
262 
263 
264 
265 
266 // Implementation
267 #include <openpose/3d/headers.hpp>
268 #include <openpose/core/headers.hpp>
269 #include <openpose/face/headers.hpp>
271 #include <openpose/gpu/gpu.hpp>
272 #include <openpose/gui/headers.hpp>
273 #include <openpose/hand/headers.hpp>
274 #include <openpose/pose/headers.hpp>
280 namespace op
281 {
282  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
284  mThreadManagerMode{threadManagerMode},
285  spVideoSeek{std::make_shared<std::pair<std::atomic<bool>, std::atomic<int>>>()},
286  mConfigured{false},
287  mThreadManager{threadManagerMode},
288  mMultiThreadEnabled{true}
289  {
290  try
291  {
292  // It cannot be directly included in the constructor (compiler error for copying std::atomic)
293  spVideoSeek->first = false;
294  spVideoSeek->second = 0;
295  }
296  catch (const std::exception& e)
297  {
298  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
299  }
300  }
301 
302  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
304  {
305  try
306  {
307  stop();
308  reset();
309  }
310  catch (const std::exception& e)
311  {
312  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
313  }
314  }
315 
316  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
318  {
319  try
320  {
321  mMultiThreadEnabled = false;
322  }
323  catch (const std::exception& e)
324  {
325  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
326  }
327  }
328 
329  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
331  const bool workerOnNewThread)
332  {
333  try
334  {
335  mUserInputWs.clear();
336  if (worker == nullptr)
337  error("Your worker is a nullptr.", __LINE__, __FILE__, __FUNCTION__);
338  mUserInputWs.emplace_back(worker);
339  mUserInputWsOnNewThread = {workerOnNewThread};
340  }
341  catch (const std::exception& e)
342  {
343  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
344  }
345  }
346 
347  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
349  const bool workerOnNewThread)
350  {
351  try
352  {
353  mUserPostProcessingWs.clear();
354  if (worker == nullptr)
355  error("Your worker is a nullptr.", __LINE__, __FILE__, __FUNCTION__);
356  mUserPostProcessingWs.emplace_back(worker);
357  mUserPostProcessingWsOnNewThread = {workerOnNewThread};
358  }
359  catch (const std::exception& e)
360  {
361  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
362  }
363  }
364 
365  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
367  const bool workerOnNewThread)
368  {
369  try
370  {
371  mUserOutputWs.clear();
372  if (worker == nullptr)
373  error("Your worker is a nullptr.", __LINE__, __FILE__, __FUNCTION__);
374  mUserOutputWs.emplace_back(worker);
375  mUserOutputWsOnNewThread = {workerOnNewThread};
376  }
377  catch (const std::exception& e)
378  {
379  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
380  }
381  }
382 
383  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
385  const WrapperStructInput& wrapperStructInput,
386  const WrapperStructOutput& wrapperStructOutput)
387  {
388  try
389  {
390  configure(wrapperStructPose, WrapperStructFace{}, WrapperStructHand{},
391  WrapperStructExtra{}, wrapperStructInput, wrapperStructOutput);
392  }
393  catch (const std::exception& e)
394  {
395  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
396  }
397  }
398 
399  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
401  const WrapperStructFace& wrapperStructFace,
402  const WrapperStructInput& wrapperStructInput,
403  const WrapperStructOutput& wrapperStructOutput)
404  {
405  try
406  {
407  configure(wrapperStructPose, wrapperStructFace, WrapperStructHand{},
408  WrapperStructExtra{}, wrapperStructInput, wrapperStructOutput);
409  }
410  catch (const std::exception& e)
411  {
412  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
413  }
414  }
415 
416  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
418  const WrapperStructHand& wrapperStructHand,
419  const WrapperStructInput& wrapperStructInput,
420  const WrapperStructOutput& wrapperStructOutput)
421  {
422  try
423  {
424  configure(wrapperStructPose, WrapperStructFace{}, wrapperStructHand,
425  WrapperStructExtra{}, wrapperStructInput, wrapperStructOutput);
426  }
427  catch (const std::exception& e)
428  {
429  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
430  }
431  }
432 
433  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
435  const WrapperStructFace& wrapperStructFace,
436  const WrapperStructHand& wrapperStructHand,
437  const WrapperStructExtra& wrapperStructExtra,
438  const WrapperStructInput& wrapperStructInput,
439  const WrapperStructOutput& wrapperStructOutput)
440  {
441  try
442  {
443  auto wrapperStructPose = wrapperStructPoseTemp;
444  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
445 
446  // Required parameters
447  const auto renderOutput = wrapperStructPose.renderMode != RenderMode::None
448  || wrapperStructFace.renderMode != RenderMode::None
449  || wrapperStructHand.renderMode != RenderMode::None;
450  const auto renderOutputGpu = wrapperStructPose.renderMode == RenderMode::Gpu
451  || wrapperStructFace.renderMode == RenderMode::Gpu
452  || wrapperStructHand.renderMode == RenderMode::Gpu;
453  const auto renderFace = wrapperStructFace.enable && wrapperStructFace.renderMode != RenderMode::None;
454  const auto renderHand = wrapperStructHand.enable && wrapperStructHand.renderMode != RenderMode::None;
455  const auto renderHandGpu = wrapperStructHand.enable && wrapperStructHand.renderMode == RenderMode::Gpu;
456 
457  // Check no wrong/contradictory flags enabled
458  const auto userOutputWsEmpty = mUserOutputWs.empty();
459  wrapperConfigureSecurityChecks(wrapperStructPose, wrapperStructFace, wrapperStructHand, wrapperStructExtra,
460  wrapperStructInput, wrapperStructOutput, renderOutput, userOutputWsEmpty,
461  mThreadManagerMode);
462 
463  // Get number threads
464  auto numberThreads = wrapperStructPose.gpuNumber;
465  auto gpuNumberStart = wrapperStructPose.gpuNumberStart;
466  // CPU --> 1 thread or no pose extraction
467  if (getGpuMode() == GpuMode::NoGpu)
468  {
469  numberThreads = (wrapperStructPose.gpuNumber == 0 ? 0 : 1);
470  gpuNumberStart = 0;
471  // Disabling multi-thread makes the code 400 ms faster (2.3 sec vs. 2.7 in i7-6850K)
472  // and fixes the bug that the screen was not properly displayed and only refreshed sometimes
473  // Note: The screen bug could be also fixed by using waitKey(30) rather than waitKey(1)
474  disableMultiThreading();
475  }
476  // GPU --> user picks (<= #GPUs)
477  else
478  {
479  // Get total number GPUs
480  const auto totalGpuNumber = getGpuNumber();
481  // If number GPU < 0 --> set it to all the available GPUs
482  if (numberThreads < 0)
483  {
484  if (totalGpuNumber <= gpuNumberStart)
485  error("Number of initial GPU (`--number_gpu_start`) must be lower than the total number of"
486  " used GPUs (`--number_gpu`)", __LINE__, __FUNCTION__, __FILE__);
487  numberThreads = totalGpuNumber - gpuNumberStart;
488  // Reset initial GPU to 0 (we want them all)
489  // Logging message
490  log("Auto-detecting all available GPUs... Detected " + std::to_string(totalGpuNumber)
491  + " GPU(s), using " + std::to_string(numberThreads) + " of them starting at GPU "
492  + std::to_string(gpuNumberStart) + ".", Priority::High);
493  }
494  // Security check
495  if (gpuNumberStart + numberThreads > totalGpuNumber)
496  error("Initial GPU selected (`--number_gpu_start`) + number GPUs to use (`--number_gpu`) must"
497  " be lower or equal than the total number of GPUs in your machine ("
498  + std::to_string(gpuNumberStart) + " + "
499  + std::to_string(numberThreads) + " vs. "
500  + std::to_string(totalGpuNumber) + ").",
501  __LINE__, __FUNCTION__, __FILE__);
502  }
503 
504  // Proper format
505  const auto writeImagesCleaned = formatAsDirectory(wrapperStructOutput.writeImages);
506  const auto writeKeypointCleaned = formatAsDirectory(wrapperStructOutput.writeKeypoint);
507  const auto writeJsonCleaned = formatAsDirectory(wrapperStructOutput.writeJson);
508  const auto writeHeatMapsCleaned = formatAsDirectory(wrapperStructOutput.writeHeatMaps);
509  const auto modelFolder = formatAsDirectory(wrapperStructPose.modelFolder);
510 
511  // Common parameters
512  auto finalOutputSize = wrapperStructPose.outputSize;
513  Point<int> producerSize{-1,-1};
514  const auto oPProducer = (wrapperStructInput.producerSharedPtr != nullptr);
515  if (oPProducer)
516  {
517  // 1. Set producer properties
518  const auto displayProducerFpsMode = (wrapperStructInput.realTimeProcessing
520  wrapperStructInput.producerSharedPtr->setProducerFpsMode(displayProducerFpsMode);
521  wrapperStructInput.producerSharedPtr->set(ProducerProperty::Flip, wrapperStructInput.frameFlip);
522  wrapperStructInput.producerSharedPtr->set(ProducerProperty::Rotation, wrapperStructInput.frameRotate);
523  wrapperStructInput.producerSharedPtr->set(ProducerProperty::AutoRepeat,
524  wrapperStructInput.framesRepeat);
525  // 2. Set finalOutputSize
526  producerSize = Point<int>{(int)wrapperStructInput.producerSharedPtr->get(CV_CAP_PROP_FRAME_WIDTH),
527  (int)wrapperStructInput.producerSharedPtr->get(CV_CAP_PROP_FRAME_HEIGHT)};
528  // Set finalOutputSize to input size if desired
529  if (finalOutputSize.x == -1 || finalOutputSize.y == -1)
530  finalOutputSize = producerSize;
531  }
532 
533  // Producer
534  if (oPProducer)
535  {
536  const auto datumProducer = std::make_shared<DatumProducer<TDatums>>(
537  wrapperStructInput.producerSharedPtr, wrapperStructInput.frameFirst, wrapperStructInput.frameLast,
538  spVideoSeek
539  );
540  wDatumProducer = std::make_shared<WDatumProducer<TDatumsSP, TDatums>>(datumProducer);
541  }
542  else
543  wDatumProducer = nullptr;
544 
545  std::vector<std::shared_ptr<PoseExtractorNet>> poseExtractorNets;
546  std::vector<std::shared_ptr<FaceExtractorNet>> faceExtractorNets;
547  std::vector<std::shared_ptr<HandExtractorNet>> handExtractorNets;
548  std::vector<std::shared_ptr<PoseGpuRenderer>> poseGpuRenderers;
549  std::shared_ptr<PoseCpuRenderer> poseCpuRenderer;
550  if (numberThreads > 0)
551  {
552  // Get input scales and sizes
553  const auto scaleAndSizeExtractor = std::make_shared<ScaleAndSizeExtractor>(
554  wrapperStructPose.netInputSize, finalOutputSize, wrapperStructPose.scalesNumber,
555  wrapperStructPose.scaleGap
556  );
557  spWScaleAndSizeExtractor = std::make_shared<WScaleAndSizeExtractor<TDatumsSP>>(scaleAndSizeExtractor);
558 
559  // Input cvMat to OpenPose input & output format
560  const auto cvMatToOpInput = std::make_shared<CvMatToOpInput>(wrapperStructPose.poseModel);
561  spWCvMatToOpInput = std::make_shared<WCvMatToOpInput<TDatumsSP>>(cvMatToOpInput);
562  if (renderOutput)
563  {
564  const auto cvMatToOpOutput = std::make_shared<CvMatToOpOutput>();
565  spWCvMatToOpOutput = std::make_shared<WCvMatToOpOutput<TDatumsSP>>(cvMatToOpOutput);
566  }
567 
568  // Pose estimators & renderers
569  std::vector<TWorker> cpuRenderers;
570  spWPoseExtractors.clear();
571  spWPoseExtractors.resize(numberThreads);
572  if (wrapperStructPose.enable)
573  {
574  // Pose estimators
575  for (auto gpuId = 0; gpuId < numberThreads; gpuId++)
576  poseExtractorNets.emplace_back(std::make_shared<PoseExtractorCaffe>(
577  wrapperStructPose.poseModel, modelFolder, gpuId + gpuNumberStart,
578  wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScale,
579  wrapperStructPose.addPartCandidates, wrapperStructPose.enableGoogleLogging
580  ));
581 
582  // Pose renderers
583  if (renderOutputGpu || wrapperStructPose.renderMode == RenderMode::Cpu)
584  {
585  // If wrapperStructPose.renderMode != RenderMode::Gpu but renderOutput, then we create an
586  // alpha = 0 pose renderer in order to keep the removing background option
587  const auto alphaKeypoint = (wrapperStructPose.renderMode != RenderMode::None
588  ? wrapperStructPose.alphaKeypoint : 0.f);
589  const auto alphaHeatMap = (wrapperStructPose.renderMode != RenderMode::None
590  ? wrapperStructPose.alphaHeatMap : 0.f);
591  // GPU rendering
592  if (renderOutputGpu)
593  {
594  for (const auto& poseExtractorNet : poseExtractorNets)
595  {
596  poseGpuRenderers.emplace_back(std::make_shared<PoseGpuRenderer>(
597  wrapperStructPose.poseModel, poseExtractorNet, wrapperStructPose.renderThreshold,
598  wrapperStructPose.blendOriginalFrame, alphaKeypoint,
599  alphaHeatMap, wrapperStructPose.defaultPartToRender
600  ));
601  }
602  }
603  // CPU rendering
604  if (wrapperStructPose.renderMode == RenderMode::Cpu)
605  {
606  poseCpuRenderer = std::make_shared<PoseCpuRenderer>(
607  wrapperStructPose.poseModel, wrapperStructPose.renderThreshold,
608  wrapperStructPose.blendOriginalFrame, alphaKeypoint, alphaHeatMap,
609  wrapperStructPose.defaultPartToRender);
610  cpuRenderers.emplace_back(std::make_shared<WPoseRenderer<TDatumsSP>>(poseCpuRenderer));
611  }
612  }
613  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
614 
615  // Pose extractor(s)
616  spWPoseExtractors.resize(poseExtractorNets.size());
617  const auto personIdExtractor = (wrapperStructExtra.identification
618  ? std::make_shared<PersonIdExtractor>() : nullptr);
619  // Keep top N people
620  // Added right after PoseExtractorNet to avoid:
621  // 1) Rendering people that are later deleted (wrong visualization).
622  // 2) Processing faces and hands on people that will be deleted (speed up).
623  // 3) Running tracking before deleting the people.
624  // Add KeepTopNPeople for each PoseExtractorNet
625  const auto keepTopNPeople = (wrapperStructPose.numberPeopleMax > 0 ?
626  std::make_shared<KeepTopNPeople>(wrapperStructPose.numberPeopleMax)
627  : nullptr);
628  // Person tracker
629  auto personTrackers = std::make_shared<std::vector<std::shared_ptr<PersonTracker>>>();
630  if (wrapperStructExtra.tracking > -1)
631  personTrackers->emplace_back(
632  std::make_shared<PersonTracker>(wrapperStructExtra.tracking == 0));
633  for (auto i = 0u; i < spWPoseExtractors.size(); i++)
634  {
635  // OpenPose keypoint detector + keepTopNPeople
636  // + ID extractor (experimental) + tracking (experimental)
637  const auto poseExtractor = std::make_shared<PoseExtractor>(
638  poseExtractorNets.at(i), keepTopNPeople, personIdExtractor, personTrackers,
639  wrapperStructPose.numberPeopleMax, wrapperStructExtra.tracking);
640  spWPoseExtractors.at(i) = {std::make_shared<WPoseExtractor<TDatumsSP>>(poseExtractor)};
641  // // Just OpenPose keypoint detector
642  // spWPoseExtractors.at(i) = {std::make_shared<WPoseExtractorNet<TDatumsSP>>(
643  // poseExtractorNets.at(i))};
644  }
645 
646  // // (Before tracking / id extractor)
647  // // Added right after PoseExtractorNet to avoid:
648  // // 1) Rendering people that are later deleted (wrong visualization).
649  // // 2) Processing faces and hands on people that will be deleted (speed up).
650  // if (wrapperStructPose.numberPeopleMax > 0)
651  // {
652  // // Add KeepTopNPeople for each PoseExtractorNet
653  // const auto keepTopNPeople = std::make_shared<KeepTopNPeople>(
654  // wrapperStructPose.numberPeopleMax);
655  // for (auto& wPose : spWPoseExtractors)
656  // wPose.emplace_back(std::make_shared<WKeepTopNPeople<TDatumsSP>>(keepTopNPeople));
657  // }
658  }
659 
660 
661  // Face extractor(s)
662  if (wrapperStructFace.enable)
663  {
664  // Face detector
665  // OpenPose face detector
666  if (wrapperStructPose.enable)
667  {
668  const auto faceDetector = std::make_shared<FaceDetector>(wrapperStructPose.poseModel);
669  for (auto& wPose : spWPoseExtractors)
670  wPose.emplace_back(std::make_shared<WFaceDetector<TDatumsSP>>(faceDetector));
671  }
672  // OpenCV face detector
673  else
674  {
675  log("Body keypoint detection is disabled. Hence, using OpenCV face detector (much less"
676  " accurate but faster).", Priority::High);
677  for (auto& wPose : spWPoseExtractors)
678  {
679  // 1 FaceDetectorOpenCV per thread, OpenCV face detector is not thread-safe
680  const auto faceDetectorOpenCV = std::make_shared<FaceDetectorOpenCV>(modelFolder);
681  wPose.emplace_back(
682  std::make_shared<WFaceDetectorOpenCV<TDatumsSP>>(faceDetectorOpenCV)
683  );
684  }
685  }
686  // Face keypoint extractor
687  for (auto gpu = 0u; gpu < spWPoseExtractors.size(); gpu++)
688  {
689  // Face keypoint extractor
690  const auto netOutputSize = wrapperStructFace.netInputSize;
691  const auto faceExtractorNet = std::make_shared<FaceExtractorCaffe>(
692  wrapperStructFace.netInputSize, netOutputSize, modelFolder,
693  gpu + gpuNumberStart, wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScale,
694  wrapperStructPose.enableGoogleLogging
695  );
696  faceExtractorNets.emplace_back(faceExtractorNet);
697  spWPoseExtractors.at(gpu).emplace_back(
698  std::make_shared<WFaceExtractorNet<TDatumsSP>>(faceExtractorNet));
699  }
700  }
701 
702  // Hand extractor(s)
703  if (wrapperStructHand.enable)
704  {
705  const auto handDetector = std::make_shared<HandDetector>(wrapperStructPose.poseModel);
706  for (auto gpu = 0u; gpu < spWPoseExtractors.size(); gpu++)
707  {
708  // Hand detector
709  // If tracking
710  if (wrapperStructHand.tracking)
711  spWPoseExtractors.at(gpu).emplace_back(
712  std::make_shared<WHandDetectorTracking<TDatumsSP>>(handDetector)
713  );
714  // If detection
715  else
716  spWPoseExtractors.at(gpu).emplace_back(
717  std::make_shared<WHandDetector<TDatumsSP>>(handDetector));
718  // Hand keypoint extractor
719  const auto netOutputSize = wrapperStructHand.netInputSize;
720  const auto handExtractorNet = std::make_shared<HandExtractorCaffe>(
721  wrapperStructHand.netInputSize, netOutputSize, modelFolder,
722  gpu + gpuNumberStart, wrapperStructHand.scalesNumber, wrapperStructHand.scaleRange,
723  wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScale,
724  wrapperStructPose.enableGoogleLogging
725  );
726  handExtractorNets.emplace_back(handExtractorNet);
727  spWPoseExtractors.at(gpu).emplace_back(
728  std::make_shared<WHandExtractorNet<TDatumsSP>>(handExtractorNet)
729  );
730  // If tracking
731  if (wrapperStructHand.tracking)
732  spWPoseExtractors.at(gpu).emplace_back(
733  std::make_shared<WHandDetectorUpdate<TDatumsSP>>(handDetector)
734  );
735  }
736  }
737 
738  // Pose renderer(s)
739  if (!poseGpuRenderers.empty())
740  for (auto i = 0u; i < spWPoseExtractors.size(); i++)
741  spWPoseExtractors.at(i).emplace_back(std::make_shared<WPoseRenderer<TDatumsSP>>(
742  poseGpuRenderers.at(i)
743  ));
744 
745  // Face renderer(s)
746  if (renderFace)
747  {
748  // CPU rendering
749  if (wrapperStructFace.renderMode == RenderMode::Cpu)
750  {
751  // Construct face renderer
752  const auto faceRenderer = std::make_shared<FaceCpuRenderer>(wrapperStructFace.renderThreshold,
753  wrapperStructFace.alphaKeypoint,
754  wrapperStructFace.alphaHeatMap);
755  // Add worker
756  cpuRenderers.emplace_back(std::make_shared<WFaceRenderer<TDatumsSP>>(faceRenderer));
757  }
758  // GPU rendering
759  else if (wrapperStructFace.renderMode == RenderMode::Gpu)
760  {
761  for (auto i = 0u; i < spWPoseExtractors.size(); i++)
762  {
763  // Construct face renderer
764  const auto faceRenderer = std::make_shared<FaceGpuRenderer>(
765  wrapperStructFace.renderThreshold, wrapperStructFace.alphaKeypoint,
766  wrapperStructFace.alphaHeatMap
767  );
768  // Performance boost -> share spGpuMemory for all renderers
769  if (!poseGpuRenderers.empty())
770  {
771  const bool isLastRenderer = !renderHandGpu;
772  const auto renderer = std::static_pointer_cast<PoseGpuRenderer>(
773  poseGpuRenderers.at(i)
774  );
775  faceRenderer->setSharedParametersAndIfLast(renderer->getSharedParameters(),
776  isLastRenderer);
777  }
778  // Add worker
779  spWPoseExtractors.at(i).emplace_back(
780  std::make_shared<WFaceRenderer<TDatumsSP>>(faceRenderer));
781  }
782  }
783  else
784  error("Unknown RenderMode.", __LINE__, __FUNCTION__, __FILE__);
785  }
786 
787  // Hand renderer(s)
788  if (renderHand)
789  {
790  // CPU rendering
791  if (wrapperStructHand.renderMode == RenderMode::Cpu)
792  {
793  // Construct hand renderer
794  const auto handRenderer = std::make_shared<HandCpuRenderer>(wrapperStructHand.renderThreshold,
795  wrapperStructHand.alphaKeypoint,
796  wrapperStructHand.alphaHeatMap);
797  // Add worker
798  cpuRenderers.emplace_back(std::make_shared<WHandRenderer<TDatumsSP>>(handRenderer));
799  }
800  // GPU rendering
801  else if (wrapperStructHand.renderMode == RenderMode::Gpu)
802  {
803  for (auto i = 0u; i < spWPoseExtractors.size(); i++)
804  {
805  // Construct hands renderer
806  const auto handRenderer = std::make_shared<HandGpuRenderer>(
807  wrapperStructHand.renderThreshold, wrapperStructHand.alphaKeypoint,
808  wrapperStructHand.alphaHeatMap
809  );
810  // Performance boost -> share spGpuMemory for all renderers
811  if (!poseGpuRenderers.empty())
812  {
813  const bool isLastRenderer = true;
814  const auto renderer = std::static_pointer_cast<PoseGpuRenderer>(
815  poseGpuRenderers.at(i)
816  );
817  handRenderer->setSharedParametersAndIfLast(renderer->getSharedParameters(),
818  isLastRenderer);
819  }
820  // Add worker
821  spWPoseExtractors.at(i).emplace_back(
822  std::make_shared<WHandRenderer<TDatumsSP>>(handRenderer));
823  }
824  }
825  else
826  error("Unknown RenderMode.", __LINE__, __FUNCTION__, __FILE__);
827  }
828 
829  // 3-D reconstruction
830  spWPoseTriangulations.clear();
831  if (wrapperStructExtra.reconstruct3d)
832  {
833  // For all (body/face/hands): PoseTriangulations ~30 msec, 8 GPUS ~30 msec for keypoint estimation
834  spWPoseTriangulations.resize(fastMax(1, int(spWPoseExtractors.size() / 4)));
835  for (auto i = 0u ; i < spWPoseTriangulations.size() ; i++)
836  {
837  const auto poseTriangulation = std::make_shared<PoseTriangulation>(
838  wrapperStructExtra.minViews3d);
839  spWPoseTriangulations.at(i) = {std::make_shared<WPoseTriangulation<TDatumsSP>>(
840  poseTriangulation)};
841  }
842  }
843  // Itermediate workers (e.g. OpenPose format to cv::Mat, json & frames recorder, ...)
844  mPostProcessingWs.clear();
845  // // Person ID identification (when no multi-thread and no dependency on tracking)
846  // if (wrapperStructExtra.identification)
847  // {
848  // const auto personIdExtractor = std::make_shared<PersonIdExtractor>();
849  // mPostProcessingWs.emplace_back(
850  // std::make_shared<WPersonIdExtractor<TDatumsSP>>(personIdExtractor)
851  // );
852  // }
853  // Frames processor (OpenPose format -> cv::Mat format)
854  if (renderOutput)
855  {
856  mPostProcessingWs = mergeVectors(mPostProcessingWs, cpuRenderers);
857  const auto opOutputToCvMat = std::make_shared<OpOutputToCvMat>();
858  mPostProcessingWs.emplace_back(std::make_shared<WOpOutputToCvMat<TDatumsSP>>(opOutputToCvMat));
859  }
860  // Re-scale pose if desired
861  // If desired scale is not the current input
862  if (wrapperStructPose.keypointScale != ScaleMode::InputResolution
863  // and desired scale is not output when size(input) = size(output)
864  && !(wrapperStructPose.keypointScale == ScaleMode::OutputResolution &&
865  (finalOutputSize == producerSize || finalOutputSize.x <= 0 || finalOutputSize.y <= 0))
866  // and desired scale is not net output when size(input) = size(net output)
867  && !(wrapperStructPose.keypointScale == ScaleMode::NetOutputResolution
868  && producerSize == wrapperStructPose.netInputSize))
869  {
870  // Then we must rescale the keypoints
871  auto keypointScaler = std::make_shared<KeypointScaler>(wrapperStructPose.keypointScale);
872  mPostProcessingWs.emplace_back(std::make_shared<WKeypointScaler<TDatumsSP>>(keypointScaler));
873  }
874  }
875 
876  // IK/Adam
877  const auto displayAdam = wrapperStructOutput.displayMode == DisplayMode::DisplayAdam
878  || (wrapperStructOutput.displayMode == DisplayMode::DisplayAll
879  && wrapperStructExtra.ikThreads > 0);
880  spWJointAngleEstimations.clear();
881 #ifdef USE_3D_ADAM_MODEL
882  if (wrapperStructExtra.ikThreads > 0)
883  {
884  spWJointAngleEstimations.resize(wrapperStructExtra.ikThreads);
885  // Pose extractor(s)
886  for (auto i = 0u; i < spWJointAngleEstimations.size(); i++)
887  {
888  const auto jointAngleEstimation = std::make_shared<JointAngleEstimation>(displayAdam);
889  spWJointAngleEstimations.at(i) = {std::make_shared<WJointAngleEstimation<TDatumsSP>>(
890  jointAngleEstimation)};
891  }
892  }
893 #endif
894 
895  // Output workers
896  mOutputWs.clear();
897  // Send information (e.g., to Unity) though UDP client-server communication
898 #ifdef USE_3D_ADAM_MODEL
899  if (!wrapperStructOutput.udpHost.empty() && !wrapperStructOutput.udpPort.empty())
900  {
901  const auto udpSender = std::make_shared<UdpSender>(wrapperStructOutput.udpHost,
902  wrapperStructOutput.udpPort);
903  mOutputWs.emplace_back(std::make_shared<WUdpSender<TDatumsSP>>(udpSender));
904  }
905 #endif
906  // Write people pose data on disk (json for OpenCV >= 3, xml, yml...)
907  if (!writeKeypointCleaned.empty())
908  {
909  const auto keypointSaver = std::make_shared<KeypointSaver>(writeKeypointCleaned,
910  wrapperStructOutput.writeKeypointFormat);
911  mOutputWs.emplace_back(std::make_shared<WPoseSaver<TDatumsSP>>(keypointSaver));
912  if (wrapperStructFace.enable)
913  mOutputWs.emplace_back(std::make_shared<WFaceSaver<TDatumsSP>>(keypointSaver));
914  if (wrapperStructHand.enable)
915  mOutputWs.emplace_back(std::make_shared<WHandSaver<TDatumsSP>>(keypointSaver));
916  }
917  // Write OpenPose output data on disk in json format (body/hand/face keypoints, body part locations if
918  // enabled, etc.)
919  if (!writeJsonCleaned.empty())
920  {
921  const auto peopleJsonSaver = std::make_shared<PeopleJsonSaver>(writeJsonCleaned);
922  mOutputWs.emplace_back(std::make_shared<WPeopleJsonSaver<TDatumsSP>>(peopleJsonSaver));
923  }
924  // Write people pose data on disk (COCO validation json format)
925  if (!wrapperStructOutput.writeCocoJson.empty())
926  {
927  // If humanFormat: bigger size (& maybe slower to process), but easier for user to read it
928  const auto humanFormat = true;
929  const auto cocoJsonSaver = std::make_shared<CocoJsonSaver>(wrapperStructOutput.writeCocoJson,
930  humanFormat, CocoJsonFormat::Body);
931  mOutputWs.emplace_back(std::make_shared<WCocoJsonSaver<TDatumsSP>>(cocoJsonSaver));
932  }
933  // Write people foot pose data on disk (COCO validation json format for foot data)
934  if (!wrapperStructOutput.writeCocoFootJson.empty())
935  {
936  // If humanFormat: bigger size (& maybe slower to process), but easier for user to read it
937  const auto humanFormat = true;
938  const auto cocoJsonSaver = std::make_shared<CocoJsonSaver>(wrapperStructOutput.writeCocoFootJson,
939  humanFormat, CocoJsonFormat::Foot);
940  mOutputWs.emplace_back(std::make_shared<WCocoJsonSaver<TDatumsSP>>(cocoJsonSaver));
941  }
942  // Write frames as desired image format on hard disk
943  if (!writeImagesCleaned.empty())
944  {
945  const auto imageSaver = std::make_shared<ImageSaver>(writeImagesCleaned,
946  wrapperStructOutput.writeImagesFormat);
947  mOutputWs.emplace_back(std::make_shared<WImageSaver<TDatumsSP>>(imageSaver));
948  }
949  // Write frames as *.avi video on hard disk
950  const auto producerFps = (wrapperStructInput.producerSharedPtr == nullptr ?
951  0. : wrapperStructInput.producerSharedPtr->get(CV_CAP_PROP_FPS));
952  const auto originalVideoFps = (wrapperStructOutput.writeVideoFps > 0 ?
953  wrapperStructOutput.writeVideoFps
954  : producerFps);
955  if (!wrapperStructOutput.writeVideo.empty())
956  {
957  if (!oPProducer)
958  error("Video file can only be recorded inside `wrapper/wrapper.hpp` if the producer"
959  " is one of the default ones (e.g. video, webcam, ...).",
960  __LINE__, __FUNCTION__, __FILE__);
961  if (finalOutputSize.x <= 0 || finalOutputSize.y <= 0)
962  error("Video can only be recorded if outputSize is fixed (e.g. video, webcam, IP camera),"
963  "but not for a image directory.", __LINE__, __FUNCTION__, __FILE__);
964  const auto videoSaver = std::make_shared<VideoSaver>(
965  wrapperStructOutput.writeVideo, CV_FOURCC('M','J','P','G'), originalVideoFps, finalOutputSize
966  );
967  mOutputWs.emplace_back(std::make_shared<WVideoSaver<TDatumsSP>>(videoSaver));
968  }
969  // Write joint angles as *.bvh file on hard disk
970 #ifdef USE_3D_ADAM_MODEL
971  if (!wrapperStructOutput.writeBvh.empty())
972  {
973  const auto bvhSaver = std::make_shared<BvhSaver>(
974  wrapperStructOutput.writeBvh, JointAngleEstimation::getTotalModel(), originalVideoFps
975  );
976  mOutputWs.emplace_back(std::make_shared<WBvhSaver<TDatumsSP>>(bvhSaver));
977  }
978 #endif
979  // Write heat maps as desired image format on hard disk
980  if (!writeHeatMapsCleaned.empty())
981  {
982  const auto heatMapSaver = std::make_shared<HeatMapSaver>(writeHeatMapsCleaned,
983  wrapperStructOutput.writeHeatMapsFormat);
984  mOutputWs.emplace_back(std::make_shared<WHeatMapSaver<TDatumsSP>>(heatMapSaver));
985  }
986  // Add frame information for GUI
987  const bool guiEnabled = (wrapperStructOutput.displayMode != DisplayMode::NoDisplay);
988  // If this WGuiInfoAdder instance is placed before the WImageSaver or WVideoSaver, then the resulting
989  // recorded frames will look exactly as the final displayed image by the GUI
990  if (wrapperStructOutput.guiVerbose && (guiEnabled || !mUserOutputWs.empty()
991  || mThreadManagerMode == ThreadManagerMode::Asynchronous
992  || mThreadManagerMode == ThreadManagerMode::AsynchronousOut))
993  {
994  const auto guiInfoAdder = std::make_shared<GuiInfoAdder>(numberThreads, guiEnabled);
995  mOutputWs.emplace_back(std::make_shared<WGuiInfoAdder<TDatumsSP>>(guiInfoAdder));
996  }
997  // Minimal graphical user interface (GUI)
998  spWGui = nullptr;
999  if (guiEnabled)
1000  {
1001  // PoseRenderers to Renderers
1002  std::vector<std::shared_ptr<Renderer>> renderers;
1003  if (wrapperStructPose.renderMode == RenderMode::Cpu)
1004  renderers.emplace_back(std::static_pointer_cast<Renderer>(poseCpuRenderer));
1005  else
1006  for (const auto& poseGpuRenderer : poseGpuRenderers)
1007  renderers.emplace_back(std::static_pointer_cast<Renderer>(poseGpuRenderer));
1008  // Display
1009  // Adam (+3-D/2-D) display
1010  if (displayAdam)
1011  {
1012 #ifdef USE_3D_ADAM_MODEL
1013  // Gui
1014  const auto gui = std::make_shared<GuiAdam>(
1015  finalOutputSize, wrapperStructOutput.fullScreen, mThreadManager.getIsRunningSharedPtr(),
1016  spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers,
1017  wrapperStructOutput.displayMode, JointAngleEstimation::getTotalModel(),
1018  wrapperStructOutput.writeVideoAdam
1019  );
1020  // WGui
1021  spWGui = {std::make_shared<WGuiAdam<TDatumsSP>>(gui)};
1022 #endif
1023  }
1024  // 3-D (+2-D) display
1025  else if (wrapperStructOutput.displayMode == DisplayMode::Display3D
1026  || wrapperStructOutput.displayMode == DisplayMode::DisplayAll)
1027  {
1028  // Gui
1029  const auto gui = std::make_shared<Gui3D>(
1030  finalOutputSize, wrapperStructOutput.fullScreen, mThreadManager.getIsRunningSharedPtr(),
1031  spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers,
1032  wrapperStructPose.poseModel, wrapperStructOutput.displayMode
1033  );
1034  // WGui
1035  spWGui = {std::make_shared<WGui3D<TDatumsSP>>(gui)};
1036  }
1037  // 2-D display
1038  else if (wrapperStructOutput.displayMode == DisplayMode::Display2D)
1039  {
1040  // Gui
1041  const auto gui = std::make_shared<Gui>(
1042  finalOutputSize, wrapperStructOutput.fullScreen, mThreadManager.getIsRunningSharedPtr(),
1043  spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers
1044  );
1045  // WGui
1046  spWGui = {std::make_shared<WGui<TDatumsSP>>(gui)};
1047  }
1048  else
1049  error("Unknown DisplayMode.", __LINE__, __FUNCTION__, __FILE__);
1050  }
1051  // Set wrapper as configured
1052  mConfigured = true;
1053  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1054  }
1055  catch (const std::exception& e)
1056  {
1057  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1058  }
1059  }
1060 
1061  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1063  {
1064  try
1065  {
1066  configureThreadManager();
1067  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1068  mThreadManager.exec();
1069  }
1070  catch (const std::exception& e)
1071  {
1072  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1073  }
1074  }
1075 
1076  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1078  {
1079  try
1080  {
1081  configureThreadManager();
1082  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1083  mThreadManager.start();
1084  }
1085  catch (const std::exception& e)
1086  {
1087  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1088  }
1089  }
1090 
1091  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1093  {
1094  try
1095  {
1096  mThreadManager.stop();
1097  }
1098  catch (const std::exception& e)
1099  {
1100  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1101  }
1102  }
1103 
1104  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1106  {
1107  try
1108  {
1109  return mThreadManager.isRunning();
1110  }
1111  catch (const std::exception& e)
1112  {
1113  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1114  return false;
1115  }
1116  }
1117 
1118  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1120  {
1121  try
1122  {
1123  if (!mUserInputWs.empty())
1124  error("Emplace cannot be called if an input worker was already selected.",
1125  __LINE__, __FUNCTION__, __FILE__);
1126  return mThreadManager.tryEmplace(tDatums);
1127  }
1128  catch (const std::exception& e)
1129  {
1130  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1131  return false;
1132  }
1133  }
1134 
1135  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1137  {
1138  try
1139  {
1140  if (!mUserInputWs.empty())
1141  error("Emplace cannot be called if an input worker was already selected.",
1142  __LINE__, __FUNCTION__, __FILE__);
1143  return mThreadManager.waitAndEmplace(tDatums);
1144  }
1145  catch (const std::exception& e)
1146  {
1147  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1148  return false;
1149  }
1150  }
1151 
1152  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1154  {
1155  try
1156  {
1157  if (!mUserInputWs.empty())
1158  error("Push cannot be called if an input worker was already selected.",
1159  __LINE__, __FUNCTION__, __FILE__);
1160  return mThreadManager.tryPush(tDatums);
1161  }
1162  catch (const std::exception& e)
1163  {
1164  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1165  return false;
1166  }
1167  }
1168 
1169  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1171  {
1172  try
1173  {
1174  if (!mUserInputWs.empty())
1175  error("Push cannot be called if an input worker was already selected.",
1176  __LINE__, __FUNCTION__, __FILE__);
1177  return mThreadManager.waitAndPush(tDatums);
1178  }
1179  catch (const std::exception& e)
1180  {
1181  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1182  return false;
1183  }
1184  }
1185 
1186  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1188  {
1189  try
1190  {
1191  if (!mUserOutputWs.empty())
1192  error("Pop cannot be called if an output worker was already selected.",
1193  __LINE__, __FUNCTION__, __FILE__);
1194  return mThreadManager.tryPop(tDatums);
1195  }
1196  catch (const std::exception& e)
1197  {
1198  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1199  return false;
1200  }
1201  }
1202 
1203  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1205  {
1206  try
1207  {
1208  if (!mUserOutputWs.empty())
1209  error("Pop cannot be called if an output worker was already selected.",
1210  __LINE__, __FUNCTION__, __FILE__);
1211  return mThreadManager.waitAndPop(tDatums);
1212  }
1213  catch (const std::exception& e)
1214  {
1215  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1216  return false;
1217  }
1218  }
1219 
1220  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1222  {
1223  try
1224  {
1225  mConfigured = false;
1226  mThreadManager.reset();
1227  mThreadId = 0ull;
1228  // Reset
1229  mUserInputWs.clear();
1230  wDatumProducer = nullptr;
1231  spWScaleAndSizeExtractor = nullptr;
1232  spWCvMatToOpInput = nullptr;
1233  spWCvMatToOpOutput = nullptr;
1234  spWPoseExtractors.clear();
1235  spWPoseTriangulations.clear();
1236  spWJointAngleEstimations.clear();
1237  mPostProcessingWs.clear();
1238  mUserPostProcessingWs.clear();
1239  mOutputWs.clear();
1240  spWGui = nullptr;
1241  mUserOutputWs.clear();
1242  }
1243  catch (const std::exception& e)
1244  {
1245  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1246  }
1247  }
1248 
1249  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1250  void Wrapper<TDatums, TDatumsSP, TWorker, TQueue>::configureThreadManager()
1251  {
1252  try
1253  {
1254  // The less number of queues -> the less threads opened, and potentially the less lag
1255 
1256  // Security checks
1257  if (!mConfigured)
1258  error("Configure the Wrapper class before calling `start()`.", __LINE__, __FUNCTION__, __FILE__);
1259  if ((wDatumProducer == nullptr) == (mUserInputWs.empty())
1260  && mThreadManagerMode != ThreadManagerMode::Asynchronous
1261  && mThreadManagerMode != ThreadManagerMode::AsynchronousIn)
1262  {
1263  const auto message = "You need to have 1 and only 1 producer selected. You can introduce your own"
1264  " producer by using setWorkerInput() or use the OpenPose default producer by"
1265  " configuring it in the configure function) or use the"
1266  " ThreadManagerMode::Asynchronous(In) mode.";
1267  error(message, __LINE__, __FUNCTION__, __FILE__);
1268  }
1269  if (mOutputWs.empty() && mUserOutputWs.empty() && spWGui == nullptr
1270  && mThreadManagerMode != ThreadManagerMode::Asynchronous
1271  && mThreadManagerMode != ThreadManagerMode::AsynchronousOut)
1272  {
1273  error("No output selected.", __LINE__, __FUNCTION__, __FILE__);
1274  }
1275 
1276  // Thread Manager
1277  // Clean previous thread manager (avoid configure to crash the program if used more than once)
1278  mThreadManager.reset();
1279  mThreadId = 0ull;
1280  auto queueIn = 0ull;
1281  auto queueOut = 1ull;
1282  // After producer
1283  // ID generator (before any multi-threading or any function that requires the ID)
1284  const auto wIdGenerator = std::make_shared<WIdGenerator<TDatumsSP>>();
1285  std::vector<TWorker> workersAux{wIdGenerator};
1286  // Scale & cv::Mat to OP format
1287  if (spWScaleAndSizeExtractor != nullptr)
1288  workersAux = mergeVectors(workersAux, {spWScaleAndSizeExtractor});
1289  if (spWCvMatToOpInput != nullptr)
1290  workersAux = mergeVectors(workersAux, {spWCvMatToOpInput});
1291  // cv::Mat to output format
1292  if (spWCvMatToOpOutput != nullptr)
1293  workersAux = mergeVectors(workersAux, {spWCvMatToOpOutput});
1294 
1295  // Producer
1296  // If custom user Worker and uses its own thread
1297  if (!mUserInputWs.empty() && mUserInputWsOnNewThread)
1298  {
1299  // Thread 0, queues 0 -> 1
1300  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1301  mThreadManager.add(mThreadId, mUserInputWs, queueIn++, queueOut++);
1302  threadIdPP();
1303  }
1304  // If custom user Worker in same thread
1305  else if (!mUserInputWs.empty())
1306  workersAux = mergeVectors(mUserInputWs, workersAux);
1307  // If OpenPose producer (same thread)
1308  else if (wDatumProducer != nullptr)
1309  workersAux = mergeVectors({wDatumProducer}, workersAux);
1310  // Otherwise
1311  else if (mThreadManagerMode != ThreadManagerMode::Asynchronous
1312  && mThreadManagerMode != ThreadManagerMode::AsynchronousIn)
1313  error("No input selected.", __LINE__, __FUNCTION__, __FILE__);
1314  // Thread 0 or 1, queues 0 -> 1
1315  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1316  mThreadManager.add(mThreadId, workersAux, queueIn++, queueOut++);
1317  // Increase thread
1318  threadIdPP();
1319 
1320  // Pose estimation & rendering
1321  // Thread 1 or 2...X, queues 1 -> 2, X = 2 + #GPUs
1322  if (!spWPoseExtractors.empty())
1323  {
1324  if (mMultiThreadEnabled)
1325  {
1326  for (auto& wPose : spWPoseExtractors)
1327  {
1328  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1329  mThreadManager.add(mThreadId, wPose, queueIn, queueOut);
1330  threadIdPP();
1331  }
1332  queueIn++;
1333  queueOut++;
1334  // Sort frames - Required own thread
1335  if (spWPoseExtractors.size() > 1u)
1336  {
1337  const auto wQueueOrderer = std::make_shared<WQueueOrderer<TDatumsSP>>();
1338  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1339  mThreadManager.add(mThreadId, wQueueOrderer, queueIn++, queueOut++);
1340  threadIdPP();
1341  }
1342  }
1343  else
1344  {
1345  if (spWPoseExtractors.size() > 1)
1346  log("Multi-threading disabled, only 1 thread running. All GPUs have been disabled but the"
1347  " first one, which is defined by gpuNumberStart (e.g. in the OpenPose demo, it is set"
1348  " with the `--num_gpu_start` flag).", Priority::High);
1349  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1350  mThreadManager.add(mThreadId, spWPoseExtractors.at(0), queueIn++, queueOut++);
1351  }
1352  }
1353  // Assemble all frames from same time instant (3-D module)
1354  const auto wQueueAssembler = std::make_shared<WQueueAssembler<TDatumsSP, TDatums>>();
1355  // 3-D reconstruction
1356  if (!spWPoseTriangulations.empty())
1357  {
1358  // Assemble frames
1359  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1360  mThreadManager.add(mThreadId, wQueueAssembler, queueIn++, queueOut++);
1361  threadIdPP();
1362  // 3-D reconstruction
1363  if (mMultiThreadEnabled)
1364  {
1365  for (auto& wPoseTriangulations : spWPoseTriangulations)
1366  {
1367  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1368  mThreadManager.add(mThreadId, wPoseTriangulations, queueIn, queueOut);
1369  threadIdPP();
1370  }
1371  queueIn++;
1372  queueOut++;
1373  // Sort frames
1374  if (spWPoseTriangulations.size() > 1u)
1375  {
1376  const auto wQueueOrderer = std::make_shared<WQueueOrderer<TDatumsSP>>();
1377  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1378  mThreadManager.add(mThreadId, wQueueOrderer, queueIn++, queueOut++);
1379  threadIdPP();
1380  }
1381  }
1382  else
1383  {
1384  if (spWPoseTriangulations.size() > 1)
1385  log("Multi-threading disabled, only 1 thread running for 3-D triangulation.",
1386  Priority::High);
1387  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1388  mThreadManager.add(mThreadId, spWPoseTriangulations.at(0), queueIn++, queueOut++);
1389  }
1390  }
1391  else
1392  mPostProcessingWs = mergeVectors({wQueueAssembler}, mPostProcessingWs);
1393  // Adam/IK step
1394  if (!spWJointAngleEstimations.empty())
1395  {
1396  if (mMultiThreadEnabled)
1397  {
1398  for (auto& wJointAngleEstimator : spWJointAngleEstimations)
1399  {
1400  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1401  mThreadManager.add(mThreadId, wJointAngleEstimator, queueIn, queueOut);
1402  threadIdPP();
1403  }
1404  queueIn++;
1405  queueOut++;
1406  // Sort frames
1407  if (spWJointAngleEstimations.size() > 1)
1408  {
1409  const auto wQueueOrderer = std::make_shared<WQueueOrderer<TDatumsSP>>();
1410  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1411  mThreadManager.add(mThreadId, wQueueOrderer, queueIn++, queueOut++);
1412  threadIdPP();
1413  }
1414  }
1415  else
1416  {
1417  if (spWJointAngleEstimations.size() > 1)
1418  log("Multi-threading disabled, only 1 thread running for joint angle estimation.",
1419  Priority::High);
1420  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1421  mThreadManager.add(mThreadId, spWJointAngleEstimations.at(0), queueIn++, queueOut++);
1422  }
1423  }
1424  // Post processing workers
1425  if (!mPostProcessingWs.empty())
1426  {
1427  // Combining mPostProcessingWs and mOutputWs
1428  mOutputWs = mergeVectors(mPostProcessingWs, mOutputWs);
1429  // // If I wanna split them
1430  // log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1431  // mThreadManager.add(mThreadId, mPostProcessingWs, queueIn++, queueOut++);
1432  // threadIdPP();
1433  }
1434  // If custom user Worker and uses its own thread
1435  if (!mUserPostProcessingWs.empty())
1436  {
1437  // If custom user Worker in its own thread
1438  if (mUserPostProcessingWsOnNewThread)
1439  {
1440  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1441  mThreadManager.add(mThreadId, mUserPostProcessingWs, queueIn++, queueOut++);
1442  threadIdPP();
1443  }
1444  // If custom user Worker in same thread
1445  // Merge with mOutputWs
1446  else
1447  mOutputWs = mergeVectors(mOutputWs, mUserPostProcessingWs);
1448  }
1449  // Output workers
1450  if (!mOutputWs.empty())
1451  {
1452  // Thread 4 or 5, queues 4 -> 5
1453  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1454  mThreadManager.add(mThreadId, mOutputWs, queueIn++, queueOut++);
1455  threadIdPP();
1456  }
1457  // User output worker
1458  // Thread Y, queues Q -> Q+1
1459  if (!mUserOutputWs.empty())
1460  {
1461  if (mUserOutputWsOnNewThread)
1462  {
1463  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1464  mThreadManager.add(mThreadId, mUserOutputWs, queueIn++, queueOut++);
1465  threadIdPP();
1466  }
1467  else
1468  {
1469  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1470  mThreadManager.add(mThreadId-1, mUserOutputWs, queueIn++, queueOut++);
1471  }
1472  }
1473  // OpenPose GUI
1474  if (spWGui != nullptr)
1475  {
1476  // Thread Y+1, queues Q+1 -> Q+2
1477  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1478  mThreadManager.add(mThreadId, spWGui, queueIn++, queueOut++);
1479  threadIdPP();
1480  }
1481  log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1482  }
1483  catch (const std::exception& e)
1484  {
1485  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1486  }
1487  }
1488 
1489  template<typename TDatums, typename TDatumsSP, typename TWorker, typename TQueue>
1490  unsigned long long Wrapper<TDatums, TDatumsSP, TWorker, TQueue>::threadIdPP()
1491  {
1492  try
1493  {
1494  if (mMultiThreadEnabled)
1495  mThreadId++;
1496  return mThreadId;
1497  }
1498  catch (const std::exception& e)
1499  {
1500  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1501  return 0ull;
1502  }
1503  }
1504 
1505  extern template class Wrapper<DATUM_BASE_NO_PTR>;
1506 }
1507 
1508 #endif // OPENPOSE_WRAPPER_WRAPPER_HPP
Definition: wHandDetectorTracking.hpp:11
Definition: wGuiInfoAdder.hpp:11
Wrapper(const ThreadManagerMode threadManagerMode=ThreadManagerMode::Synchronous)
Definition: wrapper.hpp:283
float alphaHeatMap
Definition: wrapperStructFace.hpp:45
bool guiVerbose
Definition: wrapperStructOutput.hpp:31
std::string writeHeatMapsFormat
Definition: wrapperStructOutput.hpp:106
Definition: wPoseSaver.hpp:12
Definition: wrapperStructPose.hpp:17
void start()
Definition: wrapper.hpp:1077
std::string writeKeypoint
Definition: wrapperStructOutput.hpp:44
float alphaKeypoint
Definition: wrapperStructFace.hpp:39
bool frameFlip
Definition: wrapperStructInput.hpp:41
Definition: wFaceSaver.hpp:12
float alphaKeypoint
Definition: wrapperStructHand.hpp:59
void exec()
Definition: wrapper.hpp:1062
int tracking
Definition: wrapperStructExtra.hpp:39
Definition: wPeopleJsonSaver.hpp:11
Definition: wHandRenderer.hpp:11
double writeVideoFps
Definition: wrapperStructOutput.hpp:111
Definition: wImageSaver.hpp:11
std::string udpPort
Definition: wrapperStructOutput.hpp:135
std::string writeImages
Definition: wrapperStructOutput.hpp:78
void stop()
Definition: wrapper.hpp:1092
T fastMax(const T a, const T b)
Definition: fastMath.hpp:70
float alphaHeatMap
Definition: wrapperStructHand.hpp:65
bool framesRepeat
Definition: wrapperStructInput.hpp:52
Definition: wKeypointScaler.hpp:11
int minViews3d
Definition: wrapperStructExtra.hpp:27
std::string writeHeatMaps
Definition: wrapperStructOutput.hpp:100
std::string writeCocoJson
Definition: wrapperStructOutput.hpp:67
Definition: wFaceDetector.hpp:11
void setWorkerPostProcessing(const TWorker &worker, const bool workerOnNewThread=true)
Definition: wrapper.hpp:348
bool waitAndPush(const TDatumsSP &tDatums)
Definition: wrapper.hpp:1170
Definition: wrapperStructFace.hpp:15
OP_API void error(const std::string &message, const int line=-1, const std::string &function="", const std::string &file="")
bool fullScreen
Definition: wrapperStructOutput.hpp:37
std::string udpHost
Definition: wrapperStructOutput.hpp:130
DataFormat writeKeypointFormat
Definition: wrapperStructOutput.hpp:51
Definition: wFaceExtractorNet.hpp:11
std::string writeCocoFootJson
Definition: wrapperStructOutput.hpp:72
Definition: wrapper.hpp:34
float scaleRange
Definition: wrapperStructHand.hpp:40
OP_API GpuMode getGpuMode()
RenderMode renderMode
Definition: wrapperStructFace.hpp:33
bool tracking
Definition: wrapperStructHand.hpp:47
bool enable
Definition: wrapperStructFace.hpp:20
Definition: wFaceDetectorOpenCV.hpp:11
Definition: wVideoSaver.hpp:11
Definition: wUdpSender.hpp:11
Definition: wOpOutputToCvMat.hpp:11
int ikThreads
Definition: wrapperStructExtra.hpp:46
bool realTimeProcessing
Definition: wrapperStructInput.hpp:36
bool waitAndEmplace(TDatumsSP &tDatums)
Definition: wrapper.hpp:1136
OP_API std::string formatAsDirectory(const std::string &directoryPathString)
bool tryPush(const TDatumsSP &tDatums)
Definition: wrapper.hpp:1153
Definition: wHandDetectorUpdate.hpp:11
void configure(const WrapperStructPose &wrapperStructPose, const WrapperStructInput &wrapperStructInput, const WrapperStructOutput &wrapperStructOutput=WrapperStructOutput{})
Definition: wrapper.hpp:384
Definition: wrapperStructInput.hpp:13
bool waitAndPop(TDatumsSP &tDatums)
Definition: wrapper.hpp:1204
bool tryPop(TDatumsSP &tDatums)
Definition: wrapper.hpp:1187
Definition: wHandExtractorNet.hpp:11
std::string writeBvh
Definition: wrapperStructOutput.hpp:125
Definition: wFaceRenderer.hpp:11
std::shared_ptr< Producer > producerSharedPtr
Definition: wrapperStructInput.hpp:19
bool tryEmplace(TDatumsSP &tDatums)
Definition: wrapper.hpp:1119
Definition: wPoseRenderer.hpp:11
std::string writeJson
Definition: wrapperStructOutput.hpp:61
OP_API void log(const std::string &message, const Priority priority=Priority::Max, const int line=-1, const std::string &function="", const std::string &file="")
Definition: wCocoJsonSaver.hpp:11
Definition: wrapperStructExtra.hpp:13
Definition: wrapperStructHand.hpp:15
float renderThreshold
Definition: wrapperStructHand.hpp:72
Definition: poseGpuRenderer.hpp:13
Definition: wHandDetector.hpp:11
std::string writeVideoAdam
Definition: wrapperStructOutput.hpp:118
std::vector< T > mergeVectors(const std::vector< T > &vectorA, const std::vector< T > &vectorB)
Definition: standard.hpp:40
Definition: wHandSaver.hpp:12
std::string writeVideo
Definition: wrapperStructOutput.hpp:93
unsigned long long frameLast
Definition: wrapperStructInput.hpp:31
bool identification
Definition: wrapperStructExtra.hpp:32
DisplayMode displayMode
Definition: wrapperStructOutput.hpp:25
Point< int > netInputSize
Definition: wrapperStructFace.hpp:27
void setSharedParametersAndIfLast(const std::tuple< std::shared_ptr< float * >, std::shared_ptr< bool >, std::shared_ptr< std::atomic< unsigned int >>, std::shared_ptr< std::atomic< unsigned long long >>, std::shared_ptr< const unsigned int >> &tuple, const bool isLast)
Definition: wHeatMapSaver.hpp:11
bool enable
Definition: wrapperStructHand.hpp:20
ThreadManagerMode
Definition: enumClasses.hpp:9
void setWorkerInput(const TWorker &worker, const bool workerOnNewThread=true)
Definition: wrapper.hpp:330
OP_API void wrapperConfigureSecurityChecks(WrapperStructPose &wrapperStructPose, const WrapperStructFace &wrapperStructFace, const WrapperStructHand &wrapperStructHand, const WrapperStructExtra &wrapperStructExtra, const WrapperStructInput &wrapperStructInput, const WrapperStructOutput &wrapperStructOutput, const bool renderOutput, const bool userOutputWsEmpty, const ThreadManagerMode threadManagerMode)
float renderThreshold
Definition: wrapperStructFace.hpp:52
int frameRotate
Definition: wrapperStructInput.hpp:47
std::string writeImagesFormat
Definition: wrapperStructOutput.hpp:86
unsigned long long frameFirst
Definition: wrapperStructInput.hpp:25
bool isRunning() const
Definition: wrapper.hpp:1105
Definition: wrapperStructOutput.hpp:14
bool reconstruct3d
Definition: wrapperStructExtra.hpp:21
void disableMultiThreading()
Definition: wrapper.hpp:317
OP_API int getGpuNumber()
Point< int > netInputSize
Definition: wrapperStructHand.hpp:27
~Wrapper()
Definition: wrapper.hpp:303
void setWorkerOutput(const TWorker &worker, const bool workerOnNewThread=true)
Definition: wrapper.hpp:366
int scalesNumber
Definition: wrapperStructHand.hpp:34
RenderMode renderMode
Definition: wrapperStructHand.hpp:53