From 7a9a3283bdfb6060a9fa4e6e2de871f8eb24fc36 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Fri, 9 Mar 2018 17:34:40 -0500 Subject: [PATCH] Added flag 3d_min_views --- doc/demo_overview.md | 5 +- doc/quick_start.md | 9 +- doc/release_notes.md | 3 +- examples/openpose/openpose.cpp | 5 +- .../1_custom_post_processing.cpp | 4 +- .../1_user_asynchronous_output.cpp | 4 +- .../tutorial_wrapper/2_user_synchronous.cpp | 4 +- .../tutorial_wrapper/3_user_asynchronous.cpp | 4 +- include/openpose/3d/poseTriangulation.hpp | 13 ++- include/openpose/3d/wPoseTriangulation.hpp | 17 ++-- include/openpose/gui/frameDisplayer.hpp | 5 + include/openpose/wrapper/wrapper.hpp | 3 +- include/openpose/wrapper/wrapperAuxiliary.hpp | 14 +-- .../openpose/wrapper/wrapperStructPose.hpp | 8 +- src/openpose/3d/poseTriangulation.cpp | 95 ++++++++++++------- src/openpose/gui/frameDisplayer.cpp | 32 +++++++ src/openpose/gui/gui.cpp | 43 +++------ src/openpose/utilities/keypoint.cpp | 4 +- src/openpose/wrapper/wrapperAuxiliary.cpp | 8 ++ src/openpose/wrapper/wrapperStructPose.cpp | 3 +- 20 files changed, 188 insertions(+), 95 deletions(-) diff --git a/doc/demo_overview.md b/doc/demo_overview.md index 8334052a..479bd4b1 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -184,8 +184,9 @@ Each flag is divided into flag name, default value, and description. - DEFINE_bool(hand_tracking, false, "Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it simply looks for hands in positions at which hands were located in previous frames, but it does not guarantee the same person ID among frames."); 8. OpenPose 3-D Reconstruction -DEFINE_bool(3d, false, "Running OpenPose 3-D reconstruction demo: 1) Reading from a stereo camera system. 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction results. Note that it will only display 1 person. If multiple people is present, it will fail."); -DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per iteration, allowing tasks such as stereo camera processing (`--3d`). Note that `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the parameter folder as this number indicates."); +- DEFINE_bool(3d, false, "Running OpenPose 3-D reconstruction demo: 1) Reading from a stereo camera system. 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction results. Note that it will only display 1 person. If multiple people is present, it will fail."); +- DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will require all the cameras to see the keypoint in order to reconstruct it."); +- DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per iteration, allowing tasks such as stereo camera processing (`--3d`). Note that `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the parameter folder as this number indicates."); 9. OpenPose Rendering - DEFINE_int32(part_to_show, 0, "Prediction channel to visualize (default: 0). 0 for all the body parts, 1-18 for each body part heat map, 19 for the background heat map, 20 for all the body part heat maps together, 21 for all the PAFs, 22-40 for each body part pair PAF."); diff --git a/doc/quick_start.md b/doc/quick_start.md index 1ddbcc9b..be647ec3 100644 --- a/doc/quick_start.md +++ b/doc/quick_start.md @@ -147,13 +147,20 @@ build\x64\Release\OpenPoseDemo.exe --flir_camera --3d --number_people_max 1 --fa # Ubuntu (same flags for Windows version) # Optionally add `--face` and/or `--hand` to include face and/or hands # Assuming 3 cameras -# Note: We highly recommend to reduce `--output_resolution`. E.g. for 3 cameras recording at 1920x1080, the resulting image is (3x1920)x1080, so we recommend e.g. 1920x360 (x3 reduction). +# Note: We highly recommend to reduce `--output_resolution`. E.g. for 3 cameras recording at 1920x1080, the resulting image is (3x1920)x1080, so we recommend e.g. 640x360 (x3 reduction). # Video ./build/examples/openpose/openpose.bin --video output_folder_path/video.avi --3d_views 3 --3d --number_people_max 1 --output_resolution {desired_output_resolution} # Images ./build/examples/openpose/openpose.bin --image_dir output_folder_path/ --3d_views 3 --3d --number_people_max 1 --output_resolution {desired_output_resolution} ``` +5. Reconstruction when at least n visible views +``` +# Ubuntu (same flags for Windows version) +# Assuming >=2 cameras and reconstruction when at least 2 visible views +./build/examples/openpose/openpose.bin --flir_camera --3d --number_people_max 1 --3d_min_views 2 --output_resolution {desired_output_resolution} +``` + ## Expected Visual Results diff --git a/doc/release_notes.md b/doc/release_notes.md index 720a1ae5..cadc2261 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -193,7 +193,8 @@ OpenPose Library - Release Notes 21. VideoSaver (`--write_video`) compatible with multi-camera setting. It will save all the different views concatenated. 22. OpenPose small GUI rescale the verbose text to the displayed image, to avoid the text to be either too big or small. 23. OpenPose small GUI shows the frame number w.r.t. the original producer, rather than the frame id. E.g., if video is started at frame 30, OpenPose will display 30 rather than 0 in the first frame. - 23. OpenPose GUI: 'l' and 'k' functionality swapped. + 24. OpenPose GUI: 'l' and 'k' functionality swapped. + 25. 3-D reconstruction module: Added flag `--3d_min_views` to select minimum number of cameras required for 3-D reconstruction. 2. Functions or parameters renamed: 1. Flag `no_display` renamed as `display`, able to select between `NoDisplay`, `Display2D`, `Display3D`, and `DisplayAll`. 2. 3-D reconstruction demo is now inside the OpenPose demo binary. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index f1516d71..fcafbcec 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -147,6 +147,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -276,7 +278,8 @@ int openPoseDemo() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d, FLAGS_identification}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views, + FLAGS_identification}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index 632f0d96..74df6c96 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -156,6 +156,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -278,7 +280,7 @@ int openPoseTutorialWrapper4() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_wrapper/1_user_asynchronous_output.cpp b/examples/tutorial_wrapper/1_user_asynchronous_output.cpp index 555365de..caa8bcb9 100644 --- a/examples/tutorial_wrapper/1_user_asynchronous_output.cpp +++ b/examples/tutorial_wrapper/1_user_asynchronous_output.cpp @@ -146,6 +146,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -353,7 +355,7 @@ int openPoseTutorialWrapper1() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_wrapper/2_user_synchronous.cpp b/examples/tutorial_wrapper/2_user_synchronous.cpp index 8293e76b..a23e9c00 100644 --- a/examples/tutorial_wrapper/2_user_synchronous.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous.cpp @@ -125,6 +125,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -443,7 +445,7 @@ int openPoseTutorialWrapper2() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_wrapper/3_user_asynchronous.cpp b/examples/tutorial_wrapper/3_user_asynchronous.cpp index 53f8971f..7705bff3 100644 --- a/examples/tutorial_wrapper/3_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/3_user_asynchronous.cpp @@ -125,6 +125,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -384,7 +386,7 @@ int openPoseTutorialWrapper3() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/include/openpose/3d/poseTriangulation.hpp b/include/openpose/3d/poseTriangulation.hpp index c3210f74..5a997e58 100644 --- a/include/openpose/3d/poseTriangulation.hpp +++ b/include/openpose/3d/poseTriangulation.hpp @@ -6,8 +6,17 @@ namespace op { - OP_API Array reconstructArray(const std::vector>& keypointsVector, - const std::vector& matrixEachCamera); + class OP_API PoseTriangulation + { + public: + PoseTriangulation(const int minViews3d); + + Array reconstructArray(const std::vector>& keypointsVector, + const std::vector& cameraMatrices) const; + + private: + const int mMinViews3d; + }; } #endif // OPENPOSE_3D_POSE_TRIANGULATION_HPP diff --git a/include/openpose/3d/wPoseTriangulation.hpp b/include/openpose/3d/wPoseTriangulation.hpp index 08a316cc..db949787 100644 --- a/include/openpose/3d/wPoseTriangulation.hpp +++ b/include/openpose/3d/wPoseTriangulation.hpp @@ -11,13 +11,15 @@ namespace op class WPoseTriangulation : public Worker { public: - explicit WPoseTriangulation(); + explicit WPoseTriangulation(const std::shared_ptr& poseTriangulation); void initializationOnThread(); void work(TDatums& tDatums); private: + const std::shared_ptr spPoseTriangulation; + DELETE_COPY(WPoseTriangulation); }; } @@ -31,7 +33,8 @@ namespace op namespace op { template - WPoseTriangulation::WPoseTriangulation() + WPoseTriangulation::WPoseTriangulation(const std::shared_ptr& poseTriangulation) : + spPoseTriangulation{poseTriangulation} { } @@ -66,10 +69,12 @@ namespace op cameraMatrices.emplace_back(datumsElement.cameraMatrix); } // Pose 3-D reconstruction - auto poseKeypoints3D = reconstructArray(poseKeypointVector, cameraMatrices); - auto faceKeypoints3D = reconstructArray(faceKeypointVector, cameraMatrices); - auto leftHandKeypoints3D = reconstructArray(leftHandKeypointVector, cameraMatrices); - auto rightHandKeypoints3D = reconstructArray(rightHandKeypointVector, cameraMatrices); + auto poseKeypoints3D = spPoseTriangulation->reconstructArray(poseKeypointVector, cameraMatrices); + auto faceKeypoints3D = spPoseTriangulation->reconstructArray(faceKeypointVector, cameraMatrices); + auto leftHandKeypoints3D = spPoseTriangulation->reconstructArray(leftHandKeypointVector, + cameraMatrices); + auto rightHandKeypoints3D = spPoseTriangulation->reconstructArray(rightHandKeypointVector, + cameraMatrices); // Assign to all tDatums for (auto& datumsElement : *tDatums) { diff --git a/include/openpose/gui/frameDisplayer.hpp b/include/openpose/gui/frameDisplayer.hpp index 21420292..5e4ddd1a 100644 --- a/include/openpose/gui/frameDisplayer.hpp +++ b/include/openpose/gui/frameDisplayer.hpp @@ -49,6 +49,11 @@ namespace op */ void displayFrame(const cv::Mat& frame, const int waitKeyValue = -1); + /** + * Analogous to the previous displayFrame, but first it horizontally concatenates all the frames + */ + void displayFrame(const std::vector& frames, const int waitKeyValue = -1); + private: const std::string mWindowName; Point mWindowedSize; diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index 25794669..3e85a568 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -798,8 +798,9 @@ namespace op // 3-D reconstruction if (wrapperStructPose.reconstruct3d) { + const auto poseTriangulation = std::make_shared(wrapperStructPose.minViews3d); mPostProcessingWs.emplace_back( - std::make_shared>() + std::make_shared>(poseTriangulation) ); } // Frames processor (OpenPose format -> cv::Mat format) diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index 52cb768f..43566b23 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -21,13 +21,13 @@ namespace op * @param threadManagerMode */ OP_API void wrapperConfigureSecurityChecks(const WrapperStructPose& wrapperStructPose, - const WrapperStructFace& wrapperStructFace, - const WrapperStructHand& wrapperStructHand, - const WrapperStructInput& wrapperStructInput, - const WrapperStructOutput& wrapperStructOutput, - const bool renderOutput, - const bool userOutputWsEmpty, - const ThreadManagerMode threadManagerMode); + const WrapperStructFace& wrapperStructFace, + const WrapperStructHand& wrapperStructHand, + const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput, + const bool renderOutput, + const bool userOutputWsEmpty, + const ThreadManagerMode threadManagerMode); } #endif // OPENPOSE_WRAPPER_WRAPPER_AUXILIARY_HPP diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index 5946d0f1..f2e9e761 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -175,6 +175,12 @@ namespace op */ bool reconstruct3d; + /** + * Minimum number of views required to reconstruct each keypoint. + * By default (-1), it will require all the cameras to see the keypoint in order to reconstruct it. + */ + int minViews3d; + /** * Whether to return a person ID for each body skeleton, providing temporal consistency. */ @@ -198,7 +204,7 @@ namespace op const ScaleMode heatMapScale = ScaleMode::ZeroToOne, const bool addPartCandidates = false, const float renderThreshold = 0.05f, const int numberPeopleMax = -1, const bool enableGoogleLogging = true, const bool reconstruct3d = false, - const bool identification = false); + const int minViews3d = -1, const bool identification = false); }; } diff --git a/src/openpose/3d/poseTriangulation.cpp b/src/openpose/3d/poseTriangulation.cpp index 3ea26f04..41621198 100644 --- a/src/openpose/3d/poseTriangulation.cpp +++ b/src/openpose/3d/poseTriangulation.cpp @@ -3,7 +3,7 @@ namespace op { - double calcReprojectionError(const cv::Mat& X, const std::vector& M, + double calcReprojectionError(const cv::Mat& reconstructedPoint, const std::vector& M, const std::vector& points2d) { try @@ -11,7 +11,7 @@ namespace op auto averageError = 0.; for (auto i = 0u ; i < M.size() ; i++) { - cv::Mat imageX = M[i] * X; + cv::Mat imageX = M[i] * reconstructedPoint; imageX /= imageX.at(2,0); const auto error = std::sqrt(std::pow(imageX.at(0,0) - points2d[i].x,2) + std::pow(imageX.at(1,0) - points2d[i].y,2)); @@ -27,30 +27,34 @@ namespace op } } - void triangulate(cv::Mat& X, const std::vector& matrixEachCamera, + void triangulate(cv::Mat& reconstructedPoint, const std::vector& cameraMatrices, const std::vector& pointsOnEachCamera) { try { // Security checks - if (matrixEachCamera.empty() || matrixEachCamera.size() != pointsOnEachCamera.size()) - error("numberCameras.empty() || numberCameras.size() != pointsOnEachCamera.size()", - __LINE__, __FUNCTION__, __FILE__); + if (cameraMatrices.size() != pointsOnEachCamera.size()) + error("numberCameras.size() != pointsOnEachCamera.size() (" + std::to_string(cameraMatrices.size()) + + " vs. " + std::to_string(pointsOnEachCamera.size()) + ").", + __LINE__, __FUNCTION__, __FILE__); + if (cameraMatrices.empty()) + error("numberCameras.empty()", + __LINE__, __FUNCTION__, __FILE__); // Create and fill A - const auto numberCameras = (int)matrixEachCamera.size(); + const auto numberCameras = (int)cameraMatrices.size(); cv::Mat A = cv::Mat::zeros(numberCameras*2, 4, CV_64F); for (auto i = 0 ; i < numberCameras ; i++) { - cv::Mat temp = pointsOnEachCamera[i].x*matrixEachCamera[i].rowRange(2,3) - - matrixEachCamera[i].rowRange(0,1); + cv::Mat temp = pointsOnEachCamera[i].x*cameraMatrices[i].rowRange(2,3) + - cameraMatrices[i].rowRange(0,1); temp.copyTo(A.rowRange(i*2, i*2+1)); - temp = pointsOnEachCamera[i].y*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(1,2); + temp = pointsOnEachCamera[i].y*cameraMatrices[i].rowRange(2,3) - cameraMatrices[i].rowRange(1,2); temp.copyTo(A.rowRange(i*2+1, i*2+2)); } // SVD on A cv::SVD svd{A}; - svd.solveZ(A,X); - X /= X.at(3); + svd.solveZ(A,reconstructedPoint); + reconstructedPoint /= reconstructedPoint.at(3); } catch (const std::exception& e) { @@ -59,20 +63,20 @@ namespace op } // TODO: ask for the missing function: TriangulationOptimization - double triangulateWithOptimization(cv::Mat& X, const std::vector& matrixEachCamera, + double triangulateWithOptimization(cv::Mat& reconstructedPoint, const std::vector& cameraMatrices, const std::vector& pointsOnEachCamera) { try { - triangulate(X, matrixEachCamera, pointsOnEachCamera); + triangulate(reconstructedPoint, cameraMatrices, pointsOnEachCamera); return 0.; - // return calcReprojectionError(X, matrixEachCamera, pointsOnEachCamera); + // return calcReprojectionError(X, cameraMatrices, pointsOnEachCamera); - // //if (matrixEachCamera.size() >= 3) - // //double beforeError = calcReprojectionError(&matrixEachCamera, pointsOnEachCamera, X); - // double change = TriangulationOptimization(&matrixEachCamera, pointsOnEachCamera, X); - // //double afterError = calcReprojectionError(&matrixEachCamera,pointsOnEachCamera,X); + // //if (cameraMatrices.size() >= 3) + // //double beforeError = calcReprojectionError(&cameraMatrices, pointsOnEachCamera, X); + // double change = TriangulationOptimization(&cameraMatrices, pointsOnEachCamera, X); + // //double afterError = calcReprojectionError(&cameraMatrices,pointsOnEachCamera,X); // //printfLog("!!Mine %.8f , inFunc %.8f \n",beforeError-afterError,change); // return change; } @@ -83,14 +87,30 @@ namespace op } } - Array reconstructArray(const std::vector>& keypointsVector, - const std::vector& matrixEachCamera) + PoseTriangulation::PoseTriangulation(const int minViews3d) : + mMinViews3d{minViews3d} + { + try + { + // Security checks + if (0 <= mMinViews3d && mMinViews3d < 2) + error("Minimum number of views must be at least 2 (e.g., `--3d_min_views 2`) or negative.", + __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + Array PoseTriangulation::reconstructArray(const std::vector>& keypointsVector, + const std::vector& cameraMatrices) const { try { Array keypoints3D; // Security checks - if (matrixEachCamera.size() < 2) + if (cameraMatrices.size() < 2) error("Only 1 camera detected. The 3-D reconstruction module can only be used with > 1 cameras" " simultaneously. E.g., using FLIR stereo cameras (`--flir_camera`).", __LINE__, __FUNCTION__, __FILE__); @@ -113,27 +133,33 @@ namespace op const auto threshold = 0.2f; std::vector indexesUsed; std::vector> xyPoints; + std::vector> cameraMatricesPerPoint; for (auto part = 0; part < numberBodyParts; part++) { // Create vector of points - auto missedPoint = false; + // auto missedPoint = false; std::vector xyPointsElement; + std::vector cameraMatricesElement; const auto baseIndex = part * lastChannelLength; - for (auto& keypoints : keypointsVector) + // for (auto& keypoints : keypointsVector) + for (auto i = 0u ; i < keypointsVector.size() ; i++) { + const auto& keypoints = keypointsVector[i]; if (keypoints[baseIndex+2] > threshold) - xyPointsElement.emplace_back(cv::Point2d{ keypoints[baseIndex], - keypoints[baseIndex+1]}); - else { - missedPoint = true; - break; + xyPointsElement.emplace_back(cv::Point2d{keypoints[baseIndex], + keypoints[baseIndex+1]}); + cameraMatricesElement.emplace_back(cameraMatrices[i]); } } - if (!missedPoint) + // If visible from all views (mMinViews3d < 0) + // or if visible for at least mMinViews3d views + if ((mMinViews3d < 0 && cameraMatricesElement.size() == cameraMatrices.size()) + || (mMinViews3d > 1 && mMinViews3d <= (int)xyPointsElement.size())) { indexesUsed.emplace_back(part); xyPoints.emplace_back(xyPointsElement); + cameraMatricesPerPoint.emplace_back(cameraMatricesElement); } } // 3D reconstruction @@ -144,16 +170,17 @@ namespace op std::vector xyzPoints(xyPoints.size()); for (auto i = 0u; i < xyPoints.size(); i++) { - cv::Mat X; - triangulateWithOptimization(X, matrixEachCamera, xyPoints[i]); - xyzPoints[i] = cv::Point3d{ X.at(0), X.at(1), X.at(2) }; + cv::Mat reconstructedPoint; + triangulateWithOptimization(reconstructedPoint, cameraMatricesPerPoint[i], xyPoints[i]); + xyzPoints[i] = cv::Point3d{reconstructedPoint.at(0), reconstructedPoint.at(1), + reconstructedPoint.at(2)}; } // 3D points to pose // OpenCV alternative: // http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#triangulatepoints // cv::Mat reconstructedPoints{4, firstcv::Points.size(), CV_64F}; - // cv::triangulatecv::Points(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, + // cv::triangulatePoints(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, // reconstructedcv::Points); const auto lastChannelLength = keypoints3D.getSize(2); for (auto index = 0u; index < indexesUsed.size(); index++) diff --git a/src/openpose/gui/frameDisplayer.cpp b/src/openpose/gui/frameDisplayer.cpp index 9ca87f47..11f3b2ae 100644 --- a/src/openpose/gui/frameDisplayer.cpp +++ b/src/openpose/gui/frameDisplayer.cpp @@ -87,6 +87,9 @@ namespace op { try { + // Security check + if (frame.empty()) + error("Empty frame introduced.", __LINE__, __FUNCTION__, __FILE__); // If frame > window size --> Resize window if (mWindowedSize.x < frame.cols || mWindowedSize.y < frame.rows) { @@ -106,4 +109,33 @@ namespace op error(e.what(), __LINE__, __FUNCTION__, __FILE__); } } + + void FrameDisplayer::displayFrame(const std::vector& frames, const int waitKeyValue) + { + try + { + // No frames + if (frames.empty()) + displayFrame(cv::Mat(), waitKeyValue); + // 1 frame + else if (frames.size() == 1u) + displayFrame(frames[0], waitKeyValue); + // >= 2 frames + else + { + // Prepare final cvMat + // Concat (0) + cv::Mat cvMat = frames[0].clone(); + // Concat (1,size()-1) + for (auto i = 1u; i < frames.size(); i++) + cv::hconcat(cvMat, frames[i], cvMat); + // Display it + displayFrame(cvMat, waitKeyValue); + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } } diff --git a/src/openpose/gui/gui.cpp b/src/openpose/gui/gui.cpp index 40e4fc7b..7d43335f 100644 --- a/src/openpose/gui/gui.cpp +++ b/src/openpose/gui/gui.cpp @@ -177,12 +177,7 @@ namespace op { try { - // Check tDatum integrity - const bool returnedIsValidFrame = ((spIsRunning == nullptr || *spIsRunning) && !cvMatOutput.empty()); - - // Display - if (returnedIsValidFrame) - mFrameDisplayer.displayFrame(cvMatOutput, -1); + setImage(std::vector{cvMatOutput}); } catch (const std::exception& e) { @@ -194,36 +189,20 @@ namespace op { try { - // 0 image - if (cvMatOutputs.empty()) - setImage(cvMatOutputs[0]); - // 1 image - else if (cvMatOutputs.size() == 1) - setImage(cvMatOutputs[0]); - // > 1 image - else + // Check tDatum integrity + bool returnedIsValidFrame = ((spIsRunning == nullptr || *spIsRunning) && !cvMatOutputs.empty()); + for (const auto& cvMatOutput : cvMatOutputs) { - // Check tDatum integrity - bool returnedIsValidFrame = ((spIsRunning == nullptr || *spIsRunning) && !cvMatOutputs.empty()); - if (returnedIsValidFrame) + if (cvMatOutput.empty()) { - // Security checks - for (const auto& cvMatOutput : cvMatOutputs) - if (cvMatOutput.empty()) - returnedIsValidFrame = false; - // Prepare final cvMat - if (returnedIsValidFrame) - { - // Concat (0) - cv::Mat cvMat = cvMatOutputs[0].clone(); - // Concat (1,size()-1) - for (auto i = 1u; i < cvMatOutputs.size(); i++) - cv::hconcat(cvMat, cvMatOutputs[i], cvMat); - // Display - mFrameDisplayer.displayFrame(cvMat, -1); - } + returnedIsValidFrame = false; + break; } } + + // Display + if (returnedIsValidFrame) + mFrameDisplayer.displayFrame(cvMatOutputs, -1); } catch (const std::exception& e) { diff --git a/src/openpose/utilities/keypoint.cpp b/src/openpose/utilities/keypoint.cpp index d43c92e7..b9217d49 100644 --- a/src/openpose/utilities/keypoint.cpp +++ b/src/openpose/utilities/keypoint.cpp @@ -175,8 +175,8 @@ namespace op const auto thicknessRatio = fastMax(intRound(std::sqrt(area) * thicknessCircleRatio * ratioAreas), 2); // Negative thickness in cv::circle means that a filled circle is to be drawn. - const auto thicknessCircle = (ratioAreas > 0.05f ? thicknessRatio : -1); - const auto thicknessLine = intRound(thicknessRatio * thicknessLineRatioWRTCircle); + const auto thicknessCircle = fastMax(1, (ratioAreas > 0.05f ? thicknessRatio : -1)); + const auto thicknessLine = fastMax(1, intRound(thicknessRatio * thicknessLineRatioWRTCircle)); const auto radius = thicknessRatio / 2; // Draw lines diff --git a/src/openpose/wrapper/wrapperAuxiliary.cpp b/src/openpose/wrapper/wrapperAuxiliary.cpp index c2d8e3db..dc403b38 100644 --- a/src/openpose/wrapper/wrapperAuxiliary.cpp +++ b/src/openpose/wrapper/wrapperAuxiliary.cpp @@ -115,6 +115,14 @@ namespace op if (wrapperStructPose.gpuNumber > 0) error("GPU number must be negative or 0 if CPU_ONLY is enabled.", __LINE__, __FUNCTION__, __FILE__); + // If num_gpu 0 --> output_resolution has no effect + if (wrapperStructPose.gpuNumber == 0 && + (wrapperStructPose.outputSize.x > 0 || wrapperStructPose.outputSize.y > 0)) + error("If `--num_gpu 0`, then `--output_resolution` has no effect, so either disable it or use" + " `--output_resolution -1x-1`. Current output size: (" + + std::to_string(wrapperStructPose.outputSize.x) + "x" + + std::to_string(wrapperStructPose.outputSize.y) + ").", + __LINE__, __FUNCTION__, __FILE__); // Net input resolution cannot be reshaped for Caffe OpenCL and MKL versions, only for CUDA version #if defined USE_MKL || defined CPU_ONLY // If image_dir and netInputSize == -1 --> error diff --git a/src/openpose/wrapper/wrapperStructPose.cpp b/src/openpose/wrapper/wrapperStructPose.cpp index 83a017b0..af98a046 100644 --- a/src/openpose/wrapper/wrapperStructPose.cpp +++ b/src/openpose/wrapper/wrapperStructPose.cpp @@ -14,7 +14,7 @@ namespace op const ScaleMode heatMapScale_, const bool addPartCandidates_, const float renderThreshold_, const int numberPeopleMax_, const bool enableGoogleLogging_, const bool reconstruct3d_, - const bool identification_) : + const int minViews3d_, const bool identification_) : enable{enable_}, netInputSize{netInputSize_}, outputSize{outputSize_}, @@ -37,6 +37,7 @@ namespace op numberPeopleMax{numberPeopleMax_}, enableGoogleLogging{enableGoogleLogging_}, reconstruct3d{reconstruct3d_}, + minViews3d{minViews3d_}, identification{identification_} { } -- GitLab