Utilities keypoint as template

CMakeLists.txt

@@ -252,7 +252,10 @@ option(BUILD_DOCS "Build OpenPose documentation." OFF)
 option(BUILD_PYTHON "Build OpenPose python." OFF)
 if (WIN32)
   option(BUILD_BIN_FOLDER "Copy all required 3rd-party DLL files into {build_directory}/bin. Disable to save some memory." ON)
-endif ()
+endif (WIN32)
+
+# Unity
+# option(BUILD_UNITY_SUPPORT "Build OpenPose as a Unity plugin." OFF)
 
 # Build as shared library
 option(BUILD_SHARED_LIBS "Build as shared lib." ON)
@@ -310,6 +313,12 @@ if (WITH_TRACKING)
   add_definitions(-DUSE_TRACKING)
 endif (WITH_TRACKING)
 
+# Unity
+if (BUILD_UNITY_SUPPORT)
+  # OpenPose flags
+  add_definitions(-DUSE_UNITY_SUPPORT)
+endif (BUILD_UNITY_SUPPORT)
+
 # Calibration
 # No Eigen
 if (${WITH_EIGEN} MATCHES "NONE")
@@ -547,8 +556,8 @@ if (WIN32)
   if (BUILD_PYTHON)
     if (NOT BUILD_BIN_FOLDER)
       message(FATAL_ERROR "BUILD_BIN_FOLDER must be turned on to as well to build python library")
-    endif ()
-  endif ()
+    endif (NOT BUILD_BIN_FOLDER)
+  endif (BUILD_PYTHON)
 
   # Auto copy DLLs
   if (BUILD_BIN_FOLDER)
@@ -876,8 +885,8 @@ message(STATUS "Models Downloaded.")
 if (Caffe_FOUND)
   if(BUILD_PYTHON)
     add_subdirectory(python)
-  endif ()
-endif()
+  endif (BUILD_PYTHON)
+endif(Caffe_FOUND)
 
 ### GENERATE DOCUMENTATION
 if (UNIX OR APPLE)

README.md

@@ -50,20 +50,6 @@ For further details, check [all released features](doc/released_features.md) and
 
 
 
-## Contents
-1. [Latest Features](#latest-features)
-2. [Results](#results)
-3. [Installation, Reinstallation and Uninstallation](#installation-reinstallation-and-uninstallation)
-4. [Quick Start](#quick-start)
-5. [Output](#output)
-6. [Speeding Up OpenPose and Benchmark](#speeding-up-openpose-and-benchmark)
-7. [Send Us Failure Cases and Feedback!](#send-us-failure-cases-and-feedback)
-8. [Authors and Contributors](#authors-and-contributors)
-9. [Citation](#citation)
-10. [License](#license)
-
-
-
 ## Results
 ### Body-Foot Estimation
 <p align="center">
@@ -92,6 +78,21 @@ For further details, check [all released features](doc/released_features.md) and
 
 
 
+## Contents
+1. [Features](#features)
+2. [Latest Features](#latest-features)
+3. [Results](#results)
+4. [Installation, Reinstallation and Uninstallation](#installation-reinstallation-and-uninstallation)
+5. [Quick Start](#quick-start)
+6. [Output](#output)
+7. [Speeding Up OpenPose and Benchmark](#speeding-up-openpose-and-benchmark)
+8. [Send Us Failure Cases and Feedback!](#send-us-failure-cases-and-feedback)
+9. [Authors and Contributors](#authors-and-contributors)
+10. [Citation](#citation)
+11. [License](#license)
+
+
+
 ## Installation, Reinstallation and Uninstallation
 **Windows portable version**: Simply download and use the latest version from the [Releases](https://github.com/CMU-Perceptual-Computing-Lab/openpose/releases) section.
 

doc/release_notes.md

@@ -263,6 +263,7 @@ OpenPose Library - Release Notes
 1. Main improvements:
     1. Added initial single-person tracker for further speed up or visual smoothing (`--tracking` flag).
     2. Greedy body part connector implemented in CUDA: +~33% speed up in Nvidia (CUDA) version with default flags and +~10% in maximum accuracy configuration.
+    3. OpenPose can be built as Unity plugin: Added flag `BUILD_UNITY_SUPPORT` and special Unity code.
 2. Functions or parameters renamed:
     1. By default, python example `2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368.
 3. Main bugs fixed:

include/openpose/utilities/keypoint.hpp

@@ -5,30 +5,40 @@
 
 namespace op
 {
-    OP_API float getDistance(const Array<float>& keypoints, const int person, const int elementA, const int elementB);
+    template <typename T>
+    OP_API T getDistance(const Array<T>& keypoints, const int person, const int elementA, const int elementB);
 
-    OP_API void averageKeypoints(Array<float>& keypointsA, const Array<float>& keypointsB, const int personA);
+    template <typename T>
+    OP_API void averageKeypoints(Array<T>& keypointsA, const Array<T>& keypointsB, const int personA);
 
-    OP_API void scaleKeypoints(Array<float>& keypoints, const float scale);
+    template <typename T>
+    OP_API void scaleKeypoints(Array<T>& keypoints, const T scale);
 
-    OP_API void scaleKeypoints2d(Array<float>& keypoints, const float scaleX, const float scaleY);
+    template <typename T>
+    OP_API void scaleKeypoints2d(Array<T>& keypoints, const T scaleX, const T scaleY);
 
-    OP_API void scaleKeypoints2d(Array<float>& keypoints, const float scaleX, const float scaleY, const float offsetX,
-                                 const float offsetY);
+    template <typename T>
+    OP_API void scaleKeypoints2d(Array<T>& keypoints, const T scaleX, const T scaleY, const T offsetX,
+                                 const T offsetY);
 
-    OP_API void renderKeypointsCpu(Array<float>& frameArray, const Array<float>& keypoints,
-                                   const std::vector<unsigned int>& pairs, const std::vector<float> colors,
-                                   const float thicknessCircleRatio, const float thicknessLineRatioWRTCircle,
-                                   const std::vector<float>& poseScales, const float threshold);
+    template <typename T>
+    OP_API void renderKeypointsCpu(Array<T>& frameArray, const Array<T>& keypoints,
+                                   const std::vector<unsigned int>& pairs, const std::vector<T> colors,
+                                   const T thicknessCircleRatio, const T thicknessLineRatioWRTCircle,
+                                   const std::vector<T>& poseScales, const T threshold);
 
-    OP_API Rectangle<float> getKeypointsRectangle(const Array<float>& keypoints, const int person,
-                                                  const float threshold);
+    template <typename T>
+    OP_API Rectangle<T> getKeypointsRectangle(const Array<T>& keypoints, const int person,
+                                              const T threshold);
 
-    OP_API float getAverageScore(const Array<float>& keypoints, const int person);
+    template <typename T>
+    OP_API T getAverageScore(const Array<T>& keypoints, const int person);
 
-    OP_API float getKeypointsArea(const Array<float>& keypoints, const int person, const float threshold);
+    template <typename T>
+    OP_API T getKeypointsArea(const Array<T>& keypoints, const int person, const T threshold);
 
-    OP_API int getBiggestPerson(const Array<float>& keypoints, const float threshold);
+    template <typename T>
+    OP_API int getBiggestPerson(const Array<T>& keypoints, const T threshold);
 }
 
 #endif // OPENPOSE_UTILITIES_KEYPOINT_HPP

src/openpose/gui/gui3D.cpp

@@ -498,10 +498,10 @@ namespace op
                         gKeypoints3D.rightHandKeypoints = rightHandKeypoints3D;
                         gKeypoints3D.validKeypoints = true;
                         // From m to mm
-                        scaleKeypoints(gKeypoints3D.poseKeypoints, 1e3);
-                        scaleKeypoints(gKeypoints3D.faceKeypoints, 1e3);
-                        scaleKeypoints(gKeypoints3D.leftHandKeypoints, 1e3);
-                        scaleKeypoints(gKeypoints3D.rightHandKeypoints, 1e3);
+                        scaleKeypoints(gKeypoints3D.poseKeypoints, 1e3f);
+                        scaleKeypoints(gKeypoints3D.faceKeypoints, 1e3f);
+                        scaleKeypoints(gKeypoints3D.leftHandKeypoints, 1e3f);
+                        scaleKeypoints(gKeypoints3D.rightHandKeypoints, 1e3f);
                         // Unlock mutex
                         lock.unlock();
                     }

src/openpose/utilities/errorAndLog.cpp

@@ -6,6 +6,9 @@
 
 namespace op
 {
+    #ifdef USE_UNITY_SUPPORT
+    #endif
+
     // Private auxiliar functions
     bool checkIfErrorHas(const ErrorMode errorMode)
     {
@@ -142,6 +145,10 @@ namespace op
         if (checkIfErrorHas(ErrorMode::FileLogging))
             fileLogging(errorMessageToPrint);
 
+        // Unity logError
+        #ifdef USE_UNITY_SUPPORT
+        #endif
+
         // std::runtime_error
         if (checkIfErrorHas(ErrorMode::StdRuntimeError))
             throw std::runtime_error{errorMessageToPropagate};
@@ -161,6 +168,10 @@ namespace op
             // File logging
             if (checkIfLoggingHas(LogMode::FileLogging))
                 fileLogging(infoMessage);
+
+            // Unity log
+            #ifdef USE_UNITY_SUPPORT
+            #endif
         }
     }
 

src/openpose/utilities/keypoint.cpp

@@ -5,10 +5,11 @@
 
 namespace op
 {
-    const std::string errorMessage = "The Array<float> is not a RGB image or 3-channel keypoint array. This function"
+    const std::string errorMessage = "The Array<T> is not a RGB image or 3-channel keypoint array. This function"
                                      " is only for array of dimension: [sizeA x sizeB x 3].";
 
-    float getDistance(const Array<float>& keypoints, const int person, const int elementA, const int elementB)
+    template <typename T>
+    T getDistance(const Array<T>& keypoints, const int person, const int elementA, const int elementB)
     {
         try
         {
@@ -20,11 +21,16 @@ namespace op
         catch (const std::exception& e)
         {
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
-            return -1.f;
+            return T(-1);
         }
     }
+    template float getDistance(const Array<float>& keypoints, const int person, const int elementA,
+                               const int elementB);
+    template double getDistance(const Array<double>& keypoints, const int person, const int elementA,
+                                const int elementB);
 
-    void averageKeypoints(Array<float>& keypointsA, const Array<float>& keypointsB, const int personA)
+    template <typename T>
+    void averageKeypoints(Array<T>& keypointsA, const Array<T>& keypointsB, const int personA)
     {
         try
         {
@@ -41,7 +47,7 @@ namespace op
             {
                 const auto finalIndexA = keypointsA.getSize(2)*(personA*numberParts + part);
                 const auto finalIndexB = keypointsA.getSize(2)*part;
-                if (keypointsB[finalIndexB+2] - keypointsA[finalIndexA+2] > 0.05f)
+                if (keypointsB[finalIndexB+2] - keypointsA[finalIndexA+2] > T(0.05))
                 {
                     keypointsA[finalIndexA] = keypointsB[finalIndexB];
                     keypointsA[finalIndexA+1] = keypointsB[finalIndexB+1];
@@ -54,16 +60,19 @@ namespace op
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
+    template void averageKeypoints(Array<float>& keypointsA, const Array<float>& keypointsB, const int personA);
+    template void averageKeypoints(Array<double>& keypointsA, const Array<double>& keypointsB, const int personA);
 
-    void scaleKeypoints(Array<float>& keypoints, const float scale)
+    template <typename T>
+    void scaleKeypoints(Array<T>& keypoints, const T scale)
     {
         try
         {
-            if (!keypoints.empty() && scale != 1.f)
+            if (!keypoints.empty() && scale != T(1))
             {
                 // Error check
                 if (keypoints.getSize(2) != 3 && keypoints.getSize(2) != 4)
-                    error("The Array<float> is not a (x,y,score) or (x,y,z,score) format array. This"
+                    error("The Array<T> is not a (x,y,score) or (x,y,z,score) format array. This"
                           " function is only for those 2 dimensions: [sizeA x sizeB x 3or4].",
                           __LINE__, __FUNCTION__, __FILE__);
                 // Get #people and #parts
@@ -88,12 +97,15 @@ namespace op
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
+    template void scaleKeypoints(Array<float>& keypoints, const float scale);
+    template void scaleKeypoints(Array<double>& keypoints, const double scale);
 
-    void scaleKeypoints2d(Array<float>& keypoints, const float scaleX, const float scaleY)
+    template <typename T>
+    void scaleKeypoints2d(Array<T>& keypoints, const T scaleX, const T scaleY)
     {
         try
         {
-            if (!keypoints.empty() && scaleX != 1.f && scaleY != 1.f)
+            if (!keypoints.empty() && scaleX != T(1) && scaleY != T(1))
             {
                 // Error check
                 if (keypoints.getSize(2) != 3)
@@ -119,13 +131,15 @@ namespace op
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
+    template void scaleKeypoints2d(Array<float>& keypoints, const float scaleX, const float scaleY);
+    template void scaleKeypoints2d(Array<double>& keypoints, const double scaleX, const double scaleY);
 
-    void scaleKeypoints2d(Array<float>& keypoints, const float scaleX, const float scaleY, const float offsetX,
-                          const float offsetY)
+    template <typename T>
+    void scaleKeypoints2d(Array<T>& keypoints, const T scaleX, const T scaleY, const T offsetX, const T offsetY)
     {
         try
         {
-            if (!keypoints.empty() && scaleX != 1.f && scaleY != 1.f)
+            if (!keypoints.empty() && scaleX != T(1) && scaleY != T(1))
             {
                 // Error check
                 if (keypoints.getSize(2) != 3)
@@ -151,17 +165,21 @@ namespace op
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
+    template void scaleKeypoints2d(Array<float>& keypoints, const float scaleX, const float scaleY,
+                                   const float offsetX, const float offsetY);
+    template void scaleKeypoints2d(Array<double>& keypoints, const double scaleX, const double scaleY,
+                                   const double offsetX, const double offsetY);
 
-    void renderKeypointsCpu(Array<float>& frameArray, const Array<float>& keypoints,
-                            const std::vector<unsigned int>& pairs, const std::vector<float> colors,
-                            const float thicknessCircleRatio, const float thicknessLineRatioWRTCircle,
-                            const std::vector<float>& poseScales, const float threshold)
+    template <typename T>
+    void renderKeypointsCpu(Array<T>& frameArray, const Array<T>& keypoints, const std::vector<unsigned int>& pairs,
+                            const std::vector<T> colors, const T thicknessCircleRatio,
+                            const T thicknessLineRatioWRTCircle, const std::vector<T>& poseScales, const T threshold)
     {
         try
         {
             if (!frameArray.empty())
             {
-                // Array<float> --> cv::Mat
+                // Array<T> --> cv::Mat
                 auto frame = frameArray.getCvMat();
 
                 // Security check
@@ -179,7 +197,7 @@ namespace op
                 const auto shift = 0;
                 const auto numberColors = colors.size();
                 const auto numberScales = poseScales.size();
-                const auto thresholdRectangle = 0.1f;
+                const auto thresholdRectangle = T(0.1);
                 const auto numberKeypoints = keypoints.getSize(1);
 
                 // Keypoints
@@ -188,13 +206,13 @@ namespace op
                     const auto personRectangle = getKeypointsRectangle(keypoints, person, thresholdRectangle);
                     if (personRectangle.area() > 0)
                     {
-                        const auto ratioAreas = fastMin(1.f, fastMax(personRectangle.width/(float)width,
-                                                                     personRectangle.height/(float)height));
+                        const auto ratioAreas = fastMin(T(1), fastMax(personRectangle.width/(T)width,
+                                                                     personRectangle.height/(T)height));
                         // Size-dependent variables
                         const auto thicknessRatio = fastMax(intRound(std::sqrt(area)
                                                                      * thicknessCircleRatio * ratioAreas), 2);
                         // Negative thickness in cv::circle means that a filled circle is to be drawn.
-                        const auto thicknessCircle = fastMax(1, (ratioAreas > 0.05f ? thicknessRatio : -1));
+                        const auto thicknessCircle = fastMax(1, (ratioAreas > T(0.05) ? thicknessRatio : -1));
                         const auto thicknessLine = fastMax(1, intRound(thicknessRatio * thicknessLineRatioWRTCircle));
                         const auto radius = thicknessRatio / 2;
 
@@ -248,21 +266,30 @@ namespace op
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
+    template void renderKeypointsCpu(Array<float>& frameArray, const Array<float>& keypoints,
+                                     const std::vector<unsigned int>& pairs, const std::vector<float> colors,
+                                     const float thicknessCircleRatio, const float thicknessLineRatioWRTCircle,
+                                     const std::vector<float>& poseScales, const float threshold);
+    template void renderKeypointsCpu(Array<double>& frameArray, const Array<double>& keypoints,
+                                     const std::vector<unsigned int>& pairs, const std::vector<double> colors,
+                                     const double thicknessCircleRatio, const double thicknessLineRatioWRTCircle,
+                                     const std::vector<double>& poseScales, const double threshold);
 
-    Rectangle<float> getKeypointsRectangle(const Array<float>& keypoints, const int person, const float threshold)
+    template <typename T>
+    Rectangle<T> getKeypointsRectangle(const Array<T>& keypoints, const int person, const T threshold)
     {
         try
         {
             const auto numberKeypoints = keypoints.getSize(1);
             // Security checks
             if (numberKeypoints < 1)
-                error("Number body parts must be > 0", __LINE__, __FUNCTION__, __FILE__);
+                error("Number body parts must be > 0.", __LINE__, __FUNCTION__, __FILE__);
             // Define keypointPtr
             const auto keypointPtr = keypoints.getConstPtr() + person * keypoints.getSize(1) * keypoints.getSize(2);
-            float minX = std::numeric_limits<float>::max();
-            float maxX = 0.f;
-            float minY = minX;
-            float maxY = maxX;
+            T minX = std::numeric_limits<T>::max();
+            T maxX = std::numeric_limits<T>::lowest();
+            T minY = minX;
+            T maxY = maxX;
             for (auto part = 0 ; part < numberKeypoints ; part++)
             {
                 const auto score = keypointPtr[3*part + 2];
@@ -283,18 +310,23 @@ namespace op
                 }
             }
             if (maxX >= minX && maxY >= minY)
-                return Rectangle<float>{minX, minY, maxX-minX, maxY-minY};
+                return Rectangle<T>{minX, minY, maxX-minX, maxY-minY};
             else
-                return Rectangle<float>{};
+                return Rectangle<T>{};
         }
         catch (const std::exception& e)
         {
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
-            return Rectangle<float>{};
+            return Rectangle<T>{};
         }
     }
+    template Rectangle<float> getKeypointsRectangle(const Array<float>& keypoints, const int person,
+                                                    const float threshold);
+    template Rectangle<double> getKeypointsRectangle(const Array<double>& keypoints, const int person,
+                                                     const double threshold);
 
-    float getAverageScore(const Array<float>& keypoints, const int person)
+    template <typename T>
+    T getAverageScore(const Array<T>& keypoints, const int person)
     {
         try
         {
@@ -302,7 +334,7 @@ namespace op
             if (person >= keypoints.getSize(0))
                 error("Person index out of bounds.", __LINE__, __FUNCTION__, __FILE__);
             // Get average score
-            auto score = 0.f;
+            T score = T(0);
             const auto numberKeypoints = keypoints.getSize(1);
             const auto area = numberKeypoints * keypoints.getSize(2);
             const auto personOffset = person * area;
@@ -313,11 +345,14 @@ namespace op
         catch (const std::exception& e)
         {
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
-            return 0.f;
+            return T(0);
         }
     }
+    template float getAverageScore(const Array<float>& keypoints, const int person);
+    template double getAverageScore(const Array<double>& keypoints, const int person);
 
-    float getKeypointsArea(const Array<float>& keypoints, const int person, const float threshold)
+    template <typename T>
+    T getKeypointsArea(const Array<T>& keypoints, const int person, const T threshold)
     {
         try
         {
@@ -326,11 +361,14 @@ namespace op
         catch (const std::exception& e)
         {
             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
-            return 0.f;
+            return T(0);
         }
     }
+    template float getKeypointsArea(const Array<float>& keypoints, const int person, const float threshold);
+    template double getKeypointsArea(const Array<double>& keypoints, const int person, const double threshold);
 
-    int getBiggestPerson(const Array<float>& keypoints, const float threshold)
+    template <typename T>
+    int getBiggestPerson(const Array<T>& keypoints, const T threshold)
     {
         try
         {
@@ -338,7 +376,7 @@ namespace op
             {
                 const auto numberPeople = keypoints.getSize(0);
                 auto biggestPoseIndex = -1;
-                auto biggestArea = -1.f;
+                auto biggestArea = T(-1);
                 for (auto person = 0 ; person < numberPeople ; person++)
                 {
                     const auto newPersonArea = getKeypointsArea(keypoints, person, threshold);
@@ -359,4 +397,6 @@ namespace op
             return -1;
         }
     }
+    template int getBiggestPerson(const Array<float>& keypoints, const float threshold);
+    template int getBiggestPerson(const Array<double>& keypoints, const double threshold);
 }