From ff30d4f5bb7ed99a9f89de1ec5ba55f21585c696 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Mon, 25 Sep 2017 12:28:07 -0400 Subject: [PATCH] Added net drawer + alpha BODY_19 code --- doc/release_notes.md | 1 + examples/tests/drawProtoTxt.sh | 18 +++ include/openpose/pose/enumClasses.hpp | 3 +- include/openpose/pose/poseParameters.hpp | 119 ++++++++++++++------ src/openpose/pose/bodyPartConnectorBase.cpp | 9 +- src/openpose/pose/poseParameters.cpp | 2 +- src/openpose/pose/renderPose.cu | 37 +++++- src/openpose/utilities/flagsToOpenPose.cpp | 2 + 8 files changed, 149 insertions(+), 42 deletions(-) create mode 100644 examples/tests/drawProtoTxt.sh diff --git a/doc/release_notes.md b/doc/release_notes.md index 7018ad01..a0c7a3e8 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -115,3 +115,4 @@ OpenPose Library - Release Notes ## Current version (future OpenPose 1.1.1) 1. Main improvements: + 1. COCO JSON file outputs 0 as score for non-detected keypoints. diff --git a/examples/tests/drawProtoTxt.sh b/examples/tests/drawProtoTxt.sh new file mode 100644 index 00000000..382f1547 --- /dev/null +++ b/examples/tests/drawProtoTxt.sh @@ -0,0 +1,18 @@ +# Script for internal use. We might completely change it continuously and we will not answer questions about it. + +# Required +# sudo apt-get install graphviz + +# USAGE EXAMPLE +# clear && clear && make all -j24 && bash ./examples/tests/speed_test.sh + +# # Go back to main folder +# cd ../../ + +PROTO_TXT_PATH=/mnt/DataUbuntu/openpose_train/training_results/pose/pose_training.prototxt +OUTPUT_PNG_PATH=/mnt/DataUbuntu/openpose_train/training_results/pose/pose_training.png + +# Get model speed +python ~/devel/openpose_caffe_train/python/draw_net.py $PROTO_TXT_PATH $OUTPUT_PNG_PATH +display $OUTPUT_PNG_PATH +rm $OUTPUT_PNG_PATH diff --git a/include/openpose/pose/enumClasses.hpp b/include/openpose/pose/enumClasses.hpp index 2af3b99b..3e6a9c4a 100644 --- a/include/openpose/pose/enumClasses.hpp +++ b/include/openpose/pose/enumClasses.hpp @@ -12,7 +12,8 @@ namespace op MPI_15 = 1, /**< MPI model, with 15+1 components (see poseParameters.hpp for details). */ MPI_15_4 = 2, /**< Variation of the MPI model, reduced number of CNN stages to 4: faster but less accurate.*/ BODY_18 = 3, /**< Experimental. Do not use. */ - BODY_22 = 4, /**< Experimental. Do not use. */ + BODY_19 = 4, /**< Experimental. Do not use. */ + BODY_22 = 5, /**< Experimental. Do not use. */ Size, }; diff --git a/include/openpose/pose/poseParameters.hpp b/include/openpose/pose/poseParameters.hpp index 392f5f69..5f70d1bc 100644 --- a/include/openpose/pose/poseParameters.hpp +++ b/include/openpose/pose/poseParameters.hpp @@ -103,26 +103,26 @@ namespace op const std::vector POSE_MPI_COLORS_RENDER{POSE_MPI_COLORS_RENDER_GPU}; // BODY_18 const std::map POSE_BODY_18_BODY_PARTS { // Windows map copy error if `= POSE_COCO_BODY_PARTS` - { 0, "Nose" }, - { 1, "Neck" }, - { 2, "RShoulder" }, - { 3, "RElbow" }, - { 4, "RWrist" }, - { 5, "LShoulder" }, - { 6, "LElbow" }, - { 7, "LWrist" }, - { 8, "RHip" }, - { 9, "RKnee" }, - { 10, "RAnkle" }, - { 11, "LHip" }, - { 12, "LKnee" }, - { 13, "LAnkle" }, - { 14, "REye" }, - { 15, "LEye" }, - { 16, "REar" }, - { 17, "LEar" }, - { 18, "Background" } - }; + { 0, "Nose" }, + { 1, "Neck" }, + { 2, "RShoulder" }, + { 3, "RElbow" }, + { 4, "RWrist" }, + { 5, "LShoulder" }, + { 6, "LElbow" }, + { 7, "LWrist" }, + { 8, "RHip" }, + { 9, "RKnee" }, + { 10, "RAnkle" }, + { 11, "LHip" }, + { 12, "LKnee" }, + { 13, "LAnkle" }, + { 14, "REye" }, + { 15, "LEye" }, + { 16, "REar" }, + { 17, "LEar" }, + { 18, "Background" } + }; const unsigned int POSE_BODY_18_NUMBER_PARTS {POSE_COCO_NUMBER_PARTS}; const std::vector POSE_BODY_18_MAP_IDX {POSE_COCO_MAP_IDX}; #define POSE_BODY_18_PAIRS_RENDER_GPU POSE_COCO_PAIRS_RENDER_GPU @@ -130,7 +130,56 @@ namespace op const std::vector POSE_BODY_18_PAIRS {POSE_COCO_PAIRS}; #define POSE_BODY_18_COLORS_RENDER_GPU POSE_COCO_COLORS_RENDER_GPU const std::vector POSE_BODY_18_COLORS_RENDER {POSE_BODY_18_COLORS_RENDER_GPU}; - // BODY_22 (experimental, do not use) + // BODY_19 + const std::map POSE_BODY_19_BODY_PARTS { + { 0, "Nose" }, + { 1, "Neck" }, + { 2, "RShoulder" }, + { 3, "RElbow" }, + { 4, "RWrist" }, + { 5, "LShoulder" }, + { 6, "LElbow" }, + { 7, "LWrist" }, + { 8, "LowerAbs" }, + { 9, "RHip" }, + { 10, "RKnee" }, + { 11, "RAnkle" }, + { 12, "LHip" }, + { 13, "LKnee" }, + { 14, "LAnkle" }, + { 15, "REye" }, + { 16, "LEye" }, + { 17, "REar" }, + { 18, "LEar" }, + { 19, "Background" } + }; + const unsigned int POSE_BODY_19_NUMBER_PARTS = 19u; // Equivalent to size of std::map POSE_BODY_19_BODY_PARTS - 1 (removing background) + const std::vector POSE_BODY_19_MAP_IDX {20,21, 34,35, 42,43, 36,37, 38,39, 44,45, 46,47, 26,27, 22,23, 24,25, 28,29, 30,31, 32,33, 50,51, 52,53, 56,57, 54,55, 58,59, 40,41, 48,49}; + #define POSE_BODY_19_PAIRS_RENDER_GPU {1,8, 1,2, 1,5, 2,3, 3,4, 5,6, 6,7, 8,9, 9,10, 10,11, 8,12, 12,13, 13,14, 1,0, 0,15, 15,17, 0,16, 16,18} + const std::vector POSE_BODY_19_PAIRS_RENDER {POSE_COCO_PAIRS_RENDER_GPU}; + const std::vector POSE_BODY_19_PAIRS {1,8, 1,2, 1,5, 2,3, 3,4, 5,6, 6,7, 8,9, 9,10, 10,11, 8,12, 12,13, 13,14, 1,0, 0,15, 15,17, 0,16, 16,18, 2,17, 5,18}; + #define POSE_BODY_19_COLORS_RENDER_GPU \ + 255.f, 0.f, 85.f, \ + 255.f, 0.f, 0.f, \ + 255.f, 85.f, 0.f, \ + 255.f, 170.f, 0.f, \ + 255.f, 255.f, 0.f, \ + 170.f, 255.f, 0.f, \ + 85.f, 255.f, 0.f, \ + 0.f, 255.f, 0.f, \ + 255.f, 0.f, 0.f, \ + 0.f, 255.f, 85.f, \ + 0.f, 255.f, 170.f, \ + 0.f, 255.f, 255.f, \ + 0.f, 170.f, 255.f, \ + 0.f, 85.f, 255.f, \ + 0.f, 0.f, 255.f, \ + 255.f, 0.f, 170.f, \ + 170.f, 0.f, 255.f, \ + 255.f, 0.f, 255.f, \ + 85.f, 0.f, 255.f + const std::vector POSE_BODY_19_COLORS_RENDER{POSE_BODY_19_COLORS_RENDER_GPU}; + // BODY_22 const std::map POSE_BODY_22_BODY_PARTS { {0, "Nose"}, {1, "Neck"}, @@ -154,7 +203,7 @@ namespace op {19, "LTest1"}, {20, "RTest2"}, {21, "LTest2"}, - {22, "Background"}, + {22, "Background"} }; const unsigned int POSE_BODY_22_NUMBER_PARTS = 22u; // Equivalent to size of std::map POSE_BODY_22_BODY_PARTS - 1 (removing background) const std::vector POSE_BODY_22_MAP_IDX {35,36, 43,45, 37,38, 39,40, 45,46, 47,48, 23,24, 25,26, 27,28, 29,30, 31,32, 33,34, 51,52}; @@ -184,31 +233,32 @@ namespace op // Constant Array Parameters const std::array POSE_CCN_DECREASE_FACTOR{ - 8.f, 8.f, 8.f, 8.f + 8.f, 8.f, 8.f, 8.f, 8.f, 8.f }; const std::array POSE_MAX_PEAKS{ - POSE_MAX_PEOPLE, POSE_MAX_PEOPLE, POSE_MAX_PEOPLE, POSE_MAX_PEOPLE + POSE_MAX_PEOPLE, POSE_MAX_PEOPLE, POSE_MAX_PEOPLE, POSE_MAX_PEOPLE, POSE_MAX_PEOPLE, POSE_MAX_PEOPLE }; const std::array POSE_NUMBER_BODY_PARTS{ - POSE_COCO_NUMBER_PARTS, POSE_MPI_NUMBER_PARTS, POSE_MPI_NUMBER_PARTS, POSE_BODY_18_NUMBER_PARTS, POSE_BODY_22_NUMBER_PARTS + POSE_COCO_NUMBER_PARTS, POSE_MPI_NUMBER_PARTS, POSE_MPI_NUMBER_PARTS, POSE_BODY_18_NUMBER_PARTS, POSE_BODY_19_NUMBER_PARTS, POSE_BODY_22_NUMBER_PARTS }; const std::array, (int)PoseModel::Size> POSE_BODY_PART_PAIRS{ - POSE_COCO_PAIRS, POSE_MPI_PAIRS, POSE_MPI_PAIRS, POSE_BODY_18_PAIRS, POSE_BODY_22_PAIRS + POSE_COCO_PAIRS, POSE_MPI_PAIRS, POSE_MPI_PAIRS, POSE_BODY_18_PAIRS, POSE_BODY_19_PAIRS, POSE_BODY_22_PAIRS }; const std::array, (int)PoseModel::Size> POSE_BODY_PART_PAIRS_RENDER{ - POSE_COCO_PAIRS_RENDER, POSE_MPI_PAIRS, POSE_MPI_PAIRS, POSE_BODY_18_PAIRS_RENDER, POSE_BODY_22_PAIRS_RENDER + POSE_COCO_PAIRS_RENDER, POSE_MPI_PAIRS, POSE_MPI_PAIRS, POSE_BODY_18_PAIRS_RENDER, POSE_BODY_19_PAIRS_RENDER, POSE_BODY_22_PAIRS_RENDER }; const std::array, (int)PoseModel::Size> POSE_MAP_IDX{ - POSE_COCO_MAP_IDX, POSE_MPI_MAP_IDX, POSE_MPI_MAP_IDX, POSE_BODY_18_MAP_IDX, POSE_BODY_22_MAP_IDX + POSE_COCO_MAP_IDX, POSE_MPI_MAP_IDX, POSE_MPI_MAP_IDX, POSE_BODY_18_MAP_IDX, POSE_BODY_19_MAP_IDX, POSE_BODY_22_MAP_IDX }; const std::array, (int)PoseModel::Size> POSE_COLORS{ - POSE_COCO_COLORS_RENDER,POSE_MPI_COLORS_RENDER, POSE_MPI_COLORS_RENDER, POSE_BODY_18_COLORS_RENDER, POSE_BODY_22_COLORS_RENDER + POSE_COCO_COLORS_RENDER,POSE_MPI_COLORS_RENDER, POSE_MPI_COLORS_RENDER, POSE_BODY_18_COLORS_RENDER, POSE_BODY_19_COLORS_RENDER, POSE_BODY_22_COLORS_RENDER }; const std::array POSE_PROTOTXT{ "pose/coco/pose_deploy_linevec.prototxt", "pose/mpi/pose_deploy_linevec.prototxt", "pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt", "pose/body_18/pose_deploy.prototxt", + "pose/body_19/pose_deploy.prototxt", "pose/body_22/pose_deploy.prototxt" }; const std::array POSE_TRAINED_MODEL{ @@ -216,6 +266,7 @@ namespace op "pose/mpi/pose_iter_160000.caffemodel", "pose/mpi/pose_iter_160000.caffemodel", "pose/body_18/pose_iter_XXXXXX.caffemodel", + "pose/body_19/pose_iter_XXXXXX.caffemodel", "pose/body_22/pose_iter_40000.caffemodel" }; // POSE_BODY_PART_MAPPING crashes on Windows at dynamic initialization, to avoid this crash: @@ -228,19 +279,19 @@ namespace op // Default Model Parameters // They might be modified on running time const std::array POSE_DEFAULT_NMS_THRESHOLD{ - 0.05f, 0.6f, 0.3f, 0.05f, 0.05f + 0.05f, 0.6f, 0.3f, 0.05f, 0.05f, 0.05f }; const std::array POSE_DEFAULT_CONNECT_INTER_MIN_ABOVE_THRESHOLD{ - 9, 8, 8, 9, 9 + 9, 8, 8, 9, 9, 9 }; const std::array POSE_DEFAULT_CONNECT_INTER_THRESHOLD{ - 0.05f, 0.01f, 0.01f, 0.05f, 0.05f + 0.05f, 0.01f, 0.01f, 0.05f, 0.05f, 0.05f }; const std::array POSE_DEFAULT_CONNECT_MIN_SUBSET_CNT{ - 3, 3, 3, 3, 3 + 3, 3, 3, 3, 3, 3 }; const std::array POSE_DEFAULT_CONNECT_MIN_SUBSET_SCORE{ - 0.4f, 0.4f, 0.4f, 0.4f, 0.4f + 0.4f, 0.4f, 0.4f, 0.4f, 0.4f, 0.4f }; // Rendering parameters diff --git a/src/openpose/pose/bodyPartConnectorBase.cpp b/src/openpose/pose/bodyPartConnectorBase.cpp index 19e09138..5463f304 100644 --- a/src/openpose/pose/bodyPartConnectorBase.cpp +++ b/src/openpose/pose/bodyPartConnectorBase.cpp @@ -40,7 +40,8 @@ namespace op // Change w.r.t. other if (nA == 0) // nB == 0 or not { - if (poseModel == PoseModel::COCO_18 || poseModel == PoseModel::BODY_18 || poseModel == PoseModel::BODY_22) + if (poseModel == PoseModel::COCO_18 || poseModel == PoseModel::BODY_18 + || poseModel == PoseModel::BODY_19 || poseModel == PoseModel::BODY_22) { for (auto i = 1; i <= nB; i++) { @@ -81,7 +82,8 @@ namespace op } else // if (nA != 0 && nB == 0) { - if (poseModel == PoseModel::COCO_18 || poseModel == PoseModel::BODY_18 || poseModel == PoseModel::BODY_22) + if (poseModel == PoseModel::COCO_18 || poseModel == PoseModel::BODY_18 + || poseModel == PoseModel::BODY_19 || poseModel == PoseModel::BODY_22) { for (auto i = 1; i <= nA; i++) { @@ -214,7 +216,8 @@ namespace op } } // Add ears connections (in case person is looking to opposite direction to camera) - else if (poseModel == PoseModel::COCO_18 && (pairIndex==17 || pairIndex==18)) + else if (((poseModel == PoseModel::COCO_18 || poseModel == PoseModel::BODY_18) && (pairIndex==17 || pairIndex==18)) + || (poseModel == PoseModel::BODY_19 && (pairIndex==18 || pairIndex==19))) { for (const auto& connectionKI : connectionK) { diff --git a/src/openpose/pose/poseParameters.cpp b/src/openpose/pose/poseParameters.cpp index 832996e8..903c0c4b 100644 --- a/src/openpose/pose/poseParameters.cpp +++ b/src/openpose/pose/poseParameters.cpp @@ -3,7 +3,7 @@ namespace op { const std::array, (int)PoseModel::Size> POSE_BODY_PART_MAPPING{ - POSE_COCO_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_BODY_18_BODY_PARTS, POSE_BODY_22_BODY_PARTS + POSE_COCO_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_BODY_18_BODY_PARTS, POSE_BODY_19_BODY_PARTS, POSE_BODY_22_BODY_PARTS }; unsigned int poseBodyPartMapStringToKey(const PoseModel poseModel, const std::vector& strings) diff --git a/src/openpose/pose/renderPose.cu b/src/openpose/pose/renderPose.cu index a2433f8f..5d1df7f5 100644 --- a/src/openpose/pose/renderPose.cu +++ b/src/openpose/pose/renderPose.cu @@ -10,10 +10,12 @@ namespace op __constant__ const float PI = 3.14159265358979323846264338327950288419716939937510582097494459230781640628620899862803482534211706798214808651328230664709384460955058223172535940812848111745f; __constant__ const unsigned int COCO_PAIRS_GPU[] = POSE_COCO_PAIRS_RENDER_GPU; __constant__ const unsigned int BODY_18_PAIRS_GPU[] = POSE_BODY_18_PAIRS_RENDER_GPU; + __constant__ const unsigned int BODY_19_PAIRS_GPU[] = POSE_BODY_19_PAIRS_RENDER_GPU; __constant__ const unsigned int BODY_22_PAIRS_GPU[] = POSE_BODY_22_PAIRS_RENDER_GPU; __constant__ const unsigned int MPI_PAIRS_GPU[] = POSE_MPI_PAIRS_RENDER_GPU; __constant__ const float COCO_COLORS[] = {POSE_COCO_COLORS_RENDER_GPU}; __constant__ const float BODY_18_COLORS[] = {POSE_BODY_18_COLORS_RENDER_GPU}; + __constant__ const float BODY_19_COLORS[] = {POSE_BODY_19_COLORS_RENDER_GPU}; __constant__ const float BODY_22_COLORS[] = {POSE_BODY_22_COLORS_RENDER_GPU}; __constant__ const float MPI_COLORS[] = {POSE_MPI_COLORS_RENDER_GPU}; @@ -148,6 +150,32 @@ namespace op radius, stickwidth, threshold, alphaColorToAdd, blendOriginalFrame, (googlyEyes ? 14 : -1), (googlyEyes ? 15 : -1)); } + __global__ void renderPoseBody19(float* targetPtr, const int targetWidth, const int targetHeight, const float* const posePtr, + const int numberPeople, const float threshold, const bool googlyEyes, const bool blendOriginalFrame, + const float alphaColorToAdd) + { + const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; + const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; + const auto globalIdx = threadIdx.y * blockDim.x + threadIdx.x; + + // Shared parameters + __shared__ float2 sharedMins[POSE_MAX_PEOPLE]; + __shared__ float2 sharedMaxs[POSE_MAX_PEOPLE]; + __shared__ float sharedScaleF[POSE_MAX_PEOPLE]; + + // Other parameters + const auto numberPartPairs = sizeof(BODY_19_PAIRS_GPU) / (2*sizeof(BODY_19_PAIRS_GPU[0])); + const auto numberColors = sizeof(BODY_19_COLORS) / (3*sizeof(BODY_19_COLORS[0])); + const auto radius = fastMin(targetWidth, targetHeight) / 100.f; + const auto stickwidth = fastMin(targetWidth, targetHeight) / 120.f; + + // Render key points + renderKeypoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, + globalIdx, x, y, targetWidth, targetHeight, posePtr, BODY_19_PAIRS_GPU, numberPeople, + POSE_BODY_19_NUMBER_PARTS, numberPartPairs, BODY_19_COLORS, numberColors, + radius, stickwidth, threshold, alphaColorToAdd, blendOriginalFrame, (googlyEyes ? 15 : -1), (googlyEyes ? 16 : -1)); + } + __global__ void renderPoseBody22(float* targetPtr, const int targetWidth, const int targetHeight, const float* const posePtr, const int numberPeople, const float threshold, const bool googlyEyes, const bool blendOriginalFrame, const float alphaColorToAdd) @@ -361,13 +389,16 @@ namespace op std::tie(threadsPerBlock, numBlocks) = getNumberCudaThreadsAndBlocks(frameSize); if (poseModel == PoseModel::COCO_18) - renderPoseCoco<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, + renderPoseCoco<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending); else if (poseModel == PoseModel::BODY_18) - renderPoseBody18<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, + renderPoseBody18<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, + renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending); + else if (poseModel == PoseModel::BODY_19) + renderPoseBody19<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending); else if (poseModel == PoseModel::BODY_22) - renderPoseBody22<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, + renderPoseBody22<<>>(framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending); else if (poseModel == PoseModel::MPI_15 || poseModel == PoseModel::MPI_15_4) renderPoseMpi29Parts<<>>(framePtr, frameSize.x, frameSize.y, posePtr, diff --git a/src/openpose/utilities/flagsToOpenPose.cpp b/src/openpose/utilities/flagsToOpenPose.cpp index 9a88c378..3aab7bf3 100644 --- a/src/openpose/utilities/flagsToOpenPose.cpp +++ b/src/openpose/utilities/flagsToOpenPose.cpp @@ -20,6 +20,8 @@ namespace op return PoseModel::MPI_15_4; else if (poseModeString == "BODY_18") return PoseModel::BODY_18; + else if (poseModeString == "BODY_19") + return PoseModel::BODY_19; else if (poseModeString == "BODY_22") return PoseModel::BODY_22; // else -- GitLab