From 1a544a83e8748badd82275de6651f5b385c3db61 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Wed, 5 Sep 2018 19:52:10 -0400 Subject: [PATCH] Experimental tracking released --- README.md | 6 +-- doc/quick_start.md | 22 ++++++++ doc/release_notes.md | 1 + doc/released_features.md | 3 +- src/openpose/pose/bodyPartConnectorBase.cpp | 59 +++++++++++++++------ src/openpose/tracking/personTracker.cpp | 30 +++++++---- 6 files changed, 88 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8de96049..f383ddba 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ - Compatible with Flir/Point Grey cameras, but provided C++ demos to add your custom input. - **Calibration toolbox**: - Easy estimation of distortion, intrinsic, and extrinsic camera parameters. + - **Single-person tracking** for further speed up or visual smoothing. - **Input**: Image, video, webcam, Flir/Point Grey and IP camera. Included C++ demos to add your custom input. - **Output**: Basic image + keypoint display/saving (PNG, JPG, AVI, ...), keypoint saving (JSON, XML, YML, ...), and/or keypoints as array class. - **OS**: Ubuntu (14, 16), Windows (8, 10), Mac OSX, Nvidia TX2. @@ -36,6 +37,7 @@ ## Latest Features +- Sep 2018: [**Experimental single-person tracker**](doc/quick_start.md#tracking) for further speed up or visual smoothing! - Jun 2018: [**Combined body-foot model released! 40% faster and 5% more accurate**](doc/installation.md)! - Jun 2018: [**Python API**](doc/modules/python_module.md) released! - Jun 2018: [**OpenCL/AMD graphic card version**](doc/installation.md) released! @@ -43,10 +45,6 @@ - Jun 2018: [**Mac OSX version (CPU)**](doc/installation.md) released! - Mar 2018: [**CPU version**](doc/installation.md#cpu-version)! - Mar 2018: [**3-D keypoint reconstruction module**](doc/modules/3d_reconstruction_module.md) (from multiple camera views)! -- Sep 2017: [**CMake**](doc/installation.md) installer and **IP camera** support! -- Jul 2017: [**Windows portable binaries and demo**](https://github.com/CMU-Perceptual-Computing-Lab/openpose/releases)! -- Jul 2017: **Hands** released! -- Jun 2017: **Face** released! For further details, check [all released features](doc/released_features.md) and [release notes](doc/release_notes.md). diff --git a/doc/quick_start.md b/doc/quick_start.md index b4b291c5..692769f6 100644 --- a/doc/quick_start.md +++ b/doc/quick_start.md @@ -8,6 +8,7 @@ OpenPose - Quick Start 3. [Running on Images](#running-on-images) 4. [Maximum Accuracy Configuration](#maximum-accuracy-configuration) 5. [3-D Reconstruction](#3-d-reconstruction) + 6. [Tracking](#tracking) 2. [Expected Visual Results](#expected-visual-results) @@ -167,6 +168,27 @@ build\x64\Release\OpenPoseDemo.exe --flir_camera --3d --number_people_max 1 --fa +### Tracking +1. Runtime huge speed up by reducing the accuracy: +``` +# Using OpenPose 1 frame, tracking the following e.g., 5 frames +./build/examples/openpose/openpose.bin --tracking 5 --number_people_max 1 +``` + +2. Runtime speed up while keeping most of the accuracy: +``` +:: Using OpenPose 1 frame and tracking another frame +./build/examples/openpose/openpose.bin --tracking 1 --number_people_max 1 +``` + +3. Visual smoothness: +``` +# Running both OpenPose and tracking on each frame. Note: There is no speed up/slow down +./build/examples/openpose/openpose.bin --tracking 0 --number_people_max 1 +``` + + + ## Expected Visual Results The visual GUI should show the original image with the poses blended on it, similarly to the pose of this gif:

diff --git a/doc/release_notes.md b/doc/release_notes.md index e5179104..0afea8f2 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -261,6 +261,7 @@ OpenPose Library - Release Notes ## Current version - future OpenPose 1.4.1 1. Main improvements: + 1. Added initial single-person tracker for further speed up or visual smoothing (`--tracking` flag). 2. Functions or parameters renamed: 3. Main bugs fixed: diff --git a/doc/released_features.md b/doc/released_features.md index b694453d..faacfa09 100644 --- a/doc/released_features.md +++ b/doc/released_features.md @@ -1,6 +1,7 @@ -OpenPose Library - Latest Released Features +OpenPose Library - All Released Features ==================================== +- Sep 2018: [**Experimental tracker**](./quick_start.md#tracking)! - Jun 2018: [**Combined body-foot model released! 40% faster and 5% more accurate**](./installation.md)! - Jun 2018: [**Python API**](./modules/python_module.md) released! - Jun 2018: [**OpenCL/AMD graphic card version**](./modules/calibration_module.md) released! diff --git a/src/openpose/pose/bodyPartConnectorBase.cpp b/src/openpose/pose/bodyPartConnectorBase.cpp index 1c2d02eb..4a054750 100644 --- a/src/openpose/pose/bodyPartConnectorBase.cpp +++ b/src/openpose/pose/bodyPartConnectorBase.cpp @@ -609,24 +609,46 @@ namespace op // const auto numberPeople = intRound(peaksPtr[rootNumberIndex]); poseKeypoints.reset({numberPeople, (int)numberBodyParts, 3}, 0); poseScores.reset(numberPeople, 0); + // // 48 channels + // const std::vector average{ + // 0.f, -2.76364f, -1.3345f, 0.f, -1.95322f, 3.95679f, -1.20664f, 4.76543f, + // 1.3345f, 0.f, 1.92318f, 3.96891f, 1.17999f, 4.7901f, 0.f, 7.72201f, + // -0.795236f, 7.74017f, -0.723963f, 11.209f, -0.651316f, 15.6972f, + // 0.764623f, 7.74869f, 0.70755f, 11.2307f, 0.612832f, 15.7281f, + // -0.123134f, -3.43515f, 0.111775f, -3.42761f, + // -0.387066f, -3.16603f, 0.384038f, -3.15951f, + // 0.344764f, 12.9666f, 0.624157f, 12.9057f, 0.195454f, 12.565f, + // -1.06074f, 12.9951f, -1.2427f, 12.9309f, -0.800837f, 12.5845f}; + // const std::vector sigma{ + // 3.39629f, 3.15605f, 3.16913f, 1.8234f, 5.82252f, 5.05674f, 7.09876f, 6.64574f, + // 3.16913f, 1.8234f, 5.79415f, 5.01424f, 7.03866f, 6.62427f, 5.52593f, 6.75962f, + // 5.91224f, 6.87241f, 8.66473f, 10.1792f, 11.5871f, 13.6565f, + // 5.86653f, 6.89568f, 8.68067f, 10.2127f, 11.5954f, 13.6722f, + // 3.3335f, 3.49128f, 3.34476f, 3.50079f, + // 2.93982f, 3.11151f, 2.95006f, 3.11004f, + // 9.69408f, 7.58921f, 9.71193f, 7.44185f, 9.19343f, 7.11157f, + // 9.16848f, 7.86122f, 9.07613f, 7.83682f, 8.91951f, 7.33715f}; + // 50 channels const std::vector average{ - 0.f, -2.76364f, -1.3345f, 0.f, -1.95322f, 3.95679f, -1.20664f, 4.76543f, - 1.3345f, 0.f, 1.92318f, 3.96891f, 1.17999f, 4.7901f, 0.f, 7.72201f, - -0.795236f, 7.74017f, -0.723963f, 11.209f, -0.651316f, 15.6972f, - 0.764623f, 7.74869f, 0.70755f, 11.2307f, 0.612832f, 15.7281f, - -0.123134f, -3.43515f, 0.111775f, -3.42761f, - -0.387066f, -3.16603f, 0.384038f, -3.15951f, - 0.344764f, 12.9666f, 0.624157f, 12.9057f, 0.195454f, 12.565f, - -1.06074f, 12.9951f, -1.2427f, 12.9309f, -0.800837f, 12.5845f}; + 0, -6.55251, + 0, -4.15062, -1.48818, -4.15506, -2.22408, -0.312264, -1.42204, 0.588495, + 1.51044, -4.14629, 2.2113, -0.312283, 1.41081, 0.612377, -0, 3.41112, + -0.932306, 3.45504, -0.899812, 6.79837, -0.794223, 11.4972, + 0.919047, 3.46442, 0.902314, 6.81245, 0.79518, 11.5132, + -0.243982, -7.07925, 0.28065, -7.07398, + -0.792812, -7.09374, 0.810145, -7.06958, + 0.582387, 7.46846, 0.889349, 7.40577, 0.465088, 7.03969, + -0.96686, 7.46148, -1.20773, 7.38834, -0.762135, 6.99575}; const std::vector sigma{ - 3.39629f, 3.15605f, 3.16913f, 1.8234f, 5.82252f, 5.05674f, 7.09876f, 6.64574f, - 3.16913f, 1.8234f, 5.79415f, 5.01424f, 7.03866f, 6.62427f, 5.52593f, 6.75962f, - 5.91224f, 6.87241f, 8.66473f, 10.1792f, 11.5871f, 13.6565f, - 5.86653f, 6.89568f, 8.68067f, 10.2127f, 11.5954f, 13.6722f, - 3.3335f, 3.49128f, 3.34476f, 3.50079f, - 2.93982f, 3.11151f, 2.95006f, 3.11004f, - 9.69408f, 7.58921f, 9.71193f, 7.44185f, 9.19343f, 7.11157f, - 9.16848f, 7.86122f, 9.07613f, 7.83682f, 8.91951f, 7.33715f}; + 7.26789, 9.70751, + 6.29588, 8.93472, 6.97401, 9.13746, 7.49632, 9.44757, 8.06695, 9.97319, + 6.99726, 9.14608, 7.50529, 9.43568, 8.05888, 9.98207, 6.38929, 9.29314, + 6.71801, 9.39271, 8.00608, 10.6141, 10.3416, 12.7812, + 6.69875, 9.41407, 8.01876, 10.637, 10.3475, 12.7849, + 7.30923, 9.7324, 7.27886, 9.73406, + 7.35978, 9.7289, 7.28914, 9.67711, + 7.93153, 8.10845, 7.95577, 8.01729, 7.56865, 7.87314, + 7.4655, 8.25336, 7.43958, 8.26333, 7.33667, 7.97446}; // To get ideal distance const auto numberBodyPartsAndBkgAndPAFChannels = numberBodyParts + 1 + bodyPartPairs.size(); const auto heatMapOffset = heatMapSize.area(); @@ -647,7 +669,10 @@ namespace op { if (bpOrig != rootIndex) { - const auto bpChannel = (bpOrig < rootIndex ? bpOrig : bpOrig-1); + // // 48 channels + // const auto bpChannel = (bpOrig < rootIndex ? bpOrig : bpOrig-1); + // 50 channels + const auto bpChannel = bpOrig; // Get ideal distance const auto offsetIndex = numberBodyPartsAndBkgAndPAFChannels + 2*bpChannel; const auto* mapX = heatMapPtr + offsetIndex * heatMapOffset; diff --git a/src/openpose/tracking/personTracker.cpp b/src/openpose/tracking/personTracker.cpp index 4383b943..7521c5fc 100644 --- a/src/openpose/tracking/personTracker.cpp +++ b/src/openpose/tracking/personTracker.cpp @@ -299,23 +299,23 @@ namespace op { if (personEntries.size() && !poseIds.empty()) { - int dims[] = { (int)personEntries.size(), (int)personEntries.begin()->second.keypoints.size(), 3 }; - cv::Mat opArrayMat(3,dims,CV_32FC1); + poseKeypoints.reset( + {(int)personEntries.size(), (int)personEntries.begin()->second.keypoints.size(), 3}); for (auto i=0; i(baseIndex + 0) = pe.keypoints[j].x; - opArrayMat.at(baseIndex + 1) = pe.keypoints[j].y; - opArrayMat.at(baseIndex + 2) = (int)pe.status[j]; + const auto baseIndex = baseIndexY + j*poseKeypoints.getSize(2); + poseKeypoints[baseIndex] = pe.keypoints[j].x; + poseKeypoints[baseIndex+1] = pe.keypoints[j].y; + poseKeypoints[baseIndex+2] = (int)pe.status[j]; if (pe.keypoints[j].x == 0 && pe.keypoints[j].y == 0) - opArrayMat.at(baseIndex + 2) = 0; + poseKeypoints[baseIndex+2] = 0; } } - poseKeypoints.setFrom(opArrayMat); } } catch (const std::exception& e) @@ -359,8 +359,8 @@ namespace op { try { - error("PersonTracker (`tracking` flag) buggy and not working yet, but we are working on it!" - " Coming soon!", __LINE__, __FUNCTION__, __FILE__); + log("Person tracking (`tracking` flag) is in experimental phase. Please, let us know if you" + " find any bug on this alpha version.", op::Priority::High); } catch (const std::exception& e) { @@ -377,6 +377,14 @@ namespace op { try { + // Sanity Checks + if (poseKeypoints.getSize(0) > 1) + error("Person tracking (`--tracking` flag) is in experimental phase and only allows tracking of up" + " to 1 person at the time. Please, also include the `--number_people_max 1` flag when using" + " the `--tracking` flag. Tracking more than one person at the time is not expected as" + " short- nor medium-term goal.", + __LINE__, __FUNCTION__, __FILE__); + /* * 1. Get poseKeypoints for all people - Checks * 2. If last image is empty or mPersonEntries is empty (& poseKeypoints and poseIds has data or crash it) -- GitLab