#include #include #include #include #include namespace op { template void connectBodyPartsCpu(Array& poseKeyPoints, const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const T interThreshold, const int minSubsetCnt, const T minSubsetScore, const T scaleFactor) { try { // Parts Connection const auto& bodyPartPairs = POSE_BODY_PART_PAIRS[(int)poseModel]; const auto& mapIdx = POSE_MAP_IDX[(int)poseModel]; const auto numberBodyParts = POSE_NUMBER_BODY_PARTS[(int)poseModel]; const auto numberBodyPartPairs = bodyPartPairs.size() / 2; std::vector, double>> subset; // Vector = Each body part + body parts counter; double = subsetScore const auto subsetCounterIndex = numberBodyParts+1; const auto subsetSize = numberBodyParts+2; const auto peaksOffset = 3*(maxPeaks+1); const auto heatMapOffset = heatMapSize.area(); for (auto pairIndex = 0; pairIndex < numberBodyPartPairs; pairIndex++) { const auto bodyPartA = bodyPartPairs[2*pairIndex]; const auto bodyPartB = bodyPartPairs[2*pairIndex+1]; const auto* candidateA = peaksPtr + bodyPartA*peaksOffset; const auto* candidateB = peaksPtr + bodyPartB*peaksOffset; const auto nA = intRound(candidateA[0]); const auto nB = intRound(candidateB[0]); // add parts into the subset in special case if (nA == 0 || nB == 0) { // Change w.r.t. other if (nB != 0) { if (poseModel == PoseModel::COCO_18) { for (auto i = 1; i <= nB; i++) { bool num = false; const auto indexB = bodyPartB; for (auto j = 0; j < subset.size(); j++) { const auto off = (int)bodyPartB*peaksOffset + i*3 + 2; if (subset[j].first[indexB] == off) { num = true; break; } } if (!num) { std::vector rowVector(subsetSize, 0); rowVector[ bodyPartB ] = bodyPartB*peaksOffset + i*3 + 2; //store the index rowVector[subsetCounterIndex] = 1; //last number in each row is the parts number of that person const auto subsetScore = candidateB[i*3+2]; //second last number in each row is the total score subset.emplace_back(std::make_pair(rowVector, subsetScore)); } } } else if (poseModel == PoseModel::MPI_15 || poseModel == PoseModel::MPI_15_4) { for (auto i = 1; i <= nB; i++) { std::vector rowVector(subsetSize, 0); rowVector[ bodyPartB ] = bodyPartB*peaksOffset + i*3 + 2; //store the index rowVector[subsetCounterIndex] = 1; //last number in each row is the parts number of that person const auto subsetScore = candidateB[i*3+2]; //second last number in each row is the total score subset.emplace_back(std::make_pair(rowVector, subsetScore)); } } else error("Unknown model, cast to int = " + std::to_string((int)poseModel), __LINE__, __FUNCTION__, __FILE__); } else if (nA != 0) { if (poseModel == PoseModel::COCO_18) { for (auto i = 1; i <= nA; i++) { bool num = false; const auto indexA = bodyPartA; for (auto j = 0; j < subset.size(); j++) { const auto off = (int)bodyPartA*peaksOffset + i*3 + 2; if (subset[j].first[indexA] == off) { num = true; break; } } if (!num) { std::vector rowVector(subsetSize, 0); rowVector[ bodyPartA ] = bodyPartA*peaksOffset + i*3 + 2; //store the index rowVector[subsetCounterIndex] = 1; //last number in each row is the parts number of that person const auto subsetScore = candidateA[i*3+2]; //second last number in each row is the total score subset.emplace_back(std::make_pair(rowVector, subsetScore)); } } } else if (poseModel == PoseModel::MPI_15 || poseModel == PoseModel::MPI_15_4) { for (auto i = 1; i <= nA; i++) { std::vector rowVector(subsetSize, 0); rowVector[ bodyPartA ] = bodyPartA*peaksOffset + i*3 + 2; //store the index rowVector[subsetCounterIndex] = 1; //last number in each row is the parts number of that person const auto subsetScore = candidateA[i*3+2]; //second last number in each row is the total score subset.emplace_back(std::make_pair(rowVector, subsetScore)); } } else error("Unknown model, cast to int = " + std::to_string((int)poseModel), __LINE__, __FUNCTION__, __FILE__); } } else { std::vector> temp; const auto numInter = 10; const auto* const mapX = heatMapPtr + mapIdx[2*pairIndex] * heatMapOffset; const auto* const mapY = heatMapPtr + mapIdx[2*pairIndex+1] * heatMapOffset; for (auto i = 1; i <= nA; i++) { for (auto j = 1; j <= nB; j++) { const auto dX = candidateB[j*3] - candidateA[i*3]; const auto dY = candidateB[j*3+1] - candidateA[i*3+1]; const auto normVec = T(std::sqrt( dX*dX + dY*dY )); // If the peaksPtr are coincident. Don't connect them. if (normVec > 1e-6) { const auto sX = candidateA[i*3]; const auto sY = candidateA[i*3+1]; const auto vecX = dX/normVec; const auto vecY = dY/normVec; auto sum = 0.; auto count = 0; for (auto lm=0; lm < numInter; lm++) { const auto mX = fastMin(heatMapSize.width-1, intRound(sX + lm*dX/numInter)); const auto mY = fastMin(heatMapSize.height-1, intRound(sY + lm*dY/numInter)); checkGE(mX, 0, "", __LINE__, __FUNCTION__, __FILE__); checkGE(mY, 0, "", __LINE__, __FUNCTION__, __FILE__); const auto idx = mY * heatMapSize.width + mX; const auto score = (vecX*mapX[idx] + vecY*mapY[idx]); if (score > interThreshold) { sum += score; count++; } } // parts score + cpnnection score if (count > interMinAboveThreshold) temp.emplace_back(std::make_tuple(sum/count, i, j)); } } } // select the top minAB connection, assuming that each part occur only once // sort rows in descending order based on parts + connection score if (!temp.empty()) std::sort(temp.begin(), temp.end(), std::greater>()); std::vector> connectionK; const auto minAB = fastMin(nA, nB); std::vector occurA(nA, 0); std::vector occurB(nB, 0); auto counter = 0; for (auto row = 0; row < temp.size(); row++) { const auto score = std::get<0>(temp[row]); const auto x = std::get<1>(temp[row]); const auto y = std::get<2>(temp[row]); if (!occurA[x-1] && !occurB[y-1]) { connectionK.emplace_back(std::make_tuple(bodyPartA*peaksOffset + x*3 + 2, bodyPartB*peaksOffset + y*3 + 2, score)); counter++; if (counter==minAB) break; occurA[x-1] = 1; occurB[y-1] = 1; } } // Cluster all the body part candidates into subset based on the part connection // initialize first body part connection 15&16 if (pairIndex==0) { for (const auto connectionKI : connectionK) { std::vector rowVector(numberBodyParts+3, 0); const auto indexA = std::get<0>(connectionKI); const auto indexB = std::get<1>(connectionKI); const auto score = std::get<2>(connectionKI); rowVector[bodyPartPairs[0]] = indexA; rowVector[bodyPartPairs[1]] = indexB; rowVector[subsetCounterIndex] = 2; // add the score of parts and the connection const auto subsetScore = peaksPtr[indexA] + peaksPtr[indexB] + score; subset.emplace_back(std::make_pair(rowVector, subsetScore)); } } // // Add ears connections (in case person is looking to opposite direction to camera) // else if (poseModel == PoseModel::COCO_18 && (pairIndex==16 || pairIndex==17)) // { // for (const auto& connectionKI : connectionK) // { // const auto indexA = std::get<0>(connectionKI); // const auto indexB = std::get<1>(connectionKI); // for (auto& subsetJ : subset) // { // auto& subsetJFirst = subsetJ.first[bodyPartA]; // auto& subsetJFirstPlus1 = subsetJFirst[bodyPartB]; // if (subsetJFirst == indexA && subsetJFirstPlus1 == 0) // subsetJFirstPlus1 = indexB; // else if (subsetJFirstPlus1 == indexB && subsetJFirst == 0) // subsetJFirst = indexA; // } // } // } else { if (!connectionK.empty()) { // A is already in the subset, find its connection B for (auto i = 0; i < connectionK.size(); i++) { const auto indexA = std::get<0>(connectionK[i]); const auto indexB = std::get<1>(connectionK[i]); const auto score = std::get<2>(connectionK[i]); auto num = 0; for (auto j = 0; j < subset.size(); j++) { if (subset[j].first[bodyPartA] == indexA) { subset[j].first[bodyPartB] = indexB; num++; subset[j].first[subsetCounterIndex] = subset[j].first[subsetCounterIndex] + 1; subset[j].second = subset[j].second + peaksPtr[indexB] + score; } } // if can not find partA in the subset, create a new subset if (num==0) { std::vector rowVector(subsetSize, 0); rowVector[bodyPartA] = indexA; rowVector[bodyPartB] = indexB; rowVector[subsetCounterIndex] = 2; const auto subsetScore = peaksPtr[indexA] + peaksPtr[indexB] + score; subset.emplace_back(std::make_pair(rowVector, subsetScore)); } } } } } } // Delete people below the following thresholds: // a) minSubsetCnt: removed if less than minSubsetCnt body parts // b) minSubsetScore: removed if global score smaller than this // c) POSE_MAX_PEOPLE: keep first POSE_MAX_PEOPLE people above thresholds auto numberPeople = 0; std::vector validSubsetIndexes; validSubsetIndexes.reserve(fastMin((size_t)POSE_MAX_PEOPLE, subset.size())); for (auto index = 0 ; index < subset.size() ; index++) { const auto subsetCounter = subset[index].first[subsetCounterIndex]; const auto subsetScore = subset[index].second; if (subsetCounter >= minSubsetCnt && (subsetScore/subsetCounter) > minSubsetScore) { numberPeople++; validSubsetIndexes.emplace_back(index); if (numberPeople == POSE_MAX_PEOPLE) break; } else if (subsetCounter < 1) error("Bad subsetCounter. Bug in in this function if this happens.", __LINE__, __FUNCTION__, __FILE__); } // Fill and return poseKeyPoints if (numberPeople > 0) poseKeyPoints.reset({numberPeople, (int)numberBodyParts, 3}); else poseKeyPoints.reset(); for (auto person = 0 ; person < validSubsetIndexes.size() ; person++) { const auto& subsetI = subset[validSubsetIndexes[person]].first; for (auto bodyPart = 0u; bodyPart < numberBodyParts; bodyPart++) { const auto baseOffset = (person*numberBodyParts + bodyPart) * 3; const auto bodyPartIndex = subsetI[bodyPart]; if (bodyPartIndex > 0) { poseKeyPoints[baseOffset] = peaksPtr[bodyPartIndex-2] * scaleFactor; poseKeyPoints[baseOffset + 1] = peaksPtr[bodyPartIndex-1] * scaleFactor; poseKeyPoints[baseOffset + 2] = peaksPtr[bodyPartIndex]; } else { poseKeyPoints[baseOffset] = 0.f; poseKeyPoints[baseOffset + 1] = 0.f; poseKeyPoints[baseOffset + 2] = 0.f; } } } } catch (const std::exception& e) { error(e.what(), __LINE__, __FUNCTION__, __FILE__); } } template void connectBodyPartsCpu(Array& poseKeyPoints, const float* const heatMapPtr, const float* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const float interThreshold, const int minSubsetCnt, const float minSubsetScore, const float scaleFactor); template void connectBodyPartsCpu(Array& poseKeyPoints, const double* const heatMapPtr, const double* const peaksPtr, const PoseModel poseModel, const cv::Size& heatMapSize, const int maxPeaks, const int interMinAboveThreshold, const double interThreshold, const int minSubsetCnt, const double minSubsetScore, const double scaleFactor); }