(db/snapshot) Add root path for snapshot GC (#2936)

* Add root path for snapshot GC
Signed-off-by: Nyhz <413554850@qq.com>

* Add subfolder db in root path
Signed-off-by: Nyhz <413554850@qq.com>

core/src/db/Utils.cpp

@@ -29,6 +29,7 @@
 //#include "storage/s3/S3ClientWrapper.h"
 #include "utils/CommonUtil.h"
 #include "utils/Log.h"
+#include "utils/StringHelpFunctions.h"
 
 #include <map>
 
@@ -49,6 +50,15 @@ ConstructParentFolder(const std::string& db_path, const meta::SegmentSchema& tab
 
 }  // namespace
 
+std::string
+ConstructCollectionRootPath(const std::string& root_path) {
+    if (StringHelpFunctions::EndWithSlash(root_path)) {
+        return root_path + "db" + TABLES_FOLDER;
+    }
+
+    return root_path + "/db" + TABLES_FOLDER;
+}
+
 int64_t
 GetMicroSecTimeStamp() {
     auto now = std::chrono::system_clock::now();

core/src/db/Utils.h

@@ -30,6 +30,9 @@ namespace utils {
 int64_t
 GetMicroSecTimeStamp();
 
+std::string
+ConstructCollectionRootPath(const std::string& root_path);
+
 Status
 CreateCollectionPath(const DBMetaOptions& options, const std::string& collection_id);
 Status

core/src/db/snapshot/Event.h

@@ -35,7 +35,7 @@ class ResourceGCEvent : public MetaEvent {
  public:
     using Ptr = std::shared_ptr<ResourceGCEvent>;
 
-    explicit ResourceGCEvent(class ResourceT::Ptr res) : res_(res) {
+    explicit ResourceGCEvent(const std::string& root_path, class ResourceT::Ptr res) : dir_root_(root_path), res_(res) {
     }
 
     ~ResourceGCEvent() = default;

core/src/db/snapshot/Operations.cpp

@@ -10,8 +10,12 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "db/snapshot/Operations.h"
+
 #include <chrono>
 #include <sstream>
+
+#include "config/Config.h"
+#include "db/Utils.h"
 #include "db/snapshot/Event.h"
 #include "db/snapshot/EventExecutor.h"
 #include "db/snapshot/OperationExecutor.h"
@@ -245,9 +249,15 @@ Operations::PostExecute(StorePtr store) {
 template <typename ResourceT>
 void
 ApplyRollBack(std::set<std::shared_ptr<ResourceContext<ResourceT>>>& step_context_set) {
+    auto& config = server::Config::GetInstance();
+    std::string path;
+    config.GetStorageConfigPath(path);
+    auto root_path = utils::ConstructCollectionRootPath(path);
+
     for (auto& step_context : step_context_set) {
         auto res = step_context->Resource();
-        auto evt_ptr = std::make_shared<ResourceGCEvent<ResourceT>>(res);
+
+        auto evt_ptr = std::make_shared<ResourceGCEvent<ResourceT>>(root_path, res);
         EventExecutor::GetInstance().Submit(evt_ptr);
         std::cout << "Rollback " << typeid(ResourceT).name() << ": " << res->GetID() << std::endl;
     }

core/src/db/snapshot/ResourceHelper.h

@@ -16,6 +16,7 @@
 
 #include "db/snapshot/Resources.h"
 #include "utils/Status.h"
+#include "utils/StringHelpFunctions.h"
 
 namespace milvus::engine::snapshot {
 
@@ -34,7 +35,10 @@ template <>
 inline std::string
 GetResPath<Collection>(const std::string& root, const Collection::Ptr& res_ptr) {
     std::stringstream ss;
-    ss << root << "/";
+    ss << root;
+    if (!StringHelpFunctions::EndWithSlash(root)) {
+        ss << "/";
+    }
     ss << COLLECTION_PREFIX << res_ptr->GetID();
 
     return ss.str();
@@ -44,7 +48,10 @@ template <>
 inline std::string
 GetResPath<Partition>(const std::string& root, const Partition::Ptr& res_ptr) {
     std::stringstream ss;
-    ss << root << "/";
+    ss << root;
+    if (!StringHelpFunctions::EndWithSlash(root)) {
+        ss << "/";
+    }
     ss << COLLECTION_PREFIX << res_ptr->GetCollectionId() << "/";
     ss << PARTITION_PREFIX << res_ptr->GetID();
 
@@ -55,7 +62,10 @@ template <>
 inline std::string
 GetResPath<Segment>(const std::string& root, const Segment::Ptr& res_ptr) {
     std::stringstream ss;
-    ss << root << "/";
+    ss << root;
+    if (!StringHelpFunctions::EndWithSlash(root)) {
+        ss << "/";
+    }
     ss << COLLECTION_PREFIX << res_ptr->GetCollectionId() << "/";
     ss << PARTITION_PREFIX << res_ptr->GetPartitionId() << "/";
     ss << SEGMENT_PREFIX << res_ptr->GetID();
@@ -67,7 +77,10 @@ template <>
 inline std::string
 GetResPath<SegmentFile>(const std::string& root, const SegmentFile::Ptr& res_ptr) {
     std::stringstream ss;
-    ss << root << "/";
+    ss << root;
+    if (!StringHelpFunctions::EndWithSlash(root)) {
+        ss << "/";
+    }
     ss << COLLECTION_PREFIX << res_ptr->GetCollectionId() << "/";
     ss << PARTITION_PREFIX << res_ptr->GetPartitionId() << "/";
     ss << SEGMENT_PREFIX << res_ptr->GetSegmentId() << "/";

core/src/db/snapshot/ResourceHolder.h

@@ -17,6 +17,8 @@
 #include <mutex>
 #include <string>
 #include <thread>
+
+#include "config/Config.h"
 #include "db/snapshot/Event.h"
 #include "db/snapshot/EventExecutor.h"
 #include "db/snapshot/Operations.h"
@@ -137,9 +139,14 @@ class ResourceHolder {
 
     virtual void
     OnNoRefCallBack(ResourcePtr resource) {
+        auto& config = server::Config::GetInstance();
+        std::string path;
+        config.GetStorageConfigPath(path);
+        auto root_path = utils::ConstructCollectionRootPath(path);
+
         resource->Deactivate();
         Release(resource->GetID());
-        auto evt_ptr = std::make_shared<ResourceGCEvent<ResourceT>>(resource);
+        auto evt_ptr = std::make_shared<ResourceGCEvent<ResourceT>>(root_path, resource);
         EventExecutor::GetInstance().Submit(evt_ptr);
     }
 

core/src/utils/StringHelpFunctions.cpp

@@ -160,4 +160,13 @@ StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) {
     return Status::OK();
 }
 
+bool
+StringHelpFunctions::EndWithSlash(const std::string& path) {
+    if (path.size() == 0) {
+        return false;
+    }
+
+    return *path.rbegin() == '/';
+}
+
 }  // namespace milvus

core/src/utils/StringHelpFunctions.h

@@ -69,6 +69,9 @@ class StringHelpFunctions {
     // "false", "off", "no", "0", "" ==> false
     static Status
     ConvertToBoolean(const std::string& str, bool& value);
+
+    static bool
+    EndWithSlash(const std::string& path);
 };
 
 }  // namespace milvus

core/unittest/server/test_util.cpp

@@ -788,6 +788,11 @@ TEST(ValidationUtilTest, VALIDATE_PATH_TEST) {
     ASSERT_FALSE(milvus::server::ValidateStoragePath("/tmp//milvus").ok());
 }
 
+TEST(UtilTest, ENDWITHSLASH_TEST) {
+    ASSERT_TRUE(milvus::StringHelpFunctions::EndWithSlash("/tmp/milvus/"));
+    ASSERT_FALSE(milvus::StringHelpFunctions::EndWithSlash("/tmp/milvus"));
+}
+
 TEST(UtilTest, ROLLOUTHANDLER_TEST) {
     std::string dir1 = "/tmp/milvus_test";
     std::string dir2 = "/tmp/milvus_test/log_test";