Add instance lock (#2060)

* Add instance lock
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update message
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update unittest CMakeLists.txt
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* fix clang-format
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* update
Signed-off-by: Nwxyu <xy.wang@zilliz.com>
Co-authored-by: NJin Hai <hai.jin@zilliz.com>

CHANGELOG.md

@@ -17,6 +17,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1962 Add api HasPartition
 -   \#1965 FAISS/NSG/HNSW/ANNOY use unified distance calculation algorithm
 -   \#2054 Check if CPU instruction sets are illegal
+-   \#2059 Add lock file avoid multiple instances modifying data at the same time
 -   \#2064 Warn when use SQLite as metadata management
 
 ## Improvement

core/conf/log_config.template

@@ -7,6 +7,9 @@
     SUBSECOND_PRECISION     =   3
     PERFORMANCE_TRACKING    =   false
     MAX_LOG_FILE_SIZE       =   209715200 ## Throw log files away after 200MB
+* INFO:
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-info.log"
+    ENABLED                 =   true
 * DEBUG:
     FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
     ENABLED                 =   true

core/src/CMakeLists.txt

@@ -85,6 +85,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/server/delivery delivery_files)
 set(server_files
         ${server_init_files}
         ${server_service_files}
+        ${server_init_files}
         ${delivery_request_files}
         ${delivery_hybrid_request_files}
         ${delivery_strategy_files}

core/src/main.cpp

@@ -76,6 +76,7 @@ main(int argc, char* argv[]) {
     std::string config_filename, log_config_file;
     std::string pid_filename;
     std::string app_name = argv[0];
+    milvus::Status s;
 
     milvus::server::Server& server = milvus::server::Server::GetInstance();
 
@@ -133,9 +134,11 @@ main(int argc, char* argv[]) {
 
     server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
 
-    if (server.Start().ok()) {
+    s = server.Start();
+    if (s.ok()) {
         std::cout << "Milvus server started successfully!" << std::endl;
     } else {
+        std::cout << s.message() << std::endl;
         goto FAIL;
     }
 

core/src/server/Server.cpp

@@ -10,9 +10,11 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "server/Server.h"
+#include "server/init/InstanceLockCheck.h"
 
 #include <fcntl.h>
 #include <unistd.h>
+#include <boost/filesystem.hpp>
 #include <cstring>
 
 #include "config/Config.h"
@@ -189,6 +191,41 @@ Server::Start() {
 
         InitLog(log_config_file_);
 
+        std::string deploy_mode;
+        s = config.GetServerConfigDeployMode(deploy_mode);
+        if (!s.ok()) {
+            return s;
+        }
+
+        if (deploy_mode == "single" || deploy_mode == "cluster_writable") {
+            std::string db_path;
+            s = config.GetStorageConfigPrimaryPath(db_path);
+            if (!s.ok()) {
+                return s;
+            }
+
+            s = InstanceLockCheck::Check(db_path);
+            if (!s.ok()) {
+                std::cerr << "deploy_mode: " << deploy_mode << " instance lock db path failed." << std::endl;
+                return s;
+            }
+
+            std::string wal_path;
+            s = config.GetWalConfigWalPath(wal_path);
+            if (!s.ok()) {
+                return s;
+            }
+
+            if (not boost::filesystem::create_directories(wal_path)) {
+                return Status(SERVER_UNEXPECTED_ERROR, "Cannot create wal dir");
+            }
+            s = InstanceLockCheck::Check(wal_path);
+            if (!s.ok()) {
+                std::cerr << "deploy_mode: " << deploy_mode << " instance lock wal path failed." << std::endl;
+                return s;
+            }
+        }
+
         // print version information
         LOG_SERVER_INFO_ << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME;
 #ifdef MILVUS_GPU_VERSION

core/src/server/init/InstanceLockCheck.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "server/init/InstanceLockCheck.h"
+#include "utils/Log.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace milvus {
+namespace server {
+
+Status
+InstanceLockCheck::Check(const std::string& path) {
+    std::string lock_path = path + "/lock";
+    auto fd = open(lock_path.c_str(), O_RDWR | O_CREAT | O_NOFOLLOW, 0640);
+    if (fd < 0) {
+        std::string msg;
+        if (errno == EROFS) {
+            // Not using locking for read-only lock file
+            msg += "Lock file is read-only.";
+        }
+        msg += "Could not open lock file.";
+        return Status(SERVER_UNEXPECTED_ERROR, msg);
+    }
+
+    // Acquire a write lock
+    struct flock fl;
+    // exclusive lock
+    fl.l_type = F_WRLCK;
+    fl.l_whence = SEEK_SET;
+    fl.l_start = 0;
+    fl.l_len = 0;
+    if (fcntl(fd, F_SETLK, &fl) == -1) {
+        std::string msg;
+        if (errno == EACCES || errno == EAGAIN) {
+            msg += "Permission denied. ";
+        } else if (errno == ENOLCK) {
+            // Not using locking for nfs mounted lock file
+            msg += "Using nfs. ";
+        }
+        close(fd);
+        msg += "Could not get lock.";
+        return Status(SERVER_UNEXPECTED_ERROR, msg);
+    }
+
+    LOG_SERVER_INFO_ << "InstanceLockCheck passed.";
+
+    return Status::OK();
+}
+
+}  // namespace server
+}  // namespace milvus

core/src/server/init/InstanceLockCheck.h

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <string>
+#include "utils/Status.h"
+
+namespace milvus {
+namespace server {
+
+class InstanceLockCheck {
+ public:
+    static Status
+    Check(const std::string& path);
+};  // InstanceLockCheck
+
+}  // namespace server
+}  // namespace milvus

core/unittest/server/CMakeLists.txt

@@ -40,6 +40,7 @@ set(grpc_service_files
 set(server_test_files
         ${common_files}
         ${server_files}
+        ${server_init_files}
         ${grpc_server_files}
         ${grpc_service_files}
         ${server_delivery_files}