Remove unnecessary memcpy (#2337)

* fix #2264
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* typo
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* fix ut
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* typo
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* Remove unnecessary memcpy
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

core/src/db/insert/VectorSource.cpp

@@ -18,6 +18,7 @@
 #include "db/engine/ExecutionEngine.h"
 #include "metrics/Metrics.h"
 #include "utils/Log.h"
+#include "utils/TimeRecorder.h"
 
 namespace milvus {
 namespace engine {
@@ -36,9 +37,8 @@ VectorSource::VectorSource(milvus::engine::VectorsData vectors,
 }
 
 Status
-VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,
-                  const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add,
-                  size_t& num_vectors_added) {
+VectorSource::Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema,
+                  const size_t& num_vectors_to_add, size_t& num_vectors_added) {
     uint64_t n = vectors_.vector_count_;
     server::CollectAddMetrics metrics(n, table_file_schema.dimension_);
 
@@ -61,35 +61,17 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment:
 
     Status status;
     if (!vectors_.float_data_.empty()) {
-        /*
-        status = execution_engine->AddWithIds(
-            num_vectors_added, vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_,
-            vector_ids_to_add.data());
-        */
-        std::vector<uint8_t> vectors;
+        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert float data into segment";
         auto size = num_vectors_added * table_file_schema.dimension_ * sizeof(float);
-        vectors.resize(size);
-        memcpy(vectors.data(), vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_,
-               size);
-        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment";
-        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add);
-
+        float* ptr = vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_;
+        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, (uint8_t*)ptr, size, vector_ids_to_add);
     } else if (!vectors_.binary_data_.empty()) {
-        /*
-        status = execution_engine->AddWithIds(
-            num_vectors_added,
-            vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_),
-            vector_ids_to_add.data());
-        */
+        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert binary data into segment";
         std::vector<uint8_t> vectors;
         auto size = num_vectors_added * SingleVectorSize(table_file_schema.dimension_) * sizeof(uint8_t);
-        vectors.resize(size);
-        memcpy(
-            vectors.data(),
-            vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_),
-            size);
-        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment";
-        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add);
+        uint8_t* ptr =
+            vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_);
+        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, ptr, size, vector_ids_to_add);
     }
 
     // Clear vector data
@@ -175,6 +157,7 @@ VectorSource::SingleVectorSize(uint16_t dimension) {
 
     return 0;
 }
+
 size_t
 VectorSource::SingleEntitySize(uint16_t dimension) {
     // TODO(yukun) add entity type and size compute

core/src/db/insert/VectorSource.h

@@ -36,8 +36,8 @@ class VectorSource {
                  const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data);
 
     Status
-    Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,
-        const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added);
+    Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema,
+        const size_t& num_vectors_to_add, size_t& num_vectors_added);
 
     Status
     AddEntities(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& collection_file_schema,

core/src/db/merge/MergeLayeredStrategy.cpp

@@ -20,7 +20,7 @@
 namespace milvus {
 namespace engine {
 
-const int64_t FORCE_MERGE_THREASHOLD = 10;  // force merge files older this time(in second)
+const int64_t FORCE_MERGE_THREASHOLD = 30;  // force merge files older this time(in second)
 
 Status
 MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGroups& files_groups) {

core/src/segment/SegmentWriter.cpp

@@ -52,6 +52,16 @@ SegmentWriter::AddVectors(const std::string& name, const std::vector<uint8_t>& d
     return Status::OK();
 }
 
+Status
+SegmentWriter::AddVectors(const std::string& name, const uint8_t* data, uint64_t size,
+                          const std::vector<doc_id_t>& uids) {
+    segment_ptr_->vectors_ptr_->AddData(data, size);
+    segment_ptr_->vectors_ptr_->AddUids(uids);
+    segment_ptr_->vectors_ptr_->SetName(name);
+
+    return Status::OK();
+}
+
 Status
 SegmentWriter::AddAttrs(const std::string& name, const std::unordered_map<std::string, uint64_t>& attr_nbytes,
                         const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data,

core/src/segment/SegmentWriter.h

@@ -36,6 +36,9 @@ class SegmentWriter {
     Status
     AddVectors(const std::string& name, const std::vector<uint8_t>& data, const std::vector<doc_id_t>& uids);
 
+    Status
+    AddVectors(const std::string& name, const uint8_t* data, uint64_t size, const std::vector<doc_id_t>& uids);
+
     Status
     AddAttrs(const std::string& name, const std::unordered_map<std::string, uint64_t>& attr_nbytes,
              const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data, const std::vector<doc_id_t>& uids);

core/src/segment/Vectors.cpp

@@ -34,6 +34,13 @@ Vectors::AddData(const std::vector<uint8_t>& data) {
     data_.insert(data_.end(), std::make_move_iterator(data.begin()), std::make_move_iterator(data.end()));
 }
 
+void
+Vectors::AddData(const uint8_t* data, uint64_t size) {
+    int64_t old_size = data_.size();
+    data_.resize(data_.size() + size);
+    memcpy(data_.data() + old_size, data, size);
+}
+
 void
 Vectors::AddUids(const std::vector<doc_id_t>& uids) {
     uids_.reserve(uids_.size() + uids.size());

core/src/segment/Vectors.h

@@ -33,6 +33,9 @@ class Vectors {
     void
     AddData(const std::vector<uint8_t>& data);
 
+    void
+    AddData(const uint8_t* data, uint64_t size);
+
     void
     AddUids(const std::vector<doc_id_t>& uids);