add close cases (#2339)

* add close cases
Signed-off-by: Nzw <zw@milvus.io>

* fix case bug
Signed-off-by: Nzw <zw@milvus.io>

* reset case level
Signed-off-by: Nzw <zw@milvus.io>

* reset case level
Signed-off-by: Nzw <zw@milvus.io>

* fix case
Signed-off-by: Nzw <zw@milvus.io>
Co-authored-by: Nzw <zw@milvus.io>

tests/milvus_python_test/conftest.py

@@ -55,17 +55,17 @@ def connect(request):
     return milvus
 
 
-# @pytest.fixture(scope="module")
-# def dis_connect(request):
-#     ip = request.config.getoption("--ip")
-#     port = request.config.getoption("--port")
-#     http_port = request.config.getoption("--http-port")
-#     handler = request.config.getoption("--handler")
-#     if handler == "HTTP":
-#         port = http_port
-#     milvus = get_milvus(host=ip, port=port, handler=handler)
-#     milvus.disconnect()
-#     return milvus
+@pytest.fixture(scope="module")
+def dis_connect(request):
+    ip = request.config.getoption("--ip")
+    port = request.config.getoption("--port")
+    http_port = request.config.getoption("--http-port")
+    handler = request.config.getoption("--handler")
+    if handler == "HTTP":
+        port = http_port
+    milvus = get_milvus(host=ip, port=port, handler=handler)
+    milvus.close()
+    return milvus
 
 
 @pytest.fixture(scope="module")
@@ -108,9 +108,10 @@ def collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
 
@@ -134,9 +135,10 @@ def ip_collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
 
@@ -160,9 +162,10 @@ def jac_collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
 
@@ -185,9 +188,10 @@ def ham_collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
 
@@ -210,9 +214,10 @@ def tanimoto_collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
     return collection_name
@@ -234,9 +239,10 @@ def substructure_collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
     return collection_name
@@ -258,9 +264,10 @@ def superstructure_collection(request, connect):
         pytest.exit("collection can not be created, exit pytest ...")
 
     def teardown():
-        status, collection_names = connect.list_collections()
-        for collection_name in collection_names:
-            connect.drop_collection(collection_name)
+        # status, collection_names = connect.list_collections()
+        # for collection_name in collection_names:
+        #     connect.drop_collection(collection_name)
+        connect.drop_collection(collection_name)
 
     request.addfinalizer(teardown)
     return collection_name

tests/milvus_python_test/test_add_vectors.py

@@ -521,17 +521,17 @@ class TestAddBase:
             assert status.OK()
             assert len(ids) == nq
 
-    # @pytest.mark.level(2)
-    # def test_insert_without_connect(self, dis_connect, collection):
-    #     '''
-    #     target: test add vectors without connection
-    #     method: create collection and add vectors in it, check if added successfully
-    #     expected: raise exception
-    #     '''
-    #     nq = 5
-    #     vectors = gen_vectors(nq, dim)
-    #     with pytest.raises(Exception) as e:
-    #         status, ids = dis_connect.insert(collection, vectors)
+    @pytest.mark.level(2)
+    def test_insert_without_connect(self, dis_connect, collection):
+        '''
+        target: test add vectors without connection
+        method: create collection and add vectors in it, check if added successfully
+        expected: raise exception
+        '''
+        nq = 5
+        vectors = gen_vectors(nq, dim)
+        with pytest.raises(Exception) as e:
+            status, ids = dis_connect.insert(collection, vectors)
 
     def test_add_collection_not_existed(self, connect):
         '''
@@ -580,44 +580,6 @@ class TestAddBase:
         assert status.OK()
         assert len(result) == 1
 
-    # TODO: enable
-    # @pytest.mark.repeat(10)
-    @pytest.mark.timeout(ADD_TIMEOUT)
-    def _test_add_vector_with_multiprocessing(self, args):
-        '''
-        target: test add vectors, with multi processes
-        method: 10 processed add vectors concurrently
-        expected: status ok and result length is equal to the length off added vectors
-        '''
-        collection = gen_unique_str()
-        param = {'collection_name': collection,
-                 'dimension': dim,
-                 'index_file_size': index_file_size,
-                 'metric_type': MetricType.L2}
-        milvus = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"])
-        milvus.create_collection(param)
-        vector = gen_single_vector(dim)
-        process_num = 4
-        loop_num = 5
-        processes = []
-        def add():
-            milvus = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"])
-            i = 0
-            while i < loop_num:
-                status, ids = milvus.insert(collection, vector)
-                i = i + 1
-            # milvus.disconnect()
-        for i in range(process_num):
-            p = Process(target=add, args=())
-            processes.append(p)
-            p.start()
-            time.sleep(0.2)
-        for p in processes:
-            p.join()
-        time.sleep(2)
-        status, count = milvus.count_entities(collection)
-        assert count == process_num * loop_num
-
     @pytest.mark.level(2)
     @pytest.mark.timeout(30)
     def test_collection_add_rows_count_multi_threading(self, args):
@@ -642,6 +604,7 @@ class TestAddBase:
         def add(thread_i):
             logging.getLogger().info("In thread-%d" % thread_i)
             milvus = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"])
+            assert milvus
             status, result = milvus.insert(collection, records=vectors)
             assert status.OK()
             status = milvus.flush([collection])

tests/milvus_python_test/test_connect.py

@@ -2,6 +2,7 @@ import pytest
 import pdb
 import threading
 from multiprocessing import Process
+import concurrent.futures
 from utils import *
 
 CONNECT_TIMEOUT = 12
@@ -18,30 +19,24 @@ class TestConnect:
         else:
             return False
 
-    # def test_disconnect(self, connect):
-    #     '''
-    #     target: test disconnect
-    #     method: disconnect a connected client
-    #     expected: connect failed after disconnected
-    #     '''
-    #     res = connect.disconnect()
-    #     assert res.OK()
-    #     with pytest.raises(Exception) as e:
-    #         res = connect.server_version()
+    def test_disconnect(self, connect):
+        '''
+        target: test disconnect
+        method: disconnect a connected client
+        expected: connect failed after disconnected
+        '''
+        res = connect.close()
+        with pytest.raises(Exception) as e:
+            res = connect.server_version()
 
-    # def test_disconnect_repeatedly(self, connect, args):
-    #     '''
-    #     target: test disconnect repeatedly
-    #     method: disconnect a connected client, disconnect again
-    #     expected: raise an error after disconnected
-    #     '''
-    #     if not connect.connected():
-    #         with pytest.raises(Exception) as e:
-    #             connect.disconnect()
-    #     else:
-    #         connect.disconnect()
-    #         with pytest.raises(Exception) as e:
-    #             connect.disconnect()
+    def test_disconnect_repeatedly(self, dis_connect, args):
+        '''
+        target: test disconnect repeatedly
+        method: disconnect a connected client, disconnect again
+        expected: raise an error after disconnected
+        '''
+        with pytest.raises(Exception) as e:
+            connect.close()
 
     def test_connect_correct_ip_port(self, args):
         '''
@@ -111,30 +106,21 @@ class TestConnect:
                 milvus = get_milvus(None, None, uri=uri_value, handler=args["handler"])
                 # assert not milvus.connected()
 
-    # disable
-    # def _test_connect_with_multiprocess(self, args):
-    #     '''
-    #     target: test uri connect with multiprocess
-    #     method: set correct uri, test with multiprocessing connecting
-    #     expected: all connection is connected        
-    #     '''
-    #     uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
-    #     process_num = 10
-    #     processes = []
-
-    #     def connect(milvus):
-    #         milvus.connect(uri=uri_value)
-    #         with pytest.raises(Exception) as e:
-    #             milvus.connect(uri=uri_value)
-    #         # assert milvus.connected()
-
-    #     for i in range(process_num):
-    #         milvus = get_milvus(args["ip"], args["port"], args["handler"])
-    #         p = Process(target=connect, args=(milvus, ))
-    #         processes.append(p)
-    #         p.start()
-    #     for p in processes:
-    #         p.join()
+    def test_connect_with_multiprocess(self, args):
+        '''
+        target: test uri connect with multiprocess
+        method: set correct uri, test with multiprocessing connecting
+        expected: all connection is connected        
+        '''
+        processes = []
+        def connect():
+            milvus = get_milvus(args["ip"], args["port"], handler=args["handler"])
+            assert milvus           
+        with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
+            future_results = {executor.submit(
+                connect): i for i in range(100)}
+            for future in concurrent.futures.as_completed(future_results):
+                future.result()
 
     def test_connect_repeatedly(self, args):
         '''
@@ -149,57 +135,6 @@ class TestConnect:
         milvus = Milvus(uri=uri_value, handler=args["handler"])
         # assert milvus.connected()
 
-    # def test_connect_disconnect_repeatedly_times(self, args):
-    #     '''
-    #     target: test connect and disconnect for 10 times repeatedly
-    #     method: disconnect, and then connect, assert connect status
-    #     expected: status.code is 0
-    #     '''
-    #     times = 10
-    #     for i in range(times):
-    #         milvus = get_milvus(args["ip"], args["port"], handler=args["handler"])
-    #         milvus.disconnect()
-    #         assert not milvus.connected()
-
-    # TODO: enable
-    # def _test_connect_disconnect_with_multiprocess(self, args):
-    #     '''
-    #     target: test uri connect and disconnect repeatly with multiprocess
-    #     method: set correct uri, test with multiprocessing connecting and disconnecting
-    #     expected: all connection is connected after 10 times operation       
-    #     '''
-    #     uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
-    #     process_num = 4
-    #     processes = []
-
-    #     def connect(milvus):
-    #         milvus.connect(uri=uri_value)
-    #         milvus.disconnect()
-    #         milvus.connect(uri=uri_value)
-    #         assert milvus.connected()
-
-    #     for i in range(process_num):
-    #         milvus = get_milvus(args["ip"], args["port"], args["handler"])
-    #         p = Process(target=connect, args=(milvus, ))
-    #         processes.append(p)
-    #         p.start()
-    #     for p in processes:
-    #         p.join()
-
-    # Disable, (issue: https://github.com/milvus-io/milvus/issues/288)
-    # def _test_connect_param_priority_both_hostip_uri(self, args):
-    #     '''
-    #     target: both host_ip_port / uri are both given, and not null, use the uri params
-    #     method: check if wrong uri connection is ok
-    #     expected: connect raise an exception and connected is false
-    #     '''
-    #     milvus = get_milvus(args["ip"], args["port"], args["handler"])
-    #     uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
-    #     with pytest.raises(Exception) as e:
-    #         res = milvus.connect(host=args["ip"], port=39540, uri=uri_value, timeout=1)
-    #         logging.getLogger().info(res)
-    #     # assert not milvus.connected()
-
     def _test_add_vector_and_disconnect_concurrently(self):
         '''
         Target: test disconnect in the middle of add vectors

tests/milvus_python_test/test_get_vectors_by_ids.py

@@ -4,6 +4,7 @@ import pdb
 import threading
 import logging
 from multiprocessing import Pool, Process
+import concurrent.futures
 import pytest
 from milvus import IndexType, MetricType
 from utils import *
@@ -190,6 +191,40 @@ class TestGetBase:
         status, res = connect.get_entity_by_id(collection_new, [1]) 
         assert not status.OK()
 
+    @pytest.mark.timeout(60)
+    def test_get_vector_by_id_multithreads(self, connect, collection):
+        vectors = gen_vectors(nb, dim)
+        status, ids = connect.insert(collection, vectors)
+        status = connect.flush([collection])
+        assert status.OK()
+        get_id = ids[100:200]
+        def get():
+            status, res = connect.get_entity_by_id(collection, get_id)
+            assert status.OK()
+            assert len(res) == len(get_id)
+            for i in range(len(res)):
+                assert_equal_vector(res[i], vectors[100+i])
+        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+            future_results = {executor.submit(
+                get): i for i in range(10)}
+            for future in concurrent.futures.as_completed(future_results):
+                future.result()
+
+    # TODO: autoflush
+    def _test_get_vector_by_id_after_delete_no_flush(self, connect, collection):
+        vectors = gen_vectors(nb, dim)
+        status, ids = connect.insert(collection, vectors)
+        status = connect.flush([collection])
+        assert status.OK()
+        get_id = ids[100:200]
+        status = connect.delete_entity_by_id(collection, get_id)
+        assert status.OK()
+        status, res = connect.get_entity_by_id(collection, get_id)
+        assert status.OK()
+        assert len(res) == len(get_id)
+        for i in range(len(res)):
+            assert_equal_vector(res[i], vectors[100+i])
+
 
 class TestGetIndexedVectors:
     """

tests/milvus_python_test/test_ping.py

@@ -23,27 +23,27 @@ class TestPing:
         status, msg = connect.server_status()
         assert status.OK()
 
-    def _test_server_cmd_with_params_version(self, connect):
+    def test_server_cmd_with_params_version(self, connect):
         '''
         target: test cmd: version
         method: cmd = "version" ...
         expected: when cmd = 'version', return version of server;
         '''
         cmd = "version"
-        status, msg = connect.cmd(cmd)
+        status, msg = connect._cmd(cmd)
         logging.getLogger().info(status)
         logging.getLogger().info(msg)
         assert status.OK()
         assert msg == __version__
 
-    def _test_server_cmd_with_params_others(self, connect):
+    def test_server_cmd_with_params_others(self, connect):
         '''
         target: test cmd: lalala
         method: cmd = "lalala" ...
         expected: when cmd = 'version', return version of server;
         '''
         cmd = "rm -rf test"
-        status, msg = connect.cmd(cmd)
+        status, msg = connect._cmd(cmd)
         logging.getLogger().info(status)
         logging.getLogger().info(msg)
         assert status.OK()
@@ -54,25 +54,25 @@ class TestPing:
         assert connect
 
 
-# class TestPingDisconnect:
-#     def test_server_version(self, dis_connect):
-#         '''
-#         target: test get the server version, after disconnect
-#         method: call the server_version method after connected
-#         expected: version should not be the pymilvus version
-#         '''
-#         res = None
-#         with pytest.raises(Exception) as e:
-#             status, res = connect.server_version()
-#         assert res is None
-# 
-#     def test_server_status(self, dis_connect):
-#         '''
-#         target: test get the server status, after disconnect
-#         method: call the server_status method after connected
-#         expected: status returned should be not ok
-#         '''
-#         status = None
-#         with pytest.raises(Exception) as e:
-#             status, msg = connect.server_status()
-#         assert status is None
+class TestPingDisconnect:
+    def test_server_version(self, dis_connect):
+        '''
+        target: test get the server version, after disconnect
+        method: call the server_version method after connected
+        expected: version should not be the pymilvus version
+        '''
+        res = None
+        with pytest.raises(Exception) as e:
+            status, res = connect.server_version()
+        assert res is None
+
+    def test_server_status(self, dis_connect):
+        '''
+        target: test get the server status, after disconnect
+        method: call the server_status method after connected
+        expected: status returned should be not ok
+        '''
+        status = None
+        with pytest.raises(Exception) as e:
+            status, msg = connect.server_status()
+        assert status is None

tests/milvus_python_test/test_search_vectors.py

@@ -44,7 +44,7 @@ class TestSearchBase:
         else:
             status, ids = connect.insert(collection, add_vectors, partition_tag=partition_tags)
             assert status.OK()
-        sleep(add_interval_time)
+        connect.flush([collection])
         return add_vectors, ids
 
     def init_binary_data(self, connect, collection, nb=6000, insert=True, partition_tags=None):
@@ -316,6 +316,7 @@ class TestSearchBase:
         assert check_result(result[0], ids[0])
         assert result[0][0].distance <= epsilon
 
+    @pytest.mark.level(2)
     def test_search_l2_index_params_partition_D(self, connect, collection, get_simple_index):
         '''
         target: test basic search fuction, all the search params is corrent, test all index params, and build
@@ -335,6 +336,7 @@ class TestSearchBase:
         logging.getLogger().info(result)
         assert not status.OK()
 
+    @pytest.mark.level(2)
     def test_search_l2_index_params_partition_E(self, connect, collection, get_simple_index):
         '''
         target: test basic search fuction, all the search params is corrent, test all index params, and build
@@ -402,6 +404,7 @@ class TestSearchBase:
         assert result[0][0].distance <= epsilon
         assert result[1][0].distance <= epsilon
 
+    @pytest.mark.level(2)
     def test_search_ip_index_params(self, connect, ip_collection, get_simple_index):
         '''
         target: test basic search fuction, all the search params is corrent, test all index params, and build
@@ -451,6 +454,7 @@ class TestSearchBase:
         assert check_result(result[0], ids[0])
         assert result[0][0].distance >= 1 - gen_inaccuracy(result[0][0].distance)
 
+    @pytest.mark.level(2)
     def test_search_ip_index_params_partition(self, connect, ip_collection, get_simple_index):
         '''
         target: test basic search fuction, all the search params is corrent, test all index params, and build
@@ -480,6 +484,7 @@ class TestSearchBase:
         assert status.OK()
         assert len(result) == 0
 
+    @pytest.mark.level(2)
     def test_search_ip_index_params_partition_A(self, connect, ip_collection, get_simple_index):
         '''
         target: test basic search fuction, all the search params is corrent, test all index params, and build
@@ -505,17 +510,17 @@ class TestSearchBase:
         assert check_result(result[0], ids[0])
         assert result[0][0].distance >= 1 - gen_inaccuracy(result[0][0].distance)
 
-    # @pytest.mark.level(2)
-    # def test_search_vectors_without_connect(self, dis_connect, collection):
-    #     '''
-    #     target: test search vectors without connection
-    #     method: use dis connected instance, call search method and check if search successfully
-    #     expected: raise exception
-    #     '''
-    #     query_vectors = [vectors[0]]
-    #     nprobe = 1
-    #     with pytest.raises(Exception) as e:
-    #         status, ids = dis_connect.search(collection, top_k, query_vectors)
+    @pytest.mark.level(2)
+    def test_search_vectors_without_connect(self, dis_connect, collection):
+        '''
+        target: test search vectors without connection
+        method: use dis connected instance, call search method and check if search successfully
+        expected: raise exception
+        '''
+        query_vectors = [vectors[0]]
+        nprobe = 1
+        with pytest.raises(Exception) as e:
+            status, ids = dis_connect.search(collection, top_k, query_vectors)
 
     def test_search_collection_name_not_existed(self, connect, collection):
         '''