Refactor CI workflow (#1782)

* Set helm deploy timeout to 600s

* Refactor CI workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Refactor CI workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Refactor CI workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Refactor CI workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

ci/jenkins/Jenkinsfile

@@ -34,7 +34,7 @@ pipeline {
     }
 
     stages {
-        stage ('Milvus Build and Test') {
+        stage ('Milvus CI') {
             matrix {
                 agent none
                 axes {
@@ -53,148 +53,133 @@ pipeline {
                         values 'gpu', 'cpu'
                     }
                 }
+                environment {
+                    PACKAGE_VERSION = VersionNumber([
+                        versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
+                    ]);
+                    DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+                }
 
                 stages {
-                    stage("Milvus CI") {
-                        environment {
-                            PACKAGE_VERSION = VersionNumber([
-                                versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
-                            ]);
-                            DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+                    stage("Milvus Build and Unittest") {
+                        agent {
+                            kubernetes {
+                                label "${OS_NAME}-${BINARY_VERSION}-build-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
+                                defaultContainer 'jnlp'
+                                yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
+                            }
                         }
 
                         stages {
-                            stage('Run Build') {
-                                agent {
-                                    kubernetes {
-                                        label "${OS_NAME}-${BINARY_VERSION}-build-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
-                                        defaultContainer 'jnlp'
-                                        yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
-                                    }
-                                }
-
-                                stages {
-                                    stage('Build') {
-                                        steps {
-                                            container("milvus-${BINARY_VERSION}-build-env") {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
-                                                }
-                                            }
+                            stage('Build') {
+                                steps {
+                                    container("milvus-${BINARY_VERSION}-build-env") {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
                                         }
                                     }
+                                }
+                            }
 
-                                    stage('Unittest') {
-                                        steps {
-                                            container("milvus-${BINARY_VERSION}-build-env") {
-                                                script {
-                                                    if ("${BINARY_VERSION}" == "gpu") {
-                                                        load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
-                                                    } else {
-                                                        echo "Skip Unittest"
-                                                    }
-                                                }
+                            stage('Unittest') {
+                                steps {
+                                    container("milvus-${BINARY_VERSION}-build-env") {
+                                        script {
+                                            if ("${BINARY_VERSION}" == "gpu") {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
+                                            } else {
+                                                echo "Skip Unittest"
                                             }
                                         }
                                     }
+                                }
+                            }
 
-                                    stage('Code Coverage') {
-                                        steps {
-                                            container("milvus-${BINARY_VERSION}-build-env") {
-                                                script {
-                                                    if ("${BINARY_VERSION}" == "gpu") {
-                                                        load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
-                                                    } else {
-                                                        echo "Skip Code Coverage"
-                                                    }
-                                                }
+                            stage('Code Coverage') {
+                                steps {
+                                    container("milvus-${BINARY_VERSION}-build-env") {
+                                        script {
+                                            if ("${BINARY_VERSION}" == "gpu") {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
+                                            } else {
+                                                echo "Skip Code Coverage"
                                             }
                                         }
                                     }
+                                }
+                            }
 
-                                    stage('Upload Package') {
-                                        steps {
-                                            container("milvus-${BINARY_VERSION}-build-env") {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
-                                                }
-                                            }
+                            stage('Upload Package') {
+                                steps {
+                                    container("milvus-${BINARY_VERSION}-build-env") {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
                                         }
                                     }
                                 }
                             }
+                        }
+                    }
 
-                            stage('Publish docker images') {
-                                agent {
-                                    kubernetes {
-                                        label "${OS_NAME}-${BINARY_VERSION}-publish-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
-                                        defaultContainer 'jnlp'
-                                        yamlFile 'ci/jenkins/pod/docker-pod.yaml'
-                                    }
-                                }
+                    stage('Publish Docker images') {
+                        agent {
+                            kubernetes {
+                                label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
+                                defaultContainer 'jnlp'
+                                yamlFile 'ci/jenkins/pod/docker-pod.yaml'
+                            }
+                        }
 
-                                stages {
-                                    stage('Publish') {
-                                        steps {
-                                            container('publish-images') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
-                                                }
-                                            }
+                        stages {
+                            stage('Publish') {
+                                steps {
+                                    container('publish-images') {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
                                         }
                                     }
                                 }
                             }
+                        }
+                    }
 
-                            stage('Deploy to Development') {
-                                environment {
-                                    FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
-                                    FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
-                                    HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
-                                }
-
-                                agent {
-                                    kubernetes {
-                                        label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
-                                        defaultContainer 'jnlp'
-                                        yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
-                                    }
-                                }
+                    stage('Deploy to Development') {
+                        environment {
+                            FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
+                            FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
+                            HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
+                        }
 
-                                stages {
-                                    stage('Deploy to Dev') {
-                                        steps {
-                                            container('milvus-test-env') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
+                        agent {
+                            kubernetes {
+                                label "${OS_NAME}-${BINARY_VERSION}-dev-test-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
+                                defaultContainer 'jnlp'
+                                yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
+                            }
+                        }
 
-                                    stage('Dev Test') {
-                                        steps {
-                                            container('milvus-test-env') {
-                                                script {
-                                                    boolean isNightlyTest = isTimeTriggeredBuild()
-                                                    if (isNightlyTest) {
-                                                        load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
-                                                    } else {
-                                                        load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
-                                                    }
-                                                }
+                        stages {
+                            stage('Dev Test') {
+                                steps {
+                                    container('milvus-test-env') {
+                                        script {
+                                            boolean isNightlyTest = isTimeTriggeredBuild()
+                                            if (isNightlyTest) {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
+                                            } else {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
                                             }
                                         }
                                     }
                                 }
+                            }
+                        }
 
-                                post {
-                                    cleanup {
-                                        container('milvus-test-env') {
-                                            script {
-                                                load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
-                                            }
-                                        }
+                        post {
+                            cleanup {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
                                     }
                                 }
                             }

ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml

@@ -36,6 +36,13 @@ spec:
     ports:
     - containerPort: 3306
       name: mysql
+    resources:
+      limits:
+        memory: "500Mi"
+        cpu: "0.5"
+      requests:
+        memory: "200Mi"
+        cpu: "0.2"
   tolerations:
   - key: dedicated
     operator: Equal

ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml

@@ -36,6 +36,13 @@ spec:
     ports:
     - containerPort: 3306
       name: mysql
+    resources:
+      limits:
+        memory: "500Mi"
+        cpu: "0.5"
+      requests:
+        memory: "200Mi"
+        cpu: "0.2"
   tolerations:
   - key: dedicated
     operator: Equal

ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml

@@ -38,6 +38,13 @@ spec:
     ports:
     - containerPort: 3306
       name: mysql
+    resources:
+      limits:
+        memory: "500Mi"
+        cpu: "0.5"
+      requests:
+        memory: "200Mi"
+        cpu: "0.2"
   tolerations:
   - key: dedicated
     operator: Equal

ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml

@@ -38,6 +38,13 @@ spec:
     ports:
     - containerPort: 3306
       name: mysql
+    resources:
+      limits:
+        memory: "500Mi"
+        cpu: "0.5"
+      requests:
+        memory: "200Mi"
+        cpu: "0.2"
   tolerations:
   - key: dedicated
     operator: Equal

ci/jenkins/step/deploySingle2Dev.groovy

-sh 'helm version'
-sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-sh 'helm repo update'
-dir ('milvus-helm') {
-    checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
-    sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
-}

ci/jenkins/step/singleDevNightlyTest.groovy

+dir ('milvus-helm') {
+	sh 'helm version'
+	sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+	sh 'helm repo update'
+    checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
+    sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
+}
+
 dir ("tests/milvus_python_test") {
     // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
     sh 'python3 -m pip install -r requirements.txt'
-    sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
+    sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
 }
-// mysql database backend test
+// sqlite database backend test
 load "ci/jenkins/step/cleanupSingleDev.groovy"
 
 if (!fileExists('milvus-helm')) {
@@ -12,8 +20,8 @@ if (!fileExists('milvus-helm')) {
     }
 }
 dir ("milvus-helm") {
-    sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
+    sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
 }
 dir ("tests/milvus_python_test") {
-    sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
+    sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
 }

ci/jenkins/step/singleDevTest.groovy

+dir ('milvus-helm') {
+	sh 'helm version'
+	sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+	sh 'helm repo update'
+    checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
+    sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
+}
+
 dir ("tests/milvus_python_test") {
     // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
     sh 'python3 -m pip install -r requirements.txt'
-    sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
+    sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
 }