Improve efficiency in Docker build stage (#2334)

Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

.github/workflows/core.yml

@@ -8,6 +8,7 @@ on:
       - 'ci/**'
       - 'core/**'
       - '.github/workflows/core.yml'
+      - docker-compose.yml
       - '!**.md'
       - '!ci/jenkins/**'
   pull_request:
@@ -16,6 +17,7 @@ on:
       - 'ci/**'
       - 'core/**'
       - '.github/workflows/core.yml'
+      - docker-compose.yml
       - '!**.md'
       - '!ci/jenkins/**'
 

ci/jenkins/Jenkinsfile

@@ -64,6 +64,8 @@ pipeline {
                         versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
                     ]);
                     DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+                    CHANGE_DOCKER_VERSION = "${CHANGE_TARGET}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+                    CACHA_DOCKER_VERSION = "${CHANGE_TARGET ? CHANGE_DOCKER_VERSION : DOCKER_VERSION}"
                 }
 
                 stages {
@@ -91,11 +93,7 @@ pipeline {
                                 steps {
                                     container("milvus-${BINARY_VERSION}-build-env") {
                                         script {
-                                            if ("${BINARY_VERSION}" == "gpu") {
-                                                load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
-                                            } else {
-                                                echo "Skip Unittest"
-                                            }
+                                            load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
                                         }
                                     }
                                 }
@@ -105,11 +103,7 @@ pipeline {
                                 steps {
                                     container("milvus-${BINARY_VERSION}-build-env") {
                                         script {
-                                            if ("${BINARY_VERSION}" == "gpu") {
-                                                load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
-                                            } else {
-                                                echo "Skip Code Coverage"
-                                            }
+                                            load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
                                         }
                                     }
                                 }
@@ -128,6 +122,13 @@ pipeline {
                     }
 
                     stage('Publish Docker images') {
+                        environment {
+                            SOURCE_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
+                            TARGET_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
+                            SOURCE_TAG = "${CACHA_DOCKER_VERSION}"
+                            TARGET_TAG = "${DOCKER_VERSION}"
+                        }
+
                         agent {
                             kubernetes {
                                 label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"

ci/jenkins/pod/docker-pod.yaml

@@ -7,7 +7,8 @@ metadata:
 spec:
   containers:
   - name: publish-images
-    image: registry.zilliz.com/library/docker:v1.0.0
+    image: registry.zilliz.com/library/docker:v1.1.0
+    imagePullPolicy: Always
     securityContext:
       privileged: true
     command:

ci/jenkins/step/publishImages.groovy

-dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") {
+dir ("docker/deploy") {
     def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
 
     withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
@@ -9,23 +9,27 @@ dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") {
         }
     }
     sh "tar zxvf ${binaryPackage}"
-    def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+    def sourceImage = "${params.DOKCER_REGISTRY_URL}/${PROJECT_NAME}/engine:${SOURCE_TAG}"
 
     try {
-        deleteImages("${imageName}", true)
-
-        def customImage = docker.build("${imageName}")
-
-        deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
-
-        docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") {
-            customImage.push()
+        sh(returnStatus: true, script: "docker pull ${sourceImage}")
+        sh "docker-compose build --force-rm ${BINARY_VERSION}_${OS_NAME}"
+        try {
+            withCredentials([usernamePassword(credentialsId: "${params.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
+                sh "docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${params.DOKCER_REGISTRY_URL}"
+                sh "docker-compose push ${BINARY_VERSION}_${OS_NAME}"
+            }
+        } catch (exc) {
+            throw exc
+        } finally {
+            sh "docker logout ${params.DOKCER_REGISTRY_URL}"
         }
     } catch (exc) {
         throw exc
     } finally {
-        deleteImages("${imageName}", true)
-        deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
+        deleteImages("${sourceImage}", true)
+        sh "docker-compose down --rmi all"
+        sh(returnStatus: true, script: "docker rmi -f \$(docker images | grep '<none>' | awk '{print \$3}')")
     }
 }
 
@@ -47,4 +51,3 @@ boolean deleteImages(String imageName, boolean force) {
     }
     return true
 }
-

docker-compose.yml

@@ -39,7 +39,7 @@ services:
     networks:
       - milvus
     command: &ubuntu-command
-      ["/milvus/ci/scripts/build.sh -t Release -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
+      ["/milvus/ci/scripts/build.sh -t Release -j4 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
         /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql &&
         /milvus/ci/scripts/coverage.sh"]
 
@@ -59,7 +59,7 @@ services:
     networks:
       - milvus
     command: &centos-command
-      ["/milvus/ci/scripts/build.sh -t Release -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
+      ["/milvus/ci/scripts/build.sh -t Release -j4 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
         /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql &&
         /milvus/ci/scripts/coverage.sh"]
 

docker/build_env/cpu/ubuntu16.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" = 'start' ]; then
-   tail -f /dev/null
-fi
-
-exec "$@"
-

docker/build_env/cpu/ubuntu18.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" = 'start' ]; then
-   tail -f /dev/null
-fi
-
-exec "$@"
-

docker/build_env/docker-compose.yaml

+version: '2.3'
+
+services:
+  cpu_centos7:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: cpu/centos7/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  gpu_centos7:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: gpu/centos7/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  cpu_ubuntu16.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: cpu/ubuntu16.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  gpu_ubuntu16.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: gpu/ubuntu16.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  cpu_ubuntu18.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: cpu/ubuntu18.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  gpu_ubuntu18.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: gpu/ubuntu18.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}

docker/build_env/gpu/centos7/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" = 'start' ]; then
-   tail -f /dev/null
-fi
-
-exec "$@"
-

docker/build_env/gpu/ubuntu16.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" = 'start' ]; then
-   tail -f /dev/null
-fi
-
-exec "$@"
-

docker/build_env/gpu/ubuntu18.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" = 'start' ]; then
-   tail -f /dev/null
-fi
-
-exec "$@"
-

docker/deploy/cpu/ubuntu16.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" == 'start' ]; then
-    cd /var/lib/milvus/scripts && ./start_server.sh
-fi
-
-exec "$@"

docker/deploy/cpu/ubuntu18.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" == 'start' ]; then
-    cd /var/lib/milvus/scripts && ./start_server.sh
-fi
-
-exec "$@"

docker/deploy/docker-compose.yaml

+version: '2.3'
+
+services:
+  cpu_centos7:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: cpu/centos7/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  gpu_centos7:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: gpu/centos7/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  cpu_ubuntu16.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: cpu/ubuntu16.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  gpu_ubuntu16.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: gpu/ubuntu16.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  cpu_ubuntu18.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: cpu/ubuntu18.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}
+  gpu_ubuntu18.04:
+    image: ${TARGET_REPO}:${TARGET_TAG}
+    build:
+      context: ./
+      dockerfile: gpu/ubuntu18.04/Dockerfile
+      cache_from:
+        - ${SOURCE_REPO}:${SOURCE_TAG}

docker/deploy/gpu/centos7/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" == 'start' ]; then
-    cd /var/lib/milvus/scripts && ./start_server.sh
-fi
-
-exec "$@"

docker/deploy/gpu/ubuntu16.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" == 'start' ]; then
-    cd /var/lib/milvus/scripts && ./start_server.sh
-fi
-
-exec "$@"

docker/deploy/gpu/ubuntu18.04/docker-entrypoint.sh

-#!/bin/bash
-
-set -e
-
-if [ "$1" == 'start' ]; then
-    cd /var/lib/milvus/scripts && ./start_server.sh
-fi
-
-exec "$@"