From bbbe18cf09801bf3cf804b946bf75ee7618a08d9 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sat, 31 Jul 2021 23:59:51 +0800 Subject: [PATCH] [1.3.7-prepare] Fix and Impove docker and k8s (#5898) * [Improvement][Docker/K8s] Support RESOURCE_MANAGER_HTTPADDRESS_PORT * [Fix-5719][K8s] Fix Ingress tls: got map expected array On TLS enabled On Kubernetes * [Improvement][Install] Fix workflow ut * [Improvement][K8s] Optimize template yaml of master and worker * [Improvement][Docker] Optimize docker-swarm/check script * [Docker/K8s] Specify version for all images * [Improvement][Docker] Optimize PS1 and WORKDIR * [Fix-5523][Docker/K8x] Fix unable to receive enterprise wechat alarm normally in docker and k8s * [Improvement-5858][Docker] Improve docker image support multi-arch like arm64 in docker-compose * [Improvement][DB] Update default username and password of database * [Improvement][Docker] Optimize PS1 * [Fix-5431][K8s] Fix master and worker cannot get the right address with custom DNS --- .github/workflows/ci_ut.yml | 7 +-- docker/build/Dockerfile | 9 ++-- .../dolphinscheduler/alert.properties.tpl | 8 ++-- .../dolphinscheduler/common.properties.tpl | 3 ++ .../datasource.properties.tpl | 14 ++---- docker/build/hooks/build | 16 ++++++- docker/build/startup-init-conf.sh | 1 + docker/docker-swarm/check | 43 +++++++++++-------- docker/docker-swarm/config.env.sh | 1 + docker/docker-swarm/docker-compose.yml | 24 +++++------ docker/docker-swarm/docker-stack.yml | 24 +++++------ docker/kubernetes/dolphinscheduler/Chart.yaml | 8 ++-- .../dolphinscheduler/templates/_helpers.tpl | 2 +- .../dolphinscheduler/templates/ingress.yaml | 4 +- .../statefulset-dolphinscheduler-master.yaml | 9 ++-- .../statefulset-dolphinscheduler-worker.yaml | 6 +-- .../kubernetes/dolphinscheduler/values.yaml | 3 +- .../src/main/resources/common.properties | 5 ++- .../src/main/resources/datasource.properties | 12 +++--- 19 files changed, 110 insertions(+), 89 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index c9c027ffd..ccb1ceed4 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -47,10 +47,11 @@ jobs: ${{ runner.os }}-maven- - name: Bootstrap database run: | - sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: postgres/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql - until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "wait for postgresql ..."; sleep 1; done - docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql bitnami/postgresql:latest bash -c "PGPASSWORD=test psql -h docker-swarm_dolphinscheduler-postgresql_1 -U test -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" + until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "waiting for postgresql ready ..."; sleep 1; done + docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql postgres:11.12 bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 1d5861db1..56e74bba2 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -23,6 +23,7 @@ ARG DEBIAN_FRONTEND=noninteractive ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DOCKER true +ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler # 1. install command/library/software # If install slowly, you can replcae debian's mirror with new mirror, Example: @@ -41,9 +42,8 @@ RUN apt-get update && \ # 2. add dolphinscheduler ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/ -RUN ln -s /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -WORKDIR ${DOLPHINSCHEDULER_HOME} +RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler +WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin # 3. add configuration and modify permissions and set soft links COPY ./checkpoint.sh /root/checkpoint.sh @@ -59,9 +59,10 @@ RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ - rm -rf /bin/sh && \ + rm -f /bin/sh && \ ln -s /bin/bash /bin/sh && \ mkdir -p /tmp/xls && \ + echo PS1=\"\\w \\$ \" >> ~/.bashrc && \ echo "Set disable_coredump false" >> /etc/sudo.conf # 4. expose port diff --git a/docker/build/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl index b47952163..6ce3a8ae0 100644 --- a/docker/build/conf/dolphinscheduler/alert.properties.tpl +++ b/docker/build/conf/dolphinscheduler/alert.properties.tpl @@ -40,9 +40,9 @@ enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID} enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET} enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID} enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS} -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} +enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=\$corpId&corpsecret=\$secret +enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=\$token +enterprise.wechat.team.send.msg={\"toparty\":\"\$toParty\",\"agentid\":\"\$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"\$msg\"},\"safe\":\"0\"} +enterprise.wechat.user.send.msg={\"touser\":\"\$toUser\",\"agentid\":\"\$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"\$msg\"}} plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index 84a449657..d1ba621bf 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -57,6 +57,9 @@ fs.s3a.access.key=${FS_S3A_ACCESS_KEY} # if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=${FS_S3A_SECRET_KEY} +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT} + # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS} diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index b414d2143..9177732ba 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -15,23 +15,17 @@ # limitations under the License. # -# db +# datasource configuration spring.datasource.driver-class-name=${DATABASE_DRIVER} spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}${DATABASE_PARAMS:+?${DATABASE_PARAMS}} spring.datasource.username=${DATABASE_USERNAME} spring.datasource.password=${DATABASE_PASSWORD} -# postgresql -#spring.datasource.driver-class-name=org.postgresql.Driver -#spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler -#spring.datasource.username=test -#spring.datasource.password=test - -# mysql +# mysql example #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.username=xxxx -#spring.datasource.password=xxxx +#spring.datasource.username=ds_user +#spring.datasource.password=dolphinscheduler # connection configuration #spring.datasource.initialSize=5 diff --git a/docker/build/hooks/build b/docker/build/hooks/build index 0590761ee..ef89aaeab 100755 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -47,7 +47,21 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSIO mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/ # docker build -BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" +if [ "$1" = "x" ] +then + # build multi-arch images with buildx + if !(docker buildx use dolphinscheduler &> /dev/null) + then + docker buildx create --use --name dolphinscheduler + fi + docker buildx inspect --bootstrap + echo "" + BUILD_PLATFORM=linux/amd64,linux/arm64 + BUILD_COMMAND="docker buildx build --platform $BUILD_PLATFORM --push --build-arg VERSION=$VERSION -t $DOCKER_REPO:$VERSION $(pwd)/docker/build/" +else + BUILD_COMMAND="docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$VERSION $(pwd)/docker/build/" +fi + echo -e "$BUILD_COMMAND\n" if (docker info 2> /dev/null | grep -i "ERROR"); then sudo $BUILD_COMMAND diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 0a2163339..e3c3a42ed 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -59,6 +59,7 @@ export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"} export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"} export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"} +export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"} export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""} export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:8088/ws/v1/cluster/apps/%s"} # skywalking diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check index f4093664d..66484ae0c 100755 --- a/docker/docker-swarm/check +++ b/docker/docker-swarm/check @@ -15,21 +15,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # -echo "------ dolphinscheduler check - server - status -------" -sleep 60 -server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -if [ $server_num -eq 5 ] -then - echo "Server all start successfully" -else - echo "Server start failed "$server_num - exit 1 -fi -ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` -if [ $ready -eq 1 ] -then - echo "Servers is ready" -else - echo "Servers is not ready" - exit 1 -fi + +echo "------- check server status -------" +cd $(dirname $0) +until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do + counter=$((counter+1)) + if [ $counter -eq 30 ]; then + echo "error: all servers start failed" + exit 1 + fi + echo "waiting for all servers started ..." + sleep 5 +done +echo "all servers start successfully" + +counter=0 +until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do + counter=$((counter+1)) + if [ $counter -eq 30 ]; then + echo "error: api server is not ready" + exit 1 + fi + echo "waiting for api server ready ..." + sleep 5 +done +echo "api server is ready" diff --git a/docker/docker-swarm/config.env.sh b/docker/docker-swarm/config.env.sh index aac31f544..5649f7578 100755 --- a/docker/docker-swarm/config.env.sh +++ b/docker/docker-swarm/config.env.sh @@ -61,6 +61,7 @@ LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab KERBEROS_EXPIRE_TIME=2 HDFS_ROOT_USER=hdfs +RESOURCE_MANAGER_HTTPADDRESS_PORT=8088 YARN_RESOURCEMANAGER_HA_RM_IDS= YARN_APPLICATION_STATUS_ADDRESS=http://ds1:8088/ws/v1/cluster/apps/%s # skywalking diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index aca9a70ea..71413815d 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -19,32 +19,32 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:latest + image: postgres:11.12 environment: TZ: Asia/Shanghai - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: dolphinscheduler volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql:/var/lib/postgresql/data restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-zookeeper: - image: bitnami/zookeeper:latest + image: zookeeper:3.6.3 environment: TZ: Asia/Shanghai - ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_DATA_LOG_DIR: /data ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper + - dolphinscheduler-zookeeper:/data restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-api: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: api-server ports: - 12345:12345 @@ -68,7 +68,7 @@ services: - dolphinscheduler dolphinscheduler-alert: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: alert-server environment: TZ: Asia/Shanghai @@ -87,7 +87,7 @@ services: - dolphinscheduler dolphinscheduler-master: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: master-server environment: TZ: Asia/Shanghai @@ -108,7 +108,7 @@ services: - dolphinscheduler dolphinscheduler-worker: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: worker-server environment: TZ: Asia/Shanghai diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index a073fdc12..b320cf3f7 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -19,14 +19,14 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:latest + image: postgres:11.12 environment: TZ: Asia/Shanghai - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: dolphinscheduler volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql:/var/lib/postgresql/data networks: - dolphinscheduler deploy: @@ -34,13 +34,13 @@ services: replicas: 1 dolphinscheduler-zookeeper: - image: bitnami/zookeeper:latest + image: zookeeper:3.6.3 environment: TZ: Asia/Shanghai - ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_DATA_LOG_DIR: /data ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper + - dolphinscheduler-zookeeper:/data networks: - dolphinscheduler deploy: @@ -48,7 +48,7 @@ services: replicas: 1 dolphinscheduler-api: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: api-server ports: - 12345:12345 @@ -71,7 +71,7 @@ services: replicas: 1 dolphinscheduler-alert: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: alert-server environment: TZ: Asia/Shanghai @@ -90,7 +90,7 @@ services: replicas: 1 dolphinscheduler-master: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: master-server environment: TZ: Asia/Shanghai @@ -110,7 +110,7 @@ services: replicas: 1 dolphinscheduler-worker: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: worker-server environment: TZ: Asia/Shanghai diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index e04936bf6..f97e15c47 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -35,18 +35,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 1.2.0 +version: 1.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 1.3.6 +appVersion: 1.3.7-SNAPSHOT dependencies: - name: postgresql - version: 10.x.x + version: 10.3.18 repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: zookeeper - version: 6.x.x + version: 6.5.3 repository: https://charts.bitnami.com/bitnami condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index ff6fa21c5..69ac6d13d 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -30,7 +30,7 @@ If release name contains chart name it will be used as a full name. Create a default docker image fullname. */}} {{- define "dolphinscheduler.image.fullname" -}} -{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}} +{{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion -}} {{- end -}} {{/* diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index 7a8d6ac8b..cc4b554da 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -49,8 +49,8 @@ spec: {{- end }} {{- if .Values.ingress.tls.enabled }} tls: - hosts: - - {{ .Values.ingress.host }} + - hosts: + - {{ .Values.ingress.host }} secretName: {{ .Values.ingress.tls.secretName }} {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index e64f0ac5d..735d3cd08 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -115,13 +115,12 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-master labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: - {{- range .Values.master.persistentVolumeClaim.accessModes }} - - {{ . | quote }} - {{- end }} + {{- range .Values.master.persistentVolumeClaim.accessModes }} + - {{ . | quote }} + {{- end }} storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }} resources: requests: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index ffb929138..aef1b7026 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -131,8 +131,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-data labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }} @@ -148,8 +147,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-logs labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }} diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index a2737728b..e5fd0811c 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -23,7 +23,7 @@ timezone: "Asia/Shanghai" image: repository: "apache/dolphinscheduler" - tag: "latest" + tag: "1.3.7-SNAPSHOT" pullPolicy: "IfNotPresent" pullSecret: "" @@ -83,6 +83,7 @@ common: LOGIN_USER_KEYTAB_PATH: "/opt/hdfs.keytab" KERBEROS_EXPIRE_TIME: "2" HDFS_ROOT_USER: "hdfs" + RESOURCE_MANAGER_HTTPADDRESS_PORT: "8088" YARN_RESOURCEMANAGER_HA_RM_IDS: "" YARN_APPLICATION_STATUS_ADDRESS: "http://ds1:8088/ws/v1/cluster/apps/%s" # skywalking diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 31498474c..4e81fe507 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -57,13 +57,14 @@ fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 + # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s -# if custom you resourcemanager port ,you need to replace 8088 else default value. -resource.manager.httpaddress.port=8088 # network interface preferred like eth0, default: empty #dolphin.scheduler.network.interface.preferred= diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index 535b7493c..672008080 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -15,17 +15,17 @@ # limitations under the License. # -# postgresql +# datasource configuration spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler -spring.datasource.username=test -spring.datasource.password=test +spring.datasource.username=root +spring.datasource.password=root -# mysql +# mysql example #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.username=xxxx -#spring.datasource.password=xxxx +#spring.datasource.username=ds_user +#spring.datasource.password=dolphinscheduler # connection configuration #spring.datasource.initialSize=5 -- GitLab