diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index c9c027ffd293c2acc257aaead725d4db080fff8d..ccb1ceed4a82513e624455f1ea2a2dac6dd24dd6 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -47,10 +47,11 @@ jobs: ${{ runner.os }}-maven- - name: Bootstrap database run: | - sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: postgres/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql - until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "wait for postgresql ..."; sleep 1; done - docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql bitnami/postgresql:latest bash -c "PGPASSWORD=test psql -h docker-swarm_dolphinscheduler-postgresql_1 -U test -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" + until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "waiting for postgresql ready ..."; sleep 1; done + docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql postgres:11.12 bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 1d5861db1127dc9115e1b74034b9409d5e915fb8..56e74bba282ed8a2194e70cf3ded2326325f47fd 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -23,6 +23,7 @@ ARG DEBIAN_FRONTEND=noninteractive ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DOCKER true +ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler # 1. install command/library/software # If install slowly, you can replcae debian's mirror with new mirror, Example: @@ -41,9 +42,8 @@ RUN apt-get update && \ # 2. add dolphinscheduler ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/ -RUN ln -s /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -WORKDIR ${DOLPHINSCHEDULER_HOME} +RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler +WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin # 3. add configuration and modify permissions and set soft links COPY ./checkpoint.sh /root/checkpoint.sh @@ -59,9 +59,10 @@ RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ - rm -rf /bin/sh && \ + rm -f /bin/sh && \ ln -s /bin/bash /bin/sh && \ mkdir -p /tmp/xls && \ + echo PS1=\"\\w \\$ \" >> ~/.bashrc && \ echo "Set disable_coredump false" >> /etc/sudo.conf # 4. expose port diff --git a/docker/build/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl index b47952163eb59c796226284791283ab1d7250ed5..6ce3a8ae0ec556169b6133f853445c65373d7f4a 100644 --- a/docker/build/conf/dolphinscheduler/alert.properties.tpl +++ b/docker/build/conf/dolphinscheduler/alert.properties.tpl @@ -40,9 +40,9 @@ enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID} enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET} enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID} enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS} -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} +enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=\$corpId&corpsecret=\$secret +enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=\$token +enterprise.wechat.team.send.msg={\"toparty\":\"\$toParty\",\"agentid\":\"\$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"\$msg\"},\"safe\":\"0\"} +enterprise.wechat.user.send.msg={\"touser\":\"\$toUser\",\"agentid\":\"\$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"\$msg\"}} plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index 84a449657e8e6a20c81c71e5c75d78637b14c774..d1ba621bfd62afda976b40a41f991d02a7dfc338 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -57,6 +57,9 @@ fs.s3a.access.key=${FS_S3A_ACCESS_KEY} # if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=${FS_S3A_SECRET_KEY} +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT} + # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS} diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index b414d21430ec6e3a7b18b0702cc3d6a484440e8b..9177732ba69a28cbe7f6b00a4c7040b6e035a181 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -15,23 +15,17 @@ # limitations under the License. # -# db +# datasource configuration spring.datasource.driver-class-name=${DATABASE_DRIVER} spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}${DATABASE_PARAMS:+?${DATABASE_PARAMS}} spring.datasource.username=${DATABASE_USERNAME} spring.datasource.password=${DATABASE_PASSWORD} -# postgresql -#spring.datasource.driver-class-name=org.postgresql.Driver -#spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler -#spring.datasource.username=test -#spring.datasource.password=test - -# mysql +# mysql example #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.username=xxxx -#spring.datasource.password=xxxx +#spring.datasource.username=ds_user +#spring.datasource.password=dolphinscheduler # connection configuration #spring.datasource.initialSize=5 diff --git a/docker/build/hooks/build b/docker/build/hooks/build index 0590761eeb91aa99578de8e9cb3411b6230afd29..ef89aaeabadc4e54f44c9cc3cad7b81704808681 100755 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -47,7 +47,21 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSIO mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/ # docker build -BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" +if [ "$1" = "x" ] +then + # build multi-arch images with buildx + if !(docker buildx use dolphinscheduler &> /dev/null) + then + docker buildx create --use --name dolphinscheduler + fi + docker buildx inspect --bootstrap + echo "" + BUILD_PLATFORM=linux/amd64,linux/arm64 + BUILD_COMMAND="docker buildx build --platform $BUILD_PLATFORM --push --build-arg VERSION=$VERSION -t $DOCKER_REPO:$VERSION $(pwd)/docker/build/" +else + BUILD_COMMAND="docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$VERSION $(pwd)/docker/build/" +fi + echo -e "$BUILD_COMMAND\n" if (docker info 2> /dev/null | grep -i "ERROR"); then sudo $BUILD_COMMAND diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 0a21633391df36c966980fcd7095706a356c7e8c..e3c3a42ed17a7adfe63fd2071ace8fff41557ec1 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -59,6 +59,7 @@ export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"} export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"} export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"} +export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"} export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""} export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:8088/ws/v1/cluster/apps/%s"} # skywalking diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check index f4093664da83c3b8ada5e94cb0ec7dd035872ffa..66484ae0c508c36183d2eb8562b66031e56cba60 100755 --- a/docker/docker-swarm/check +++ b/docker/docker-swarm/check @@ -15,21 +15,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # -echo "------ dolphinscheduler check - server - status -------" -sleep 60 -server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -if [ $server_num -eq 5 ] -then - echo "Server all start successfully" -else - echo "Server start failed "$server_num - exit 1 -fi -ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` -if [ $ready -eq 1 ] -then - echo "Servers is ready" -else - echo "Servers is not ready" - exit 1 -fi + +echo "------- check server status -------" +cd $(dirname $0) +until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do + counter=$((counter+1)) + if [ $counter -eq 30 ]; then + echo "error: all servers start failed" + exit 1 + fi + echo "waiting for all servers started ..." + sleep 5 +done +echo "all servers start successfully" + +counter=0 +until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do + counter=$((counter+1)) + if [ $counter -eq 30 ]; then + echo "error: api server is not ready" + exit 1 + fi + echo "waiting for api server ready ..." + sleep 5 +done +echo "api server is ready" diff --git a/docker/docker-swarm/config.env.sh b/docker/docker-swarm/config.env.sh index aac31f54456fdea5c3fbabc470d28bad2aa56b45..5649f7578bc668086a4f26c2d6b0351b6c07ba88 100755 --- a/docker/docker-swarm/config.env.sh +++ b/docker/docker-swarm/config.env.sh @@ -61,6 +61,7 @@ LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab KERBEROS_EXPIRE_TIME=2 HDFS_ROOT_USER=hdfs +RESOURCE_MANAGER_HTTPADDRESS_PORT=8088 YARN_RESOURCEMANAGER_HA_RM_IDS= YARN_APPLICATION_STATUS_ADDRESS=http://ds1:8088/ws/v1/cluster/apps/%s # skywalking diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index aca9a70ea1da33c6d2062ac112a8eff4cde8bde3..71413815d88ae1b6c8d8b1528e74363dd3cfba06 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -19,32 +19,32 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:latest + image: postgres:11.12 environment: TZ: Asia/Shanghai - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: dolphinscheduler volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql:/var/lib/postgresql/data restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-zookeeper: - image: bitnami/zookeeper:latest + image: zookeeper:3.6.3 environment: TZ: Asia/Shanghai - ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_DATA_LOG_DIR: /data ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper + - dolphinscheduler-zookeeper:/data restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-api: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: api-server ports: - 12345:12345 @@ -68,7 +68,7 @@ services: - dolphinscheduler dolphinscheduler-alert: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: alert-server environment: TZ: Asia/Shanghai @@ -87,7 +87,7 @@ services: - dolphinscheduler dolphinscheduler-master: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: master-server environment: TZ: Asia/Shanghai @@ -108,7 +108,7 @@ services: - dolphinscheduler dolphinscheduler-worker: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: worker-server environment: TZ: Asia/Shanghai diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index a073fdc12005523a368d6f4823652979f79f07c7..b320cf3f7fde8fe7925020c67590d9b2717dd588 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -19,14 +19,14 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:latest + image: postgres:11.12 environment: TZ: Asia/Shanghai - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: dolphinscheduler volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql:/var/lib/postgresql/data networks: - dolphinscheduler deploy: @@ -34,13 +34,13 @@ services: replicas: 1 dolphinscheduler-zookeeper: - image: bitnami/zookeeper:latest + image: zookeeper:3.6.3 environment: TZ: Asia/Shanghai - ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_DATA_LOG_DIR: /data ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper + - dolphinscheduler-zookeeper:/data networks: - dolphinscheduler deploy: @@ -48,7 +48,7 @@ services: replicas: 1 dolphinscheduler-api: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: api-server ports: - 12345:12345 @@ -71,7 +71,7 @@ services: replicas: 1 dolphinscheduler-alert: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: alert-server environment: TZ: Asia/Shanghai @@ -90,7 +90,7 @@ services: replicas: 1 dolphinscheduler-master: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: master-server environment: TZ: Asia/Shanghai @@ -110,7 +110,7 @@ services: replicas: 1 dolphinscheduler-worker: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.7-SNAPSHOT command: worker-server environment: TZ: Asia/Shanghai diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index e04936bf6e41c4a564fed2d75f66402c1d68a8d8..f97e15c470011850e52e87d7572406883136131d 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -35,18 +35,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 1.2.0 +version: 1.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 1.3.6 +appVersion: 1.3.7-SNAPSHOT dependencies: - name: postgresql - version: 10.x.x + version: 10.3.18 repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: zookeeper - version: 6.x.x + version: 6.5.3 repository: https://charts.bitnami.com/bitnami condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index ff6fa21c5a043c988525b13d76f8642527d35142..69ac6d13df919bb45e4b992d46e5671c699d066e 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -30,7 +30,7 @@ If release name contains chart name it will be used as a full name. Create a default docker image fullname. */}} {{- define "dolphinscheduler.image.fullname" -}} -{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}} +{{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion -}} {{- end -}} {{/* diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index 7a8d6ac8be8037e3af29af01885c33a4f6c8eb54..cc4b554da56fc40b5ca495350ca51086286813d7 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -49,8 +49,8 @@ spec: {{- end }} {{- if .Values.ingress.tls.enabled }} tls: - hosts: - - {{ .Values.ingress.host }} + - hosts: + - {{ .Values.ingress.host }} secretName: {{ .Values.ingress.tls.secretName }} {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index e64f0ac5da9d5ab4f72d65452914d4724224019a..735d3cd08c2c734754892bd7080e54c154dae517 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -115,13 +115,12 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-master labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: - {{- range .Values.master.persistentVolumeClaim.accessModes }} - - {{ . | quote }} - {{- end }} + {{- range .Values.master.persistentVolumeClaim.accessModes }} + - {{ . | quote }} + {{- end }} storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }} resources: requests: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index ffb92913814f03a8145d55330ffa9a7d7759abd5..aef1b70269ae7a375582e5f92df3eff5368d69f0 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -131,8 +131,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-data labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }} @@ -148,8 +147,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-logs labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }} diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index a2737728bc5bc1ec963b6816094703b5971e7b51..e5fd0811c72848d676dfd06d31415c1a8e1b57ae 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -23,7 +23,7 @@ timezone: "Asia/Shanghai" image: repository: "apache/dolphinscheduler" - tag: "latest" + tag: "1.3.7-SNAPSHOT" pullPolicy: "IfNotPresent" pullSecret: "" @@ -83,6 +83,7 @@ common: LOGIN_USER_KEYTAB_PATH: "/opt/hdfs.keytab" KERBEROS_EXPIRE_TIME: "2" HDFS_ROOT_USER: "hdfs" + RESOURCE_MANAGER_HTTPADDRESS_PORT: "8088" YARN_RESOURCEMANAGER_HA_RM_IDS: "" YARN_APPLICATION_STATUS_ADDRESS: "http://ds1:8088/ws/v1/cluster/apps/%s" # skywalking diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 31498474cb7664ef27525c8d5c7ef76ac7efcde6..4e81fe507ece9b937275f80cbb35cfada43e1336 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -57,13 +57,14 @@ fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 + # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s -# if custom you resourcemanager port ,you need to replace 8088 else default value. -resource.manager.httpaddress.port=8088 # network interface preferred like eth0, default: empty #dolphin.scheduler.network.interface.preferred= diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index 535b7493ce7adcfd54bd8575d99be60eaa646e36..6720080806ecce2a3595bbf7e532a47d3150c1e0 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -15,17 +15,17 @@ # limitations under the License. # -# postgresql +# datasource configuration spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler -spring.datasource.username=test -spring.datasource.password=test +spring.datasource.username=root +spring.datasource.password=root -# mysql +# mysql example #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.username=xxxx -#spring.datasource.password=xxxx +#spring.datasource.username=ds_user +#spring.datasource.password=dolphinscheduler # connection configuration #spring.datasource.initialSize=5