diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index cb1576367828f26f90b3bcf909b942cdc77fc6f4..a33cf490b1a70d9fd210133333b4269c4d4a6539 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -53,7 +53,7 @@ RUN dos2unix /root/checkpoint.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ - mkdir -p /tmp/xls && \ + mkdir -p /var/mail /tmp/xls && \ echo "Set disable_coredump false" >> /etc/sudo.conf # 4. expose port diff --git a/docker/build/README.md b/docker/build/README.md index 68f33e2387d3830e71f3aefc7a8537f7999a6d1e..6d83e898b444d67b11ca5e60e1c6475b2184c883 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -65,6 +65,8 @@ apache/dolphinscheduler:latest master-server ``` $ docker run -d --name dolphinscheduler-worker \ -e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -v dolphinscheduler-resource-local:/dolphinscheduler \ apache/dolphinscheduler:latest worker-server ``` diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index dd801ee83b25052db852e47581a79350e4fc34fa..68b33cbb366dcb6c33b597835654ef1c0896f030 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -64,6 +64,8 @@ apache/dolphinscheduler:latest master-server ``` $ docker run -d --name dolphinscheduler-worker \ -e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -v dolphinscheduler-resource-local:/dolphinscheduler \ apache/dolphinscheduler:latest worker-server ``` diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh index 78c8f98bc1d09d16af825ddce083f905520f8f95..7fd39335ae3e7d346cfd7cd217beb9497293381b 100755 --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh @@ -15,6 +15,14 @@ # limitations under the License. # -export PYTHON_HOME=/usr/bin/python2 +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/build/startup.sh b/docker/build/startup.sh index eb6a96a6a82062a7e32cf4fde4e6139f7ab626b4..8bd1895cc0c6f475d0ab2afc6d4619a487ec137b 100755 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -143,6 +143,7 @@ case "$1" in ;; (worker-server) waitZK + waitDatabase initWorkerServer initLoggerServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 2af0a8fc6c9b3a2bf419b366319d1b259288ab02..201e0f310207ba6f5d8deb07add3d304ae20ca73 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -173,6 +173,20 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 RESOURCE_STORAGE_TYPE: HDFS RESOURCE_UPLOAD_PATH: /dolphinscheduler diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index bec281fbb38363cb23d9b5e2954e9b96a2c40661..d68b6b90b409d697432c37c10bc8a481fef0f622 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -167,6 +167,20 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 RESOURCE_STORAGE_TYPE: HDFS RESOURCE_UPLOAD_PATH: /dolphinscheduler diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh index 654318cb41cdaf2c60b1b873b83155b9b10f06ee..7fd39335ae3e7d346cfd7cd217beb9497293381b 100755 --- a/docker/docker-swarm/dolphinscheduler_env.sh +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -15,6 +15,14 @@ # limitations under the License. # -export PYTHON_HOME=/usr/bin/python2 +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH \ No newline at end of file +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 4e692da9e0caed877509a7633681b184b506ca1d..fe83bd591306bc9e3a84cf99c3b105b240ba9a7d 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -99,6 +99,51 @@ spec: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-worker key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH + - name: XLS_FILE_PATH + valueFrom: + configMapKeyRef: + key: XLS_FILE_PATH + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_HOST + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_HOST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_PORT + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_PORT + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SENDER + valueFrom: + configMapKeyRef: + key: MAIL_SENDER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_USER + valueFrom: + configMapKeyRef: + key: MAIL_USER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_PASSWD + valueFrom: + configMapKeyRef: + key: MAIL_PASSWD + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_STARTTLS_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_STARTTLS_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_TRUST + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_TRUST + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}