提交 c7f32e81 编写于 作者: Q qiaozhanwei 提交者: lgcareer

dockerfile modify (#1800)

* fix #1775 bug,delete process definition when process instance is running

* revert CONTRIBUTING.md

* dockerfile modify

* dockerfile modify

* dockerfile modify
上级 8c724d1f
......@@ -61,28 +61,13 @@ ADD ./dockerfile/conf/maven/settings.xml /opt/maven/conf
ENV MAVEN_HOME=/opt/maven
ENV PATH $PATH:$MAVEN_HOME/bin
#4,install node
RUN cd /opt && \
wget https://nodejs.org/download/release/v8.9.4/node-v8.9.4-linux-x64.tar.gz && \
tar -zxvf node-v8.9.4-linux-x64.tar.gz && \
mv node-v8.9.4-linux-x64 node && \
rm -rf ./node-v8.9.4-*tar.gz
ENV NODE_HOME=/opt/node
ENV PATH $PATH:$NODE_HOME/bin
#5,install postgresql
#4,install postgresql
RUN apt-get update && \
apt-get install -y postgresql postgresql-contrib sudo && \
sed -i 's/localhost/*/g' /etc/postgresql/10/main/postgresql.conf
#6,install nginx
RUN apt-get update && \
apt-get install -y nginx && \
rm -rf /var/lib/apt/lists/* && \
echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \
chown -R www-data:www-data /var/lib/nginx
#7,install sudo,python,vim,ping and ssh command
#5,install sudo,python,vim,ping and ssh command
RUN apt-get update && \
apt-get -y install sudo && \
apt-get -y install python && \
......@@ -90,40 +75,30 @@ RUN apt-get update && \
apt-get -y install iputils-ping && \
apt-get -y install net-tools && \
apt-get -y install openssh-server && \
apt-get -y install python-pip && \
pip install kazoo
apt-get -y install python-pip
#8,add dolphinscheduler source code to /opt/dolphinscheduler_source
#6,add dolphinscheduler source code to /opt/dolphinscheduler_source
ADD . /opt/dolphinscheduler_source
#9,backend compilation
#7,backend compilation
RUN cd /opt/dolphinscheduler_source && \
mvn clean package -Prelease -Dmaven.test.skip=true
#10,frontend compilation
RUN chmod -R 777 /opt/dolphinscheduler_source/dolphinscheduler-ui && \
cd /opt/dolphinscheduler_source/dolphinscheduler-ui && \
rm -rf /opt/dolphinscheduler_source/dolphinscheduler-ui/node_modules && \
npm install node-sass --unsafe-perm && \
npm install && \
npm run build
#11,modify dolphinscheduler configuration file
#8,modify dolphinscheduler configuration file
#backend configuration
RUN tar -zxvf /opt/dolphinscheduler_source/dolphinscheduler-dist/dolphinscheduler-backend/target/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-backend-bin.tar.gz -C /opt && \
mv /opt/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-backend-bin /opt/dolphinscheduler && \
RUN tar -zxvf /opt/dolphinscheduler_source/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-bin.tar.gz -C /opt && \
mv /opt/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-bin /opt/dolphinscheduler && \
rm -rf /opt/dolphinscheduler/conf
ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf
#frontend nginx configuration
ADD ./dockerfile/conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#12,open port
#9,open port
EXPOSE 2181 2888 3888 3306 80 12345 8888
COPY ./dockerfile/startup.sh /root/startup.sh
#13,modify permissions and set soft links
#10,modify permissions and set soft links
RUN chmod +x /root/startup.sh && \
chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \
chmod +x /opt/zookeeper/bin/zkServer.sh && \
......
......@@ -17,9 +17,12 @@
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
# postgresql
# postgre
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
# mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
......@@ -92,8 +95,47 @@ mybatis-plus.configuration.cache-enabled=false
mybatis-plus.configuration.call-setters-on-nulls=true
mybatis-plus.configuration.jdbc-type-for-null=null
# master settings
# master execute thread num
master.exec.threads=100
# master execute task number in parallel
master.exec.task.num=20
# master heartbeat interval
master.heartbeat.interval=10
# master commit task retry times
master.task.commit.retryTimes=5
# master commit task interval
master.task.commit.interval=1000
# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
master.max.cpuload.avg=100
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
master.reserved.memory=0.1
# worker settings
# worker execute thread num
worker.exec.threads=100
# worker heartbeat interval
worker.heartbeat.interval=10
# submit the number of tasks at a time
worker.fetch.task.num = 3
# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
worker.max.cpuload.avg=100
# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.
worker.reserved.memory=0.1
# data quality analysis is not currently in use. please ignore the following configuration
# task record flag
# task record
task.record.flag=false
task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8
task.record.datasource.username=xx
......
......@@ -18,14 +18,21 @@
#task queue implementation, default "zookeeper"
dolphinscheduler.queue.impl=zookeeper
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
zookeeper.quorum=127.0.0.1:2181
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/tmp/dolphinscheduler/download
#dolphinscheduler root directory
zookeeper.dolphinscheduler.root=/dolphinscheduler
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/dolphinscheduler/exec
#dolphinscheduler failover directory
zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.base.sleep=100
zookeeper.retry.max.sleep=30000
zookeeper.retry.maxtime=5
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
# Users who have permission to create directories under the HDFS root path
hdfs.root.user=hdfs
......@@ -33,8 +40,15 @@ hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/tmp/dolphinscheduler/download
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/dolphinscheduler/exec
# whether kerberos starts
hadoop.security.authentication.startup.state=false
......@@ -52,8 +66,29 @@ login.user.keytab.path=/opt/hdfs.headless.keytab
dolphinscheduler.env.path=/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
#resource.view.suffixs
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
# is development state? default "false"
development.state=true
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directory,support s3,for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 need,s3 endpoint
fs.s3a.endpoint=http://192.168.199.91:9010
# s3 need,s3 access key
fs.s3a.access.key=A3DXS30FO22544RE
# s3 need,s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directory,support s3,for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 need,s3 endpoint
fs.s3a.endpoint=http://192.168.199.91:9010
# s3 need,s3 access key
fs.s3a.access.key=A3DXS30FO22544RE
# s3 need,s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
\ No newline at end of file
......@@ -18,3 +18,8 @@
installPath=/data1_1T/dolphinscheduler
deployUser=dolphinscheduler
ips=ark0,ark1,ark2,ark3,ark4
sshPort=22
masters=ark0,ark1
workers=ark2,ark3,ark4
alertServer=ark3
apiServers=ark1
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
masters=ark0,ark1
workers=ark2,ark3,ark4
alertServer=ark3
apiServers=ark1
\ No newline at end of file
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export PYTHON_HOME=/usr/bin/python
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH
\ No newline at end of file
......@@ -34,9 +34,6 @@
left join (select process_definition_id,release_state as schedule_release_state from t_ds_schedules group by process_definition_id,release_state) sc on sc.process_definition_id = td.id
left join t_ds_user tu on td.user_id = tu.id
where td.project_id = #{projectId}
<if test=" isAdmin == false ">
and tu.user_type=1
</if>
<if test=" searchVal != null and searchVal != ''">
and td.name like concat('%', #{searchVal}, '%')
</if>
......@@ -70,9 +67,6 @@
FROM t_ds_process_definition td
JOIN t_ds_user tu on tu.id=td.user_id
where 1 = 1
<if test=" isAdmin == false ">
and tu.user_type=1
</if>
<if test="projectIds != null and projectIds.length != 0">
and td.project_id in
<foreach collection="projectIds" index="index" item="i" open="(" separator="," close=")">
......
......@@ -126,7 +126,7 @@
where 1 = 1
and t.is_sub_process = 0
<if test="startTime != null and endTime != null">
and t.start_time >= #{startTime} and t.start_time <![CDATA[ <= ]]> #{endTime}
and t.start_time <![CDATA[ >= ]]> #{startTime} and t.start_time <![CDATA[ <= ]]> #{endTime}
</if>
<if test="projectIds != null and projectIds.length != 0">
and p.id in
......@@ -147,7 +147,7 @@
from t_ds_process_instance
where process_definition_id=#{processDefinitionId}
<if test="startTime!=null and endTime != null ">
and schedule_time between #{startTime} and #{endTime}
and schedule_time <![CDATA[ >= ]]> #{startTime} and schedule_time <![CDATA[ <= ]]> #{endTime}
</if>
order by end_time desc limit 1
</select>
......@@ -163,7 +163,8 @@
</if>
<if test="startTime!=null and endTime != null ">
and process_definition_id=#{processDefinitionId}
and (schedule_time between #{startTime} and #{endTime} or start_time between #{startTime} and #{endTime})
and (schedule_time <![CDATA[ >= ]]> #{startTime} and schedule_time <![CDATA[ <= ]]> #{endTime}
or start_time <![CDATA[ >= ]]> #{startTime} and start_time <![CDATA[ <= ]]> #{endTime})
</if>
order by start_time desc limit 1
</select>
......@@ -173,7 +174,7 @@
where process_definition_id=#{processDefinitionId}
and schedule_time is null
<if test="startTime!=null and endTime != null ">
and start_time between #{startTime} and #{endTime}
and start_time <![CDATA[ >= ]]> #{startTime} and start_time <![CDATA[ <= ]]> #{endTime}
</if>
order by end_time desc limit 1
</select>
......
......@@ -39,8 +39,10 @@
<if test="type != -1">
and type=#{type}
</if>
and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId}
union select id as resources_id from t_ds_resources where user_id=#{userId})
<if test="userId != 0">
and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId}
union select id as resources_id from t_ds_resources where user_id=#{userId})
</if>
</select>
<select id="queryResourcePaging" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select *
......@@ -71,4 +73,18 @@
where t.id = u.tenant_id and u.id = res.user_id and res.type=0
and res.alias= #{resName}
</select>
<select id="listAuthorizedResource" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select *
from t_ds_resources
where type=0
and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId}
union select id as resources_id from t_ds_resources where user_id=#{userId})
<if test="resNames != null and resNames != ''">
and alias in
<foreach collection="resNames" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
</if>
</select>
</mapper>
......@@ -18,6 +18,12 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper">
<select id="selectUdfById" resultType="org.apache.dolphinscheduler.dao.entity.UdfFunc">
select *
from t_ds_udfs
where id = #{id}
</select>
<select id="queryUdfByIdStr" resultType="org.apache.dolphinscheduler.dao.entity.UdfFunc">
select *
from t_ds_udfs
......
......@@ -69,4 +69,14 @@
from t_ds_user u ,t_ds_access_token t
where u.id = t.user_id and token=#{token} and t.expire_time > NOW()
</select>
<select id="queryUserListByQueue" resultType="org.apache.dolphinscheduler.dao.entity.User">
select *
from t_ds_user
where queue = #{queueName}
</select>
<update id="updateUserQueue" parameterType="java.lang.String">
update t_ds_user
set queue = #{newQueue}
where queue = #{oldQueue}
</update>
</mapper>
\ No newline at end of file
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#zookeeper cluster
zookeeper.quorum=127.0.0.1:2181
#dolphinscheduler root directory
zookeeper.dolphinscheduler.root=/dolphinscheduler
#zookeeper server dirctory
zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers
zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters
zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers
#zookeeper lock dirctory
zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters
zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers
#dolphinscheduler failover directory
zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters
zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers
zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters
#dolphinscheduler failover directory
zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.sleep=1000
zookeeper.retry.maxtime=5
\ No newline at end of file
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
server {
listen 8888;
server_name localhost;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /opt/dolphinscheduler_source/dolphinscheduler-ui/dist;
index index.html index.html;
}
location /dolphinscheduler {
proxy_pass http://127.0.0.1:12345;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header x_real_ipP $remote_addr;
proxy_set_header remote_addr $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_http_version 1.1;
proxy_connect_timeout 300s;
proxy_read_timeout 300s;
proxy_send_timeout 300s;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
......@@ -41,12 +41,12 @@ echo "start api-server"
echo "start master-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop master-server
python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/masters
sh /opt/dolphinscheduler/script/remove-zk-node.sh /dolphinscheduler/masters
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start master-server
echo "start worker-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop worker-server
python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/workers
sh /opt/dolphinscheduler/script/remove-zk-node.sh /dolphinscheduler/workers
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start worker-server
......@@ -60,14 +60,6 @@ echo "start alert-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start alert-server
echo "start nginx"
/etc/init.d/nginx stop
nginx &
while true
do
sleep 101
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册