Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
DolphinScheduler社区
DolphinScheduler
提交
f59bb10c
DolphinScheduler
项目概览
DolphinScheduler社区
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
67
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
f59bb10c
编写于
12月 26, 2019
作者:
D
dk.technoboy
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor TaskQueueZkImpl
上级
35b6442f
变更
21
隐藏空白更改
内联
并排
Showing
21 changed file
with
44 addition
and
163 deletion
+44
-163
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java
...pache/dolphinscheduler/common/queue/TaskQueueFactory.java
+2
-1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
...apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
+23
-140
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java
...lphinscheduler/common/utils/SpringApplicationContext.java
+1
-1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java
.../apache/dolphinscheduler/common/zk/ZookeeperOperator.java
+0
-3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
...main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
+2
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
...g/apache/dolphinscheduler/server/master/MasterServer.java
+1
-2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
...eduler/server/master/runner/MasterBaseTaskExecThread.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
...lphinscheduler/server/master/runner/MasterExecThread.java
+0
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java
...scheduler/server/master/runner/MasterSchedulerThread.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
...g/apache/dolphinscheduler/server/worker/WorkerServer.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
...olphinscheduler/server/worker/runner/FetchTaskThread.java
+2
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java
...dolphinscheduler/server/worker/task/AbstractYarnTask.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java
...eduler/server/worker/task/dependent/DependentExecute.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java
...scheduler/server/worker/task/dependent/DependentTask.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
...he/dolphinscheduler/server/worker/task/http/HttpTask.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
...heduler/server/worker/task/processdure/ProcedureTask.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java
...olphinscheduler/server/worker/task/python/PythonTask.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
.../dolphinscheduler/server/worker/task/shell/ShellTask.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
...ache/dolphinscheduler/server/worker/task/sql/SqlTask.java
+1
-1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
...heduler/server/worker/shell/ShellCommandExecutorTest.java
+1
-1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
...e/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
+1
-1
未找到文件。
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java
浏览文件 @
f59bb10c
...
...
@@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.queue;
import
org.apache.dolphinscheduler.common.utils.CommonUtils
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -43,7 +44,7 @@ public class TaskQueueFactory {
String
queueImplValue
=
CommonUtils
.
getQueueImplValue
();
if
(
StringUtils
.
isNotBlank
(
queueImplValue
))
{
logger
.
info
(
"task queue impl use zookeeper "
);
return
TaskQueueZkImpl
.
getInstance
(
);
return
SpringApplicationContext
.
getBean
(
TaskQueueZkImpl
.
class
);
}
else
{
logger
.
error
(
"property dolphinscheduler.queue.impl can't be blank, system will exit "
);
System
.
exit
(-
1
);
...
...
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
浏览文件 @
f59bb10c
...
...
@@ -17,22 +17,14 @@
package
org.apache.dolphinscheduler.common.queue
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.configuration.ConfigurationException
;
import
org.apache.commons.configuration.PropertiesConfiguration
;
import
org.apache.curator.framework.CuratorFramework
;
import
org.apache.curator.framework.CuratorFrameworkFactory
;
import
org.apache.curator.retry.ExponentialBackoffRetry
;
import
org.apache.dolphinscheduler.common.Constants
;
import
org.apache.dolphinscheduler.common.utils.Bytes
;
import
org.apache.dolphinscheduler.common.utils.IpUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
import
org.apache.dolphinscheduler.common.zk.DefaultEnsembleProvider
;
import
org.apache.dolphinscheduler.common.zk.ZookeeperConfig
;
import
org.apache.zookeeper.CreateMode
;
import
org.apache.zookeeper.data.Stat
;
import
org.apache.dolphinscheduler.common.zk.ZookeeperOperator
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Service
;
import
java.util.*
;
...
...
@@ -40,35 +32,13 @@ import java.util.*;
* A singleton of a task queue implemented with zookeeper
* tasks queue implemention
*/
@Service
public
class
TaskQueueZkImpl
implements
ITaskQueue
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
TaskQueueZkImpl
.
class
);
private
static
volatile
TaskQueueZkImpl
instance
;
private
CuratorFramework
zkClient
;
private
ZookeeperConfig
zookeeperConfig
;
private
CuratorFramework
getZkClient
()
{
return
zkClient
;
}
private
TaskQueueZkImpl
(){
init
();
}
public
static
TaskQueueZkImpl
getInstance
(){
if
(
null
==
instance
)
{
synchronized
(
TaskQueueZkImpl
.
class
)
{
if
(
null
==
instance
)
{
instance
=
new
TaskQueueZkImpl
();
}
}
}
return
instance
;
}
@Autowired
private
ZookeeperOperator
zookeeperOperator
;
/**
* get all tasks from tasks queue
...
...
@@ -78,14 +48,12 @@ public class TaskQueueZkImpl implements ITaskQueue {
@Override
public
List
<
String
>
getAllTasks
(
String
key
)
{
try
{
List
<
String
>
list
=
getZkClient
().
getChildren
().
forPath
(
getTasksPath
(
key
));
List
<
String
>
list
=
zookeeperOperator
.
getChildrenKeys
(
getTasksPath
(
key
));
return
list
;
}
catch
(
Exception
e
)
{
logger
.
error
(
"get all tasks from tasks queue exception"
,
e
);
}
return
new
ArrayList
<
String
>();
return
new
ArrayList
<>();
}
/**
...
...
@@ -99,22 +67,8 @@ public class TaskQueueZkImpl implements ITaskQueue {
public
boolean
checkTaskExists
(
String
key
,
String
task
)
{
String
taskPath
=
getTasksPath
(
key
)
+
Constants
.
SINGLE_SLASH
+
task
;
try
{
Stat
stat
=
zkClient
.
checkExists
().
forPath
(
taskPath
);
if
(
null
==
stat
){
logger
.
info
(
"check task:{} not exist in task queue"
,
task
);
return
false
;
}
else
{
logger
.
info
(
"check task {} exists in task queue "
,
task
);
return
true
;
}
return
zookeeperOperator
.
isExisted
(
taskPath
);
}
catch
(
Exception
e
)
{
logger
.
info
(
String
.
format
(
"task {} check exists in task queue exception "
,
task
),
e
);
}
return
false
;
}
...
...
@@ -128,9 +82,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
public
boolean
add
(
String
key
,
String
value
){
try
{
String
taskIdPath
=
getTasksPath
(
key
)
+
Constants
.
SINGLE_SLASH
+
value
;
String
result
=
getZkClient
().
create
().
withMode
(
CreateMode
.
PERSISTENT
).
forPath
(
taskIdPath
,
Bytes
.
toBytes
(
value
));
logger
.
info
(
"add task : {} to tasks queue , result success"
,
result
);
zookeeperOperator
.
persist
(
taskIdPath
,
value
);
return
true
;
}
catch
(
Exception
e
)
{
logger
.
error
(
"add task to tasks queue exception"
,
e
);
...
...
@@ -153,8 +105,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
@Override
public
List
<
String
>
poll
(
String
key
,
int
tasksNum
)
{
try
{
CuratorFramework
zk
=
getZkClient
();
List
<
String
>
list
=
zk
.
getChildren
().
forPath
(
getTasksPath
(
key
));
List
<
String
>
list
=
zookeeperOperator
.
getChildrenKeys
(
getTasksPath
(
key
));
if
(
list
!=
null
&&
list
.
size
()
>
0
){
...
...
@@ -277,15 +228,12 @@ public class TaskQueueZkImpl implements ITaskQueue {
@Override
public
void
removeNode
(
String
key
,
String
nodeValue
){
CuratorFramework
zk
=
getZkClient
();
String
tasksQueuePath
=
getTasksPath
(
key
)
+
Constants
.
SINGLE_SLASH
;
String
taskIdPath
=
tasksQueuePath
+
nodeValue
;
logger
.
info
(
"
consum
e task {}"
,
taskIdPath
);
logger
.
info
(
"
removeNod
e task {}"
,
taskIdPath
);
try
{
Stat
stat
=
zk
.
checkExists
().
forPath
(
taskIdPath
);
if
(
stat
!=
null
){
zk
.
delete
().
forPath
(
taskIdPath
);
}
zookeeperOperator
.
remove
(
taskIdPath
);
}
catch
(
Exception
e
){
logger
.
error
(
String
.
format
(
"delete task:%s from zookeeper fail, exception:"
,
nodeValue
)
,
e
);
}
...
...
@@ -307,13 +255,10 @@ public class TaskQueueZkImpl implements ITaskQueue {
if
(
value
!=
null
&&
value
.
trim
().
length
()
>
0
){
String
path
=
getTasksPath
(
key
)
+
Constants
.
SINGLE_SLASH
;
CuratorFramework
zk
=
getZkClient
();
Stat
stat
=
zk
.
checkExists
().
forPath
(
path
+
value
);
if
(
null
==
stat
){
String
result
=
zk
.
create
().
withMode
(
CreateMode
.
PERSISTENT
).
forPath
(
path
+
value
,
Bytes
.
toBytes
(
value
));
logger
.
info
(
"add task:{} to tasks set result:{} "
,
value
,
result
);
}
else
{
if
(!
zookeeperOperator
.
isExisted
(
path
+
value
)){
zookeeperOperator
.
persist
(
path
+
value
,
value
);
logger
.
info
(
"add task:{} to tasks set "
,
value
);
}
else
{
logger
.
info
(
"task {} exists in tasks set "
,
value
);
}
...
...
@@ -336,15 +281,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
public
void
srem
(
String
key
,
String
value
)
{
try
{
String
path
=
getTasksPath
(
key
)
+
Constants
.
SINGLE_SLASH
;
CuratorFramework
zk
=
getZkClient
();
Stat
stat
=
zk
.
checkExists
().
forPath
(
path
+
value
);
if
(
null
!=
stat
){
zk
.
delete
().
forPath
(
path
+
value
);
logger
.
info
(
"delete task:{} from tasks set "
,
value
);
}
else
{
logger
.
info
(
"delete task:{} from tasks set fail, there is no this task"
,
value
);
}
zookeeperOperator
.
remove
(
path
+
value
);
}
catch
(
Exception
e
){
logger
.
error
(
String
.
format
(
"delete task:"
+
value
+
" exception"
),
e
);
...
...
@@ -363,7 +300,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
Set
<
String
>
tasksSet
=
new
HashSet
<>();
try
{
List
<
String
>
list
=
getZkClient
().
getChildren
().
forPath
(
getTasksPath
(
key
));
List
<
String
>
list
=
zookeeperOperator
.
getChildrenKeys
(
getTasksPath
(
key
));
for
(
String
task
:
list
)
{
tasksSet
.
add
(
task
);
...
...
@@ -377,56 +314,6 @@ public class TaskQueueZkImpl implements ITaskQueue {
return
tasksSet
;
}
/**
* Init the task queue of zookeeper node
*/
private
void
init
(){
initZkClient
();
try
{
String
tasksQueuePath
=
getTasksPath
(
Constants
.
DOLPHINSCHEDULER_TASKS_QUEUE
);
String
tasksCancelPath
=
getTasksPath
(
Constants
.
DOLPHINSCHEDULER_TASKS_KILL
);
for
(
String
taskQueuePath
:
new
String
[]{
tasksQueuePath
,
tasksCancelPath
}){
if
(
zkClient
.
checkExists
().
forPath
(
taskQueuePath
)
==
null
){
// create a persistent parent node
zkClient
.
create
().
creatingParentContainersIfNeeded
()
.
withMode
(
CreateMode
.
PERSISTENT
).
forPath
(
taskQueuePath
);
logger
.
info
(
"create tasks queue parent node success : {} "
,
taskQueuePath
);
}
}
}
catch
(
Exception
e
)
{
logger
.
error
(
"create zk node failure"
,
e
);
}
}
private
void
initZkClient
()
{
Configuration
conf
=
null
;
try
{
conf
=
new
PropertiesConfiguration
(
Constants
.
ZOOKEEPER_PROPERTIES_PATH
);
}
catch
(
ConfigurationException
ex
)
{
logger
.
error
(
"load zookeeper properties file failed, system exit"
);
System
.
exit
(-
1
);
}
zkClient
=
CuratorFrameworkFactory
.
builder
().
ensembleProvider
(
new
DefaultEnsembleProvider
(
conf
.
getString
(
"zookeeper.quorum"
)))
.
retryPolicy
(
new
ExponentialBackoffRetry
(
conf
.
getInt
(
"zookeeper.retry.base.sleep"
),
conf
.
getInt
(
"zookeeper.retry.maxtime"
),
conf
.
getInt
(
"zookeeper.retry.max.sleep"
)))
.
sessionTimeoutMs
(
conf
.
getInt
(
"zookeeper.session.timeout"
))
.
connectionTimeoutMs
(
conf
.
getInt
(
"zookeeper.connection.timeout"
))
.
build
();
zkClient
.
start
();
try
{
zkClient
.
blockUntilConnected
();
}
catch
(
final
Exception
ex
)
{
throw
new
RuntimeException
(
ex
);
}
}
/**
* Clear the task queue of zookeeper node
*/
...
...
@@ -437,16 +324,12 @@ public class TaskQueueZkImpl implements ITaskQueue {
String
tasksCancelPath
=
getTasksPath
(
Constants
.
DOLPHINSCHEDULER_TASKS_KILL
);
for
(
String
taskQueuePath
:
new
String
[]{
tasksQueuePath
,
tasksCancelPath
}){
if
(
zkClient
.
checkExists
().
forPath
(
taskQueuePath
)
!=
null
){
List
<
String
>
list
=
zkClient
.
getChildren
().
forPath
(
taskQueuePath
);
if
(
zookeeperOperator
.
isExisted
(
taskQueuePath
)){
List
<
String
>
list
=
zookeeperOperator
.
getChildrenKeys
(
taskQueuePath
);
for
(
String
task
:
list
)
{
z
kClient
.
delete
().
forPath
(
taskQueuePath
+
Constants
.
SINGLE_SLASH
+
task
);
z
ookeeperOperator
.
remove
(
taskQueuePath
+
Constants
.
SINGLE_SLASH
+
task
);
logger
.
info
(
"delete task from tasks queue : {}/{} "
,
taskQueuePath
,
task
);
}
}
}
...
...
dolphinscheduler-
server/src/main/java/org/apache/dolphinscheduler/server
/utils/SpringApplicationContext.java
→
dolphinscheduler-
common/src/main/java/org/apache/dolphinscheduler/common
/utils/SpringApplicationContext.java
浏览文件 @
f59bb10c
...
...
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.apache.dolphinscheduler.
server
.utils
;
package
org.apache.dolphinscheduler.
common
.utils
;
import
org.springframework.beans.BeansException
;
import
org.springframework.context.ApplicationContext
;
...
...
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java
浏览文件 @
f59bb10c
...
...
@@ -127,9 +127,6 @@ public class ZookeeperOperator implements InitializingBean {
List
<
String
>
values
;
try
{
values
=
zkClient
.
getChildren
().
forPath
(
key
);
if
(
CollectionUtils
.
isEmpty
(
values
))
{
logger
.
warn
(
"getChildrenKeys key : {} is empty"
,
key
);
}
return
values
;
}
catch
(
InterruptedException
ex
)
{
logger
.
error
(
"getChildrenKeys key : {} InterruptedException"
,
key
);
...
...
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
浏览文件 @
f59bb10c
...
...
@@ -105,7 +105,8 @@ public class ProcessDao {
/**
* task queue impl
*/
protected
ITaskQueue
taskQueue
=
TaskQueueFactory
.
getTaskQueueInstance
();
@Autowired
private
ITaskQueue
taskQueue
;
/**
* handle Command (construct ProcessInstance from Command) , wrapped in transaction
* @param logger logger
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
浏览文件 @
f59bb10c
...
...
@@ -23,18 +23,17 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import
org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors
;
import
org.apache.dolphinscheduler.common.thread.ThreadUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.server.master.config.MasterConfig
;
import
org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread
;
import
org.apache.dolphinscheduler.server.quartz.ProcessScheduleJob
;
import
org.apache.dolphinscheduler.server.quartz.QuartzExecutors
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.zk.ZKMasterClient
;
import
org.quartz.SchedulerException
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.WebApplicationType
;
import
org.springframework.boot.builder.SpringApplicationBuilder
;
import
org.springframework.context.annotation.ComponentScan
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
浏览文件 @
f59bb10c
...
...
@@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.server.master.runner;
import
org.apache.dolphinscheduler.common.queue.ITaskQueue
;
import
org.apache.dolphinscheduler.common.queue.TaskQueueFactory
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.AlertDao
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.ProcessInstance
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.dao.utils.BeanContext
;
import
org.apache.dolphinscheduler.server.master.config.MasterConfig
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
浏览文件 @
f59bb10c
...
...
@@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import
org.apache.dolphinscheduler.dao.utils.DagHelper
;
import
org.apache.dolphinscheduler.server.master.config.MasterConfig
;
import
org.apache.dolphinscheduler.server.utils.AlertManager
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java
浏览文件 @
f59bb10c
...
...
@@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.Constants;
import
org.apache.dolphinscheduler.common.thread.Stopper
;
import
org.apache.dolphinscheduler.common.thread.ThreadUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.common.zk.AbstractZKClient
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.Command
;
import
org.apache.dolphinscheduler.dao.entity.ProcessInstance
;
import
org.apache.dolphinscheduler.server.master.config.MasterConfig
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.zk.ZKMasterClient
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
浏览文件 @
f59bb10c
...
...
@@ -29,12 +29,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import
org.apache.dolphinscheduler.common.thread.ThreadUtils
;
import
org.apache.dolphinscheduler.common.utils.CollectionUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.common.zk.AbstractZKClient
;
import
org.apache.dolphinscheduler.dao.AlertDao
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.server.utils.ProcessUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.config.WorkerConfig
;
import
org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread
;
import
org.apache.dolphinscheduler.server.zk.ZKWorkerClient
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
浏览文件 @
f59bb10c
...
...
@@ -25,12 +25,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import
org.apache.dolphinscheduler.common.utils.CollectionUtils
;
import
org.apache.dolphinscheduler.common.utils.FileUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.common.zk.AbstractZKClient
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.dao.entity.Tenant
;
import
org.apache.dolphinscheduler.dao.entity.WorkerGroup
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.config.WorkerConfig
;
import
org.apache.dolphinscheduler.server.zk.ZKWorkerClient
;
import
org.slf4j.Logger
;
...
...
@@ -155,6 +155,7 @@ public class FetchTaskThread implements Runnable{
//whether have tasks, if no tasks , no need lock //get all tasks
List
<
String
>
tasksQueueList
=
taskQueue
.
getAllTasks
(
Constants
.
DOLPHINSCHEDULER_TASKS_QUEUE
);
if
(
CollectionUtils
.
isEmpty
(
tasksQueueList
)){
Thread
.
sleep
(
Constants
.
SLEEP_TIME_MILLIS
);
continue
;
}
// creating distributed locks, lock path /dolphinscheduler/lock/worker
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java
浏览文件 @
f59bb10c
...
...
@@ -16,10 +16,10 @@
*/
package
org.apache.dolphinscheduler.server.worker.task
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.server.utils.ProcessUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.slf4j.Logger
;
/**
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java
浏览文件 @
f59bb10c
...
...
@@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import
org.apache.dolphinscheduler.common.model.DateInterval
;
import
org.apache.dolphinscheduler.common.model.DependentItem
;
import
org.apache.dolphinscheduler.common.utils.DependentUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.ProcessInstance
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java
浏览文件 @
f59bb10c
...
...
@@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import
org.apache.dolphinscheduler.common.thread.Stopper
;
import
org.apache.dolphinscheduler.common.utils.DependentUtils
;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
import
org.slf4j.Logger
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
浏览文件 @
f59bb10c
...
...
@@ -30,10 +30,10 @@ import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import
org.apache.dolphinscheduler.common.utils.Bytes
;
import
org.apache.dolphinscheduler.common.utils.DateUtils
;
import
org.apache.dolphinscheduler.common.utils.ParameterUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.ProcessInstance
;
import
org.apache.dolphinscheduler.server.utils.ParamUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
import
org.apache.http.HttpEntity
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
浏览文件 @
f59bb10c
...
...
@@ -29,10 +29,10 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import
org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters
;
import
org.apache.dolphinscheduler.common.utils.CollectionUtils
;
import
org.apache.dolphinscheduler.common.utils.ParameterUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.DataSource
;
import
org.apache.dolphinscheduler.server.utils.ParamUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
import
org.slf4j.Logger
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java
浏览文件 @
f59bb10c
...
...
@@ -22,9 +22,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import
org.apache.dolphinscheduler.common.task.python.PythonParameters
;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
org.apache.dolphinscheduler.common.utils.ParameterUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.server.utils.ParamUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
浏览文件 @
f59bb10c
...
...
@@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import
org.apache.dolphinscheduler.common.task.shell.ShellParameters
;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
org.apache.dolphinscheduler.common.utils.ParameterUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.server.utils.ParamUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
浏览文件 @
f59bb10c
...
...
@@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.common.task.sql.SqlType;
import
org.apache.dolphinscheduler.common.utils.CollectionUtils
;
import
org.apache.dolphinscheduler.common.utils.CommonUtils
;
import
org.apache.dolphinscheduler.common.utils.ParameterUtils
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.AlertDao
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.DataSource
;
...
...
@@ -43,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import
org.apache.dolphinscheduler.dao.entity.UdfFunc
;
import
org.apache.dolphinscheduler.dao.entity.User
;
import
org.apache.dolphinscheduler.server.utils.ParamUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.utils.UDFUtils
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
...
...
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
浏览文件 @
f59bb10c
...
...
@@ -20,10 +20,10 @@ import com.alibaba.fastjson.JSONObject;
import
org.apache.dolphinscheduler.common.Constants
;
import
org.apache.dolphinscheduler.common.enums.ExecutionStatus
;
import
org.apache.dolphinscheduler.common.model.TaskNode
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.server.utils.LoggerUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.TaskManager
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
...
...
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
浏览文件 @
f59bb10c
...
...
@@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.Constants;
import
org.apache.dolphinscheduler.common.enums.CommandType
;
import
org.apache.dolphinscheduler.common.enums.ExecutionStatus
;
import
org.apache.dolphinscheduler.common.model.TaskNode
;
import
org.apache.dolphinscheduler.common.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.dao.ProcessDao
;
import
org.apache.dolphinscheduler.dao.entity.TaskInstance
;
import
org.apache.dolphinscheduler.server.utils.LoggerUtils
;
import
org.apache.dolphinscheduler.server.utils.SpringApplicationContext
;
import
org.apache.dolphinscheduler.server.worker.task.AbstractTask
;
import
org.apache.dolphinscheduler.server.worker.task.TaskManager
;
import
org.apache.dolphinscheduler.server.worker.task.TaskProps
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录