Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
DolphinScheduler社区
DolphinScheduler
提交
07a6aec1
DolphinScheduler
项目概览
DolphinScheduler社区
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
67
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
07a6aec1
编写于
7月 01, 2020
作者:
L
lgcareer
提交者:
GitHub
7月 01, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix 2975:download resource need find the tenant code of resource owner (#3109)
上级
8e0dbd98
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
42 addition
and
37 deletion
+42
-37
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
.../org/apache/dolphinscheduler/dao/entity/TaskInstance.java
+11
-9
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java
.../dolphinscheduler/server/entity/TaskExecutionContext.java
+4
-5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
...ler/server/master/consumer/TaskPriorityQueueConsumer.java
+13
-14
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
...phinscheduler/server/worker/runner/TaskExecuteThread.java
+14
-9
未找到文件。
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
浏览文件 @
07a6aec1
...
...
@@ -16,20 +16,20 @@
*/
package
org.apache.dolphinscheduler.dao.entity
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.annotation.TableField
;
import
com.baomidou.mybatisplus.annotation.TableId
;
import
com.baomidou.mybatisplus.annotation.TableName
;
import
org.apache.dolphinscheduler.common.enums.ExecutionStatus
;
import
org.apache.dolphinscheduler.common.enums.Flag
;
import
org.apache.dolphinscheduler.common.enums.Priority
;
import
org.apache.dolphinscheduler.common.enums.TaskType
;
import
org.apache.dolphinscheduler.common.model.TaskNode
;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.annotation.TableId
;
import
com.baomidou.mybatisplus.annotation.TableName
;
import
java.io.Serializable
;
import
java.util.Date
;
import
java.util.
List
;
import
java.util.
Map
;
/**
* task instance
...
...
@@ -209,7 +209,7 @@ public class TaskInstance implements Serializable {
@TableField
(
exist
=
false
)
private
List
<
String
>
resources
;
private
Map
<
String
,
String
>
resources
;
...
...
@@ -451,10 +451,14 @@ public class TaskInstance implements Serializable {
||
(
this
.
getState
().
typeIsFailure
()
&&
!
taskCanRetry
());
}
public
List
<
String
>
getResources
()
{
public
Map
<
String
,
String
>
getResources
()
{
return
resources
;
}
public
void
setResources
(
Map
<
String
,
String
>
resources
)
{
this
.
resources
=
resources
;
}
public
boolean
isSubProcess
(){
return
TaskType
.
SUB_PROCESS
.
equals
(
TaskType
.
valueOf
(
this
.
taskType
));
}
...
...
@@ -467,9 +471,7 @@ public class TaskInstance implements Serializable {
return
TaskType
.
CONDITIONS
.
equals
(
TaskType
.
valueOf
(
this
.
taskType
));
}
public
void
setResources
(
List
<
String
>
resources
)
{
this
.
resources
=
resources
;
}
/**
* determine if you can try again
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java
浏览文件 @
07a6aec1
...
...
@@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import
java.io.Serializable
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.Map
;
/**
...
...
@@ -168,9 +167,9 @@ public class TaskExecutionContext implements Serializable{
private
String
workerGroup
;
/**
* resources full name
* resources full name
and tenant code
*/
private
List
<
String
>
resources
;
private
Map
<
String
,
String
>
resources
;
/**
* sql TaskExecutionContext
...
...
@@ -443,11 +442,11 @@ public class TaskExecutionContext implements Serializable{
this
.
dependenceTaskExecutionContext
=
dependenceTaskExecutionContext
;
}
public
List
<
String
>
getResources
()
{
public
Map
<
String
,
String
>
getResources
()
{
return
resources
;
}
public
void
setResources
(
List
<
String
>
resources
)
{
public
void
setResources
(
Map
<
String
,
String
>
resources
)
{
this
.
resources
=
resources
;
}
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
浏览文件 @
07a6aec1
...
...
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.consumer;
import
com.alibaba.fastjson.JSONObject
;
import
org.apache.dolphinscheduler.common.Constants
;
import
org.apache.dolphinscheduler.common.enums.ExecutionStatus
;
import
org.apache.dolphinscheduler.common.enums.ResourceType
;
import
org.apache.dolphinscheduler.common.enums.TaskType
;
import
org.apache.dolphinscheduler.common.enums.UdfType
;
import
org.apache.dolphinscheduler.common.model.TaskNode
;
...
...
@@ -32,7 +33,6 @@ import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import
org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter
;
import
org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter
;
import
org.apache.dolphinscheduler.common.thread.Stopper
;
import
org.apache.dolphinscheduler.common.thread.ThreadUtils
;
import
org.apache.dolphinscheduler.common.utils.*
;
import
org.apache.dolphinscheduler.dao.entity.*
;
import
org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder
;
...
...
@@ -50,14 +50,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import
org.springframework.stereotype.Component
;
import
javax.annotation.PostConstruct
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.Set
;
import
java.util.*
;
import
java.util.stream.Collectors
;
import
java.util.stream.Stream
;
import
static
org
.
apache
.
dolphinscheduler
.
common
.
Constants
.
SLEEP_TIME_MILLIS
;
/**
* TaskUpdateQueue consumer
*/
...
...
@@ -362,10 +358,10 @@ public class TaskPriorityQueueConsumer extends Thread{
}
/**
* get resource
full name list
* get resource
map key is full name and value is tenantCode
*/
private
List
<
String
>
getResourceFullNames
(
TaskNode
taskNode
)
{
List
<
String
>
resourceFullNameList
=
new
ArrayList
<>();
private
Map
<
String
,
String
>
getResourceFullNames
(
TaskNode
taskNode
)
{
Map
<
String
,
String
>
resourceMap
=
new
HashMap
<>();
AbstractParameters
baseParam
=
TaskParametersUtils
.
getParameters
(
taskNode
.
getType
(),
taskNode
.
getParams
());
if
(
baseParam
!=
null
)
{
...
...
@@ -375,7 +371,10 @@ public class TaskPriorityQueueConsumer extends Thread{
// filter the resources that the resource id equals 0
Set
<
ResourceInfo
>
oldVersionResources
=
projectResourceFiles
.
stream
().
filter
(
t
->
t
.
getId
()
==
0
).
collect
(
Collectors
.
toSet
());
if
(
CollectionUtils
.
isNotEmpty
(
oldVersionResources
))
{
resourceFullNameList
.
addAll
(
oldVersionResources
.
stream
().
map
(
resource
->
resource
.
getRes
()).
collect
(
Collectors
.
toSet
()));
oldVersionResources
.
forEach
(
(
t
)->
resourceMap
.
put
(
t
.
getRes
(),
processService
.
queryTenantCodeByResName
(
t
.
getRes
(),
ResourceType
.
FILE
))
);
}
// get the resource id in order to get the resource names in batch
...
...
@@ -386,13 +385,13 @@ public class TaskPriorityQueueConsumer extends Thread{
Integer
[]
resourceIds
=
resourceIdsSet
.
toArray
(
new
Integer
[
resourceIdsSet
.
size
()]);
List
<
Resource
>
resources
=
processService
.
listResourceByIds
(
resourceIds
);
resource
FullNameList
.
addAll
(
resources
.
stream
()
.
map
(
resourceInfo
->
resourceInfo
.
getFullName
(
))
.
collect
(
Collectors
.
toList
())
);
resource
s
.
forEach
(
(
t
)->
resourceMap
.
put
(
t
.
getFullName
(),
processService
.
queryTenantCodeByResName
(
t
.
getFullName
(),
ResourceType
.
FILE
))
);
}
}
}
return
resource
FullNameList
;
return
resource
Map
;
}
}
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
浏览文件 @
07a6aec1
...
...
@@ -18,13 +18,16 @@ package org.apache.dolphinscheduler.server.worker.runner;
import
com.alibaba.fastjson.JSONObject
;
import
org.apache.commons.collections.MapUtils
;
import
org.apache.dolphinscheduler.common.Constants
;
import
org.apache.dolphinscheduler.common.enums.ExecutionStatus
;
import
org.apache.dolphinscheduler.common.model.TaskNode
;
import
org.apache.dolphinscheduler.common.process.Property
;
import
org.apache.dolphinscheduler.common.task.TaskTimeoutParameter
;
import
org.apache.dolphinscheduler.common.thread.ThreadUtils
;
import
org.apache.dolphinscheduler.common.utils.*
;
import
org.apache.dolphinscheduler.common.utils.CommonUtils
;
import
org.apache.dolphinscheduler.common.utils.HadoopUtils
;
import
org.apache.dolphinscheduler.common.utils.LoggerUtils
;
import
org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand
;
import
org.apache.dolphinscheduler.server.entity.TaskExecutionContext
;
import
org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager
;
...
...
@@ -94,7 +97,6 @@ public class TaskExecuteThread implements Runnable {
// copy hdfs/minio file to local
downloadResource
(
taskExecutionContext
.
getExecutePath
(),
taskExecutionContext
.
getResources
(),
taskExecutionContext
.
getTenantCode
(),
logger
);
taskExecutionContext
.
setTaskParams
(
taskNode
.
getParams
());
...
...
@@ -224,22 +226,25 @@ public class TaskExecuteThread implements Runnable {
* @param logger
*/
private
void
downloadResource
(
String
execLocalPath
,
List
<
String
>
projectRes
,
String
tenantCode
,
Map
<
String
,
String
>
projectRes
,
Logger
logger
)
throws
Exception
{
if
(
Collection
Utils
.
isEmpty
(
projectRes
)){
if
(
Map
Utils
.
isEmpty
(
projectRes
)){
return
;
}
for
(
String
resource
:
projectRes
)
{
File
resFile
=
new
File
(
execLocalPath
,
resource
);
Set
<
Map
.
Entry
<
String
,
String
>>
resEntries
=
projectRes
.
entrySet
();
for
(
Map
.
Entry
<
String
,
String
>
resource
:
resEntries
)
{
String
fullName
=
resource
.
getKey
();
String
tenantCode
=
resource
.
getValue
();
File
resFile
=
new
File
(
execLocalPath
,
fullName
);
if
(!
resFile
.
exists
())
{
try
{
// query the tenant code of the resource according to the name of the resource
String
resHdfsPath
=
HadoopUtils
.
getHdfsResourceFileName
(
tenantCode
,
resourc
e
);
String
resHdfsPath
=
HadoopUtils
.
getHdfsResourceFileName
(
tenantCode
,
fullNam
e
);
logger
.
info
(
"get resource file from hdfs :{}"
,
resHdfsPath
);
HadoopUtils
.
getInstance
().
copyHdfsToLocal
(
resHdfsPath
,
execLocalPath
+
File
.
separator
+
resourc
e
,
false
,
true
);
HadoopUtils
.
getInstance
().
copyHdfsToLocal
(
resHdfsPath
,
execLocalPath
+
File
.
separator
+
fullNam
e
,
false
,
true
);
}
catch
(
Exception
e
){
logger
.
error
(
e
.
getMessage
(),
e
);
throw
new
RuntimeException
(
e
.
getMessage
());
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录