diff --git a/LICENSE b/LICENSE index dba3bc5288ca4c0bc9fe6a918b03a785a75d6610..9a0c6aa66af32d8832801f74195e3e266014cf7e 100644 --- a/LICENSE +++ b/LICENSE @@ -215,3 +215,4 @@ The following components are provided under the Apache License. See project link The text of each license is the standard Apache 2.0 license. ScriptRunner from https://github.com/mybatis/mybatis-3 Apache 2.0 mvnw files from https://github.com/takari/maven-wrapper Apache 2.0 + PropertyPlaceholderHelper from https://github.com/spring-projects/spring-framework Apache 2.0 diff --git a/NOTICE b/NOTICE index 63b57af8c819431e3710873ba048ae02cce82341..3e5695849ba084f186c76128f7ce3a9cb875c15c 100644 --- a/NOTICE +++ b/NOTICE @@ -72,4 +72,16 @@ Refactored SqlBuilder class (SQL, AbstractSQL) distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. + +Spring Framework ${version} +Copyright (c) 2002-${copyright} Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. \ No newline at end of file diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 08ccc3a2463d3ff62a3a6ad223fa0936b859a2c2..215916ddf742f1f7cc526e8276f6cb609b5e80f8 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-alert ${project.artifactId} diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index f45f854abe2ac47ffebea158c40fbc794e587dfa..035551e6693fd6cba5193432065b2b36cb5c77e4 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-api ${project.artifactId} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java index 289d5060bf01e0419b9ba93d7c5192f126b010ee..2d06e1b6ef6e6d6dd2b378552b88f7b1f4720ac3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java @@ -1,5 +1,3 @@ -package org.apache.dolphinscheduler.api.dto.resources; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -16,6 +14,8 @@ package org.apache.dolphinscheduler.api.dto.resources; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.dolphinscheduler.api.dto.resources; + /** * directory */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java index b9b91821f43b0e6ad07319fc414b03cd002ff909..92aaf1332074f236ceb3685c44b49081f8c07e11 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java @@ -1,5 +1,3 @@ -package org.apache.dolphinscheduler.api.dto.resources; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -16,6 +14,8 @@ package org.apache.dolphinscheduler.api.dto.resources; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.dolphinscheduler.api.dto.resources; + /** * file leaf */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index 6a5b6e3353d22e8ed524e5753eaf53cfe2720404..999fc88b0458ef1a6147446347035d898bf8421a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -1,11 +1,3 @@ -package org.apache.dolphinscheduler.api.dto.resources; - -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import org.apache.dolphinscheduler.common.enums.ResourceType; - -import java.util.ArrayList; -import java.util.List; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -22,6 +14,14 @@ import java.util.List; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.dolphinscheduler.api.dto.resources; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.apache.dolphinscheduler.common.enums.ResourceType; + +import java.util.ArrayList; +import java.util.List; + /** * resource component */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java index 3dfce7c7c15387a57765f93ca2d379933ae5da90..b2ef79c398410ffc63acd2e0143a534d7823b4b4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java @@ -1,8 +1,3 @@ -package org.apache.dolphinscheduler.api.dto.resources.visitor; - - -import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -19,6 +14,10 @@ import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.dolphinscheduler.api.dto.resources.visitor; + +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; + /** * Visitor */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index af29cb67fb159678d472694473d3a5e567ca1011..cb7a8e653f5f6e0db2b4f24b5cafe3000300cc9e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -50,20 +50,10 @@ public class LoginHandlerInterceptor implements HandlerInterceptor { /** * Intercept the execution of a handler. Called after HandlerMapping determined - * an appropriate handler object, but before HandlerAdapter invokes the handler. - *

DispatcherServlet processes a handler in an execution chain, consisting - * of any number of interceptors, with the handler itself at the end. - * With this method, each interceptor can decide to abort the execution chain, - * typically sending a HTTP error or writing a custom response. - *

Note: special considerations apply for asynchronous - * request processing. For more details see - * {@link org.springframework.web.servlet.AsyncHandlerInterceptor}. - * @param request current HTTP request - * @param response current HTTP response - * @param handler chosen handler to execute, for type and/or instance evaluation - * @return {@code true} if the execution chain should proceed with the - * next interceptor or the handler itself. Else, DispatcherServlet assumes - * that this interceptor has already dealt with the response itself. + * @param request current HTTP request + * @param response current HTTP response + * @param handler chosen handler to execute, for type and/or instance evaluation + * @return boolean true or false */ @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 3b0f82f5b0c856acaa86495f82f452ebc3e6a334..58a4acd174a09a56b72e9791ba8284ecc1164b20 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-common dolphinscheduler-common diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 4111ef9714764bef750d83e5849037acabdec0cd..fce59d65481038665898f311acecbe70b98832dc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -408,7 +408,7 @@ public class HadoopUtils implements Closeable { * @param applicationId application id * @return the return may be null or there may be other parse exceptions */ - public ExecutionStatus getApplicationStatus(String applicationId) throws Exception{ + public ExecutionStatus getApplicationStatus(String applicationId) throws Exception { if (StringUtils.isEmpty(applicationId)) { return null; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java index 32fd298a7dfe932374cf79be25d01da38b1373e0..1fe40b97e32d9eb58c022c12b29d1f75b7d4ea58 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java @@ -18,266 +18,55 @@ package org.apache.dolphinscheduler.common.utils; /** - * A collection of static utility methods to validate input. + * utility methods for validating input * - *

This class is modelled after Google Guava's Preconditions class, and partly takes code - * from that class. We add this code to here base in order to reduce external - * dependencies. */ public final class Preconditions { - // ------------------------------------------------------------------------ - // Null checks - // ------------------------------------------------------------------------ + private Preconditions() {} /** - * Ensures that the given object reference is not null. - * Upon violation, a {@code NullPointerException} with no message is thrown. + * if obj is null will throw NPE * - * @param reference reference + * @param obj obj * @param T * @return T */ - public static T checkNotNull(T reference) { - if (reference == null) { + public static T checkNotNull(T obj) { + if (obj == null) { throw new NullPointerException(); } - return reference; + return obj; } /** - * Ensures that the given object reference is not null. - * Upon violation, a {@code NullPointerException} with the given message is thrown. - * @param reference reference - * @param errorMessage errorMessage + * if obj is null will throw NullPointerException with error message + * @param obj obj + * @param errorMsg error message * @param T * @return T */ - public static T checkNotNull(T reference, String errorMessage) { - if (reference == null) { - throw new NullPointerException(String.valueOf(errorMessage)); - } - return reference; - } - - /** - * Ensures that the given object reference is not null. - * Upon violation, a {@code NullPointerException} with the given message is thrown. - * - *

The error message is constructed from a template and an arguments array, after - * a similar fashion as {@link String#format(String, Object...)}, but supporting only - * {@code %s} as a placeholder. - * - * @param reference The object reference - * @param errorMessageTemplate The message template for the {@code NullPointerException} - * that is thrown if the check fails. The template substitutes its - * {@code %s} placeholders with the error message arguments. - * @param errorMessageArgs The arguments for the error message, to be inserted into the - * message template for the {@code %s} placeholders. - * - * @param T - * @return The object reference itself (generically typed). - */ - public static T checkNotNull(T reference, - String errorMessageTemplate, - Object... errorMessageArgs) { - - if (reference == null) { - throw new NullPointerException(format(errorMessageTemplate, errorMessageArgs)); + public static T checkNotNull(T obj, String errorMsg) { + if (obj == null) { + throw new NullPointerException(errorMsg); } - return reference; + return obj; } - // ------------------------------------------------------------------------ - // Boolean Condition Checking (Argument) - // ------------------------------------------------------------------------ /** - * Checks the given boolean condition, and throws an {@code IllegalArgumentException} if - * the condition is not met (evaluates to {@code false}). + * if condition is false will throw an IllegalArgumentException with the given message * - * @param condition The condition to check + * @param condition condition + * @param errorMsg error message * * @throws IllegalArgumentException Thrown, if the condition is violated. */ - public static void checkArgument(boolean condition) { - if (!condition) { - throw new IllegalArgumentException(); - } - } - - /** - * Checks the given boolean condition, and throws an {@code IllegalArgumentException} if - * the condition is not met (evaluates to {@code false}). The exception will have the - * given error message. - * - * @param condition The condition to check - * @param errorMessage The message for the {@code IllegalArgumentException} that is thrown if the check fails. - * - * @throws IllegalArgumentException Thrown, if the condition is violated. - */ - public static void checkArgument(boolean condition, Object errorMessage) { - if (!condition) { - throw new IllegalArgumentException(String.valueOf(errorMessage)); - } - } - - /** - * Checks the given boolean condition, and throws an {@code IllegalArgumentException} if - * the condition is not met (evaluates to {@code false}). - * - * @param condition The condition to check - * @param errorMessageTemplate The message template for the {@code IllegalArgumentException} - * that is thrown if the check fails. The template substitutes its - * {@code %s} placeholders with the error message arguments. - * @param errorMessageArgs The arguments for the error message, to be inserted into the - * message template for the {@code %s} placeholders. - * - * @throws IllegalArgumentException Thrown, if the condition is violated. - */ - public static void checkArgument(boolean condition, - String errorMessageTemplate, - Object... errorMessageArgs) { - - if (!condition) { - throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs)); - } - } - - // ------------------------------------------------------------------------ - // Boolean Condition Checking (State) - // ------------------------------------------------------------------------ - - /** - * Checks the given boolean condition, and throws an {@code IllegalStateException} if - * the condition is not met (evaluates to {@code false}). - * - * @param condition The condition to check - * - * @throws IllegalStateException Thrown, if the condition is violated. - */ - public static void checkState(boolean condition) { - if (!condition) { - throw new IllegalStateException(); - } - } - - /** - * Checks the given boolean condition, and throws an {@code IllegalStateException} if - * the condition is not met (evaluates to {@code false}). The exception will have the - * given error message. - * - * @param condition The condition to check - * @param errorMessage The message for the {@code IllegalStateException} that is thrown if the check fails. - * - * @throws IllegalStateException Thrown, if the condition is violated. - */ - public static void checkState(boolean condition, Object errorMessage) { + public static void checkArgument(boolean condition, Object errorMsg) { if (!condition) { - throw new IllegalStateException(String.valueOf(errorMessage)); + throw new IllegalArgumentException(String.valueOf(errorMsg)); } } - /** - * Checks the given boolean condition, and throws an {@code IllegalStateException} if - * the condition is not met (evaluates to {@code false}). - * - * @param condition The condition to check - * @param errorMessageTemplate The message template for the {@code IllegalStateException} - * that is thrown if the check fails. The template substitutes its - * {@code %s} placeholders with the error message arguments. - * @param errorMessageArgs The arguments for the error message, to be inserted into the - * message template for the {@code %s} placeholders. - * - * @throws IllegalStateException Thrown, if the condition is violated. - */ - public static void checkState(boolean condition, - String errorMessageTemplate, - Object... errorMessageArgs) { - - if (!condition) { - throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs)); - } - } - - /** - * Ensures that the given index is valid for an array, list or string of the given size. - * - * @param index index to check - * @param size size of the array, list or string - * - * @throws IllegalArgumentException Thrown, if size is negative. - * @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size - */ - public static void checkElementIndex(int index, int size) { - checkArgument(size >= 0, "Size was negative."); - if (index < 0 || index >= size) { - throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + size); - } - } - /** - * Ensures that the given index is valid for an array, list or string of the given size. - * - * @param index index to check - * @param size size of the array, list or string - * @param errorMessage The message for the {@code IndexOutOfBoundsException} that is thrown if the check fails. - * - * @throws IllegalArgumentException Thrown, if size is negative. - * @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size - */ - public static void checkElementIndex(int index, int size, String errorMessage) { - checkArgument(size >= 0, "Size was negative."); - if (index < 0 || index >= size) { - throw new IndexOutOfBoundsException(String.valueOf(errorMessage) + " Index: " + index + ", Size: " + size); - } - } - - // ------------------------------------------------------------------------ - // Utilities - // ------------------------------------------------------------------------ - - /** - * A simplified formatting method. Similar to {@link String#format(String, Object...)}, but - * with lower overhead (only String parameters, no locale, no format validation). - * - *

This method is taken quasi verbatim from the Guava Preconditions class. - */ - private static String format( String template, Object... args) { - final int numArgs = args == null ? 0 : args.length; - template = String.valueOf(template); // null -> "null" - - // start substituting the arguments into the '%s' placeholders - StringBuilder builder = new StringBuilder(template.length() + 16 * numArgs); - int templateStart = 0; - int i = 0; - while (i < numArgs) { - int placeholderStart = template.indexOf("%s", templateStart); - if (placeholderStart == -1) { - break; - } - builder.append(template.substring(templateStart, placeholderStart)); - builder.append(args[i++]); - templateStart = placeholderStart + 2; - } - builder.append(template.substring(templateStart)); - - // if we run out of placeholders, append the extra args in square braces - if (i < numArgs) { - builder.append(" ["); - builder.append(args[i++]); - while (i < numArgs) { - builder.append(", "); - builder.append(args[i++]); - } - builder.append(']'); - } - - return builder.toString(); - } - - // ------------------------------------------------------------------------ - - /** Private constructor to prevent instantiation. */ - private Preconditions() {} } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java index 47b24bb93cdcfa53519030fca1bafe5fe7e5e905..3bf13aa9f36288e92d8925ef5e279f5848b9e003 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java @@ -33,11 +33,9 @@ public class PreconditionsTest { */ @Test public void testCheckNotNull() throws Exception { - String testReference = "test reference"; - //test reference is not null + String testReference = "test object"; Assert.assertEquals(testReference, Preconditions.checkNotNull(testReference)); - Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"reference is null")); - Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"%s is null",testReference)); + Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"object is null")); //test reference is null try { @@ -51,120 +49,19 @@ public class PreconditionsTest { } catch (NullPointerException ex) { assertNull(ex.getMessage()); } - //test reference is null ,expect contains errorMessage - try { - Preconditions.checkNotNull(null,"reference is null"); - } catch (NullPointerException ex) { - assertThat(ex.getMessage(), containsString("reference is null")); - } - - try { - Preconditions.checkNotNull("","reference is null"); - } catch (NullPointerException ex) { - assertThat(ex.getMessage(), containsString("reference is null")); - } - //test reference is null ,expect contains errorMessageTemplate and errorMessageArgs try { - Preconditions.checkNotNull(null,"%s is null",testReference); + Preconditions.checkNotNull(null,"object is null"); } catch (NullPointerException ex) { - assertThat(ex.getMessage(), containsString(testReference + " is null")); + assertThat(ex.getMessage(), containsString("object is null")); } try { - Preconditions.checkNotNull("","%s is null",testReference); + Preconditions.checkNotNull("","object is null"); } catch (NullPointerException ex) { - assertThat(ex.getMessage(), containsString(testReference + " is null")); - } - } - - /** - * Test checkArgument - */ - @Test - public void testCheckArgument() throws Exception { - - int argument = 100; - //boolean condition is true - Preconditions.checkArgument(argument > 0 && argument < 200); - - //boolean condition is false - try { - Preconditions.checkArgument(argument > 0 && argument < 50); - } catch (IllegalArgumentException ex) { - assertNull(ex.getMessage()); - } - - //boolean condition is false ,expect contains errorMessage - try { - Preconditions.checkArgument(argument > 300, "argument is error"); - } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage(), containsString("argument is error")); - } - - //boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs - try { - Preconditions.checkArgument(argument > 0 && argument < 99, "argument %s is error",argument); - } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage(), containsString( "argument " + argument + " is error")); - } - } - - /** - * Test checkState - */ - @Test - public void testCheckState() throws Exception { - int state = 1; - //boolean condition is true - Preconditions.checkState(state == 1); - Preconditions.checkState(state > -1); - - //boolean condition is false - try { - Preconditions.checkState(state > 2); - } catch (IllegalStateException ex) { - assertNull(ex.getMessage()); - } - - //boolean condition is false ,expect contains errorMessage - try { - Preconditions.checkState(state < 1, "state is error"); - } catch (IllegalStateException ex) { - assertThat(ex.getMessage(), containsString("state is error")); + assertThat(ex.getMessage(), containsString("object is null")); } - //boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs - try { - Preconditions.checkState(state < -1 , "state %s is error",state); - } catch (IllegalStateException ex) { - assertThat(ex.getMessage(), containsString( "state " + state + " is error")); - } } - /** - * Test checkElementIndex - */ - @Test - public void testCheckElementIndex() throws Exception { - int index = 2; - int size = 30; - - //boolean condition is true - Preconditions.checkElementIndex(index, size); - - //boolean condition is false - try { - Preconditions.checkElementIndex(-1, 10); - } catch (IndexOutOfBoundsException ex) { - assertThat(ex.getMessage(), containsString("Index: -1, Size: 10")); - } - - //boolean condition is false ,expect contains errorMessage - try { - Preconditions.checkElementIndex(100, 50, "index is greater than size"); - } catch (IndexOutOfBoundsException ex) { - assertThat(ex.getMessage(), containsString("index is greater than size Index: 100, Size: 50")); - } - } } diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 433bb700474392ae0a153109d72e4f5f230b1755..c474f6d9921d1eb6c975f0fe6854452d4c069f66 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-dao ${project.artifactId} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index eb51fc50cbf4909f2216aa58efa6ba583b8443e4..a90d9271547c04011d0f92a0bbe9cd1ba451cade 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -27,10 +28,11 @@ import org.apache.dolphinscheduler.common.utils.*; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; import java.util.Date; -import java.util.List; +import java.util.Map; /** * task instance @@ -213,7 +215,7 @@ public class TaskInstance implements Serializable { @TableField(exist = false) - private List resources; + private Map resources; @@ -455,10 +457,14 @@ public class TaskInstance implements Serializable { || (this.getState().typeIsFailure() && !taskCanRetry()); } - public List getResources() { + public Map getResources() { return resources; } + public void setResources(Map resources) { + this.resources = resources; + } + public boolean isSubProcess(){ return TaskType.SUB_PROCESS.equals(TaskType.valueOf(this.taskType)); } @@ -471,9 +477,7 @@ public class TaskInstance implements Serializable { return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType)); } - public void setResources(List resources) { - this.resources = resources; - } + /** * determine if you can try again diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 166edc9f5fc44f368d0927b2ba947c6948bf11af..ca3834e90b8896bb40c6e52a4e8ace262ea3ca25 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index b72d2ed23dab385f1fb5d27080306fbf9f1c9aa9..6ce789c7fb222a9f7a2bb4fb609be137a4b81fcc 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -132,18 +132,6 @@ granted provided that the copyright notice appears in all copies. ======================================================================== -Apache Log4j NOTICE - -======================================================================== -Apache log4j -Copyright 2007 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - Joda Time NOTICE ======================================================================== @@ -438,17 +426,6 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) ======================================================================== -Apache Jakarta Commons Lang NOTICE - -======================================================================== -Apache Jakarta Commons Lang -Copyright 2001-2007 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). -======================================================================== - - Apache ZooKeeper NOTICE ======================================================================== @@ -672,109 +649,6 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================== -Apache Commons CLI NOTICE - -======================================================================== -Apache Commons CLI -Copyright 2001-2009 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - -Apache Commons Collections NOTICE - -======================================================================== -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). -======================================================================== - - -Apache Commons Collections4 NOTICE - -======================================================================== -Apache Commons Collections -Copyright 2001-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - -Apache Commons Compress NOTICE - -======================================================================== -Apache Commons Compress -Copyright 2002-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - -Apache Commons Configuration NOTICE - -======================================================================== -Apache Commons Configuration -Copyright 2001-2013 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - -Apache Commons Daemon NOTICE - -======================================================================== -Copyright 1999-2019 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - -Apache Commons Email NOTICE - -======================================================================== -Apache Commons Email -Copyright 2001-2017 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - -Apache HttpComponents Client NOTICE - -======================================================================== -Copyright 1999-2018 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - -Apache Commons IO NOTICE - -======================================================================== -Copyright 2002-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================== - - Commons Logging NOTICE ======================================================================== @@ -872,18 +746,6 @@ file. ======================================================================== -Apache Commons Pool NOTICE - -======================================================================== -Apache Commons Pool -Copyright 2001-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -========================================================================= - - Apache Derby NOTICE =========================================================================== @@ -1176,18 +1038,6 @@ No other notice covers that jar file. ========================================================================= -Apache HttpClient NOTICE - -========================================================================= -Apache HttpComponents Client -Copyright 1999-2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -========================================================================= - - Apache Httpcomponents core NOTICE ========================================================================= @@ -1229,28 +1079,6 @@ from the source code management (SCM) system project uses. ========================================================================= -Apache Avro NOTICE - -========================================================================= -Apache Avro -Copyright 2009-2013 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apach Commons IO NOTICE - -========================================================================= -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - Junit4 NOTICE ========================================================================= @@ -1335,39 +1163,6 @@ Junit4 NOTICE WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - ========================================================================= - - -Apache Thrift NOTICE - -========================================================================= - Apache Thrift - Copyright 2006-2010 The Apache Software Foundation. - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - - Apache Commons DBCP NOTICE - -========================================================================= - Apache Commons DBCP - Copyright 2001-2010 The Apache Software Foundation - - This product includes software developed by - The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - - Apache Commons Daemon NOTICE - -========================================================================= - Apache Commons Daemon - Copyright 1999-2013 The Apache Software Foundation - - This product includes software developed by - The Apache Software Foundation (http://www.apache.org/). ========================================================================= Bonecp NOTICE @@ -1500,28 +1295,6 @@ Licensed under the Apache License, Version 2.0 (the "License"); ========================================================================= - Apache EL NOTICE - -========================================================================= - Apache Tomcat - Copyright 1999-2018 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apache Commons Net NOTICE - -========================================================================= -Apache Commons Net -Copyright 2001-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - Apache Logging Log4j2 NOTICE ========================================================================= @@ -1586,50 +1359,6 @@ Portions of this software were originally based on the following: ========================================================================= -Apache Parquet Hadoop Bundle NOTICE - -========================================================================= -Apache Parquet Hadoop Bundle -Copyright 2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apache Log4j 1.x NOTICE - -========================================================================= -Apache Log4j 1.x Compatibility API -Copyright 1999-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apache log4j NOTICE - -========================================================================= -Apache log4j -Copyright 2007 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apache Java Data Objects (JDO) NOTICE - -========================================================================= -Apache Java Data Objects (JDO) -Copyright 2005-2006 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - Jackson XC NOTICE ========================================================================= @@ -1656,17 +1385,6 @@ see CREDITS file. ========================================================================= -Apache HttpClient Mime NOTICE - -========================================================================= -Apache HttpClient Mime -Copyright 1999-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - Apache HttpCore NOTICE ========================================================================= @@ -1693,17 +1411,6 @@ The Apache Software Foundation (http://www.apache.org/). ========================================================================= -Hive Storage API NOTICE - -========================================================================= -Apache Hive -Copyright 2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - Hadoop NOTICE ========================================================================= @@ -2190,60 +1897,6 @@ Other developers who have contributed code are: ========================================================================= - -Apache Yetus NOTICE - -========================================================================= -Apache Yetus - Audience Annotations -Copyright 2015-2017 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apache Directory LDAP API Utilities NOTICE - -========================================================================= -Apache Directory LDAP API Utilities -Copyright 2003-2013 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -Apache Directory API ASN.1 API NOTICE - -========================================================================= -Apache Directory API ASN.1 API -Copyright 2003-2013 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -ApacheDS Protocol Kerberos Codec NOTICE - -========================================================================= -ApacheDS Protocol Kerberos Codec -Copyright 2003-2013 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). -========================================================================= - - -ApacheDS I18n NOTICE - -========================================================================= -ApacheDS I18n -Copyright 2003-2013 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - ANT NOTICE ========================================================================= diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml index 4d9775fa704b5fb78aebbed572b09d1a1d04dabf..6b11b2e2d6adcb5c9caa7795edd5baa35ef61509 100644 --- a/dolphinscheduler-microbench/pom.xml +++ b/dolphinscheduler-microbench/pom.xml @@ -21,7 +21,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml index 5c9a13867133b6b4f3dec6e15b94c28db4a3880e..7db15e73c340fa67991e6e7df180f6b868889574 100644 --- a/dolphinscheduler-plugin-api/pom.xml +++ b/dolphinscheduler-plugin-api/pom.xml @@ -23,7 +23,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-plugin-api ${project.artifactId} diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index a20a02e90df2eb8d254674afdc4eba2dbbfb30b7..4d398f306943a79cc4b1cc03da0a953b3e1e1d23 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java index 29d48db8f8cae3adc7962588243f7e1327a8cb99..d774dc8b9eb35353eab3e4c3b39980d1a2f4b7f8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java @@ -26,69 +26,34 @@ public class RemotingException extends Exception { super(); } - /** Constructs a new runtime exception with the specified detail message. - * The cause is not initialized, and may subsequently be initialized by a - * call to {@link #initCause}. + /** + * Construct a new runtime exception with the detail message * - * @param message the detail message. The detail message is saved for - * later retrieval by the {@link #getMessage()} method. + * @param message detail message */ public RemotingException(String message) { super(message); } /** - * Constructs a new runtime exception with the specified detail message and - * cause.

Note that the detail message associated with - * {@code cause} is not automatically incorporated in - * this runtime exception's detail message. + * Construct a new runtime exception with the detail message and cause * - * @param message the detail message (which is saved for later retrieval - * by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A null value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) + * @param message the detail message + * @param cause the cause * @since 1.4 */ public RemotingException(String message, Throwable cause) { super(message, cause); } - /** Constructs a new runtime exception with the specified cause and a - * detail message of (cause==null ? null : cause.toString()) - * (which typically contains the class and detail message of - * cause). This constructor is useful for runtime exceptions - * that are little more than wrappers for other throwables. + /** + * Construct a new runtime exception with throwable * - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A null value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - * @since 1.4 + * @param cause the cause */ public RemotingException(Throwable cause) { super(cause); } - /** - * Constructs a new runtime exception with the specified detail - * message, cause, suppression enabled or disabled, and writable - * stack trace enabled or disabled. - * - * @param message the detail message. - * @param cause the cause. (A {@code null} value is permitted, - * and indicates that the cause is nonexistent or unknown.) - * @param enableSuppression whether or not suppression is enabled - * or disabled - * @param writableStackTrace whether or not the stack trace should - * be writable - * - * @since 1.7 - */ - protected RemotingException(String message, Throwable cause, - boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java index 2e3954f4bc970b411ae7d31c19f089b279875c58..bbb32c76d1a83134a819afd305de6c9cd29bbd45 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java @@ -170,21 +170,6 @@ public class ResponseFuture { } } - @Override - public String toString() { - return "ResponseFuture{" + - "opaque=" + opaque + - ", timeoutMillis=" + timeoutMillis + - ", invokeCallback=" + invokeCallback + - ", releaseSemaphore=" + releaseSemaphore + - ", latch=" + latch + - ", beginTimestamp=" + beginTimestamp + - ", responseCommand=" + responseCommand + - ", sendOk=" + sendOk + - ", cause=" + cause + - '}'; - } - /** * scan future table */ @@ -209,4 +194,19 @@ public class ResponseFuture { } } } + + @Override + public String toString() { + return "ResponseFuture{" + + "opaque=" + opaque + + ", timeoutMillis=" + timeoutMillis + + ", invokeCallback=" + invokeCallback + + ", releaseSemaphore=" + releaseSemaphore + + ", latch=" + latch + + ", beginTimestamp=" + beginTimestamp + + ", responseCommand=" + responseCommand + + ", sendOk=" + sendOk + + ", cause=" + cause + + '}'; + } } diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index b2daacf5db26b90c12a306fbfa7cb5b774976116..4cbce0ab475cc9a84c1ba48622893b9f35acfbe2 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-server dolphinscheduler-server diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java index 97afb4f6d9abdd13f02e4cca993c2ea82f07b2c0..210db5c4c41fda097ce313ef304e62aecb668f16 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.server.entity; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import java.io.Serializable; -import java.util.List; +import java.util.Map; /** * SQL Task ExecutionContext @@ -38,9 +38,9 @@ public class SQLTaskExecutionContext implements Serializable { */ private String connectionParams; /** - * udf function list + * udf function tenant code map */ - private List udfFuncList; + private Map udfFuncTenantCodeMap; public int getWarningGroupId() { @@ -51,12 +51,12 @@ public class SQLTaskExecutionContext implements Serializable { this.warningGroupId = warningGroupId; } - public List getUdfFuncList() { - return udfFuncList; + public Map getUdfFuncTenantCodeMap() { + return udfFuncTenantCodeMap; } - public void setUdfFuncList(List udfFuncList) { - this.udfFuncList = udfFuncList; + public void setUdfFuncTenantCodeMap(Map udfFuncTenantCodeMap) { + this.udfFuncTenantCodeMap = udfFuncTenantCodeMap; } public String getConnectionParams() { @@ -72,7 +72,7 @@ public class SQLTaskExecutionContext implements Serializable { return "SQLTaskExecutionContext{" + "warningGroupId=" + warningGroupId + ", connectionParams='" + connectionParams + '\'' + - ", udfFuncList=" + udfFuncList + + ", udfFuncTenantCodeMap=" + udfFuncTenantCodeMap + '}'; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index 3fc65c1853e42a64c4d86f1eddf26a55ab4c0e26..81488fb134ef3bd35c4822d3721e844e83dc0782 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; -import java.util.List; import java.util.Map; /** @@ -171,9 +170,9 @@ public class TaskExecutionContext implements Serializable{ private String workerGroup; /** - * resources full name + * resources full name and tenant code */ - private List resources; + private Map resources; /** * sql TaskExecutionContext @@ -446,11 +445,11 @@ public class TaskExecutionContext implements Serializable{ this.dependenceTaskExecutionContext = dependenceTaskExecutionContext; } - public List getResources() { + public Map getResources() { return resources; } - public void setResources(List resources) { + public void setResources(Map resources) { this.resources = resources; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 68db1f2061989ea362b69cdee845ab7aee255653..21995c386708eceb27c1ef5f5f17c1123070a24a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.consumer; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.SqoopJobType; +import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -49,14 +50,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; - /** * TaskUpdateQueue consumer */ @@ -330,7 +327,13 @@ public class TaskPriorityQueueConsumer extends Thread{ } List udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray); - sqlTaskExecutionContext.setUdfFuncList(udfFuncList); + Map udfFuncMap = new HashMap<>(); + for(UdfFunc udfFunc : udfFuncList) { + String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF); + udfFuncMap.put(udfFunc,tenantCode); + } + + sqlTaskExecutionContext.setUdfFuncTenantCodeMap(udfFuncMap); } } @@ -364,20 +367,23 @@ public class TaskPriorityQueueConsumer extends Thread{ } /** - * get resource full name list + * get resource map key is full name and value is tenantCode */ - private List getResourceFullNames(TaskNode taskNode) { - List resourceFullNameList = new ArrayList<>(); + private Map getResourceFullNames(TaskNode taskNode) { + Map resourceMap = new HashMap<>(); AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams()); if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); - if (projectResourceFiles != null) { + if (CollectionUtils.isNotEmpty(projectResourceFiles)) { // filter the resources that the resource id equals 0 Set oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(oldVersionResources)) { - resourceFullNameList.addAll(oldVersionResources.stream().map(resource -> resource.getRes()).collect(Collectors.toSet())); + + oldVersionResources.forEach( + (t)->resourceMap.put(t.getRes(), processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE)) + ); } // get the resource id in order to get the resource names in batch @@ -388,13 +394,13 @@ public class TaskPriorityQueueConsumer extends Thread{ Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); List resources = processService.listResourceByIds(resourceIds); - resourceFullNameList.addAll(resources.stream() - .map(resourceInfo -> resourceInfo.getFullName()) - .collect(Collectors.toList())); + resources.forEach( + (t)->resourceMap.put(t.getFullName(),processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE)) + ); } } } - return resourceFullNameList; + return resourceMap; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java index 8a441b9de11d83c33d9725e404825a23194491ca..eaf614c0233fd05c0e5a9f8d5e3e08bb33748320 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java @@ -26,68 +26,21 @@ public class ExecuteException extends Exception{ super(); } - /** - * Constructs a new exception with the specified detail message. The - * cause is not initialized, and may subsequently be initialized by - * a call to {@link #initCause}. - * - * @param message the detail message. The detail message is saved for - * later retrieval by the {@link #getMessage()} method. - */ + public ExecuteException(String message) { super(message); } - /** - * Constructs a new exception with the specified detail message and - * cause.

Note that the detail message associated with - * {@code cause} is not automatically incorporated in - * this exception's detail message. - * - * @param message the detail message (which is saved for later retrieval - * by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A null value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - * @since 1.4 - */ + public ExecuteException(String message, Throwable cause) { super(message, cause); } - /** - * Constructs a new exception with the specified cause and a detail - * message of (cause==null ? null : cause.toString()) (which - * typically contains the class and detail message of cause). - * This constructor is useful for exceptions that are little more than - * wrappers for other throwables (for example, {@link - * java.security.PrivilegedActionException}). - * - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A null value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - * @since 1.4 - */ + public ExecuteException(Throwable cause) { super(cause); } - /** - * Constructs a new exception with the specified detail message, - * cause, suppression enabled or disabled, and writable stack - * trace enabled or disabled. - * - * @param message the detail message. - * @param cause the cause. (A {@code null} value is permitted, - * and indicates that the cause is nonexistent or unknown.) - * @param enableSuppression whether or not suppression is enabled - * or disabled - * @param writableStackTrace whether or not the stack trace should - * be writable - * @since 1.7 - */ protected ExecuteException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java index 63efb24a3efd3110611f94a70a724c85a9681c0b..3a8c8fe7d67a652edfd0e03db71dea1be6541f72 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.utils; +import org.apache.commons.collections.MapUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; @@ -24,10 +25,8 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.slf4j.Logger; import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; +import java.util.stream.Collectors; import static org.apache.dolphinscheduler.common.utils.CollectionUtils.isNotEmpty; @@ -43,53 +42,44 @@ public class UDFUtils { /** * create function list - * @param udfFuncs udf functions - * @param tenantCode tenant code - * @param logger logger + * @param udfFuncTenantCodeMap key is udf function,value is tenant code + * @param logger logger * @return create function list */ - public static List createFuncs(List udfFuncs, String tenantCode,Logger logger){ + public static List createFuncs(Map udfFuncTenantCodeMap, Logger logger){ - if (CollectionUtils.isEmpty(udfFuncs)){ + if (MapUtils.isEmpty(udfFuncTenantCodeMap)){ logger.info("can't find udf function resource"); return null; } - // get hive udf jar path - String hiveUdfJarPath = HadoopUtils.getHdfsUdfDir(tenantCode); - logger.info("hive udf jar path : {}" , hiveUdfJarPath); - - // is the root directory of udf defined - if (StringUtils.isEmpty(hiveUdfJarPath)) { - logger.error("not define hive udf jar path"); - throw new RuntimeException("hive udf jar base path not defined "); - } - Set resources = getFuncResouces(udfFuncs); List funcList = new ArrayList<>(); // build jar sql - buildJarSql(funcList, resources, hiveUdfJarPath); + buildJarSql(funcList, udfFuncTenantCodeMap); // build temp function sql - buildTempFuncSql(funcList, udfFuncs); + buildTempFuncSql(funcList, udfFuncTenantCodeMap.keySet().stream().collect(Collectors.toList())); return funcList; } /** * build jar sql - * @param sqls sql list - * @param resources resource set - * @param uploadPath upload path + * @param sqls sql list + * @param udfFuncTenantCodeMap key is udf function,value is tenant code */ - private static void buildJarSql(List sqls, Set resources, String uploadPath) { + private static void buildJarSql(List sqls, Map udfFuncTenantCodeMap) { String defaultFS = HadoopUtils.getInstance().getConfiguration().get(Constants.FS_DEFAULTFS); - if (!uploadPath.startsWith("hdfs:")) { - uploadPath = defaultFS + uploadPath; - } - for (String resource : resources) { - sqls.add(String.format("add jar %s/%s", uploadPath, resource)); + Set> entries = udfFuncTenantCodeMap.entrySet(); + for (Map.Entry entry:entries){ + String uploadPath = HadoopUtils.getHdfsUdfDir(entry.getValue()); + if (!uploadPath.startsWith("hdfs:")) { + uploadPath = defaultFS + uploadPath; + } + sqls.add(String.format("add jar %s%s", uploadPath, entry.getKey().getResourceName())); } + } /** @@ -106,20 +96,5 @@ public class UDFUtils { } } - /** - * get the resource names of all functions - * @param udfFuncs udf function list - * @return - */ - private static Set getFuncResouces(List udfFuncs) { - Set resources = new HashSet<>(); - - for (UdfFunc udfFunc : udfFuncs) { - resources.add(udfFunc.getResourceName()); - } - - return resources; - } - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index bf4c46f1b31f38647c0a3e102e1803330ef07614..26494bc77bcf2b307335e242e82ff4f27e258548 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -22,7 +22,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.Set; +import org.apache.commons.collections.MapUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -106,7 +108,6 @@ public class TaskExecuteThread implements Runnable { // copy hdfs/minio file to local downloadResource(taskExecutionContext.getExecutePath(), taskExecutionContext.getResources(), - taskExecutionContext.getTenantCode(), logger); taskExecutionContext.setTaskParams(taskNode.getParams()); @@ -227,22 +228,25 @@ public class TaskExecuteThread implements Runnable { * @param logger */ private void downloadResource(String execLocalPath, - List projectRes, - String tenantCode, + Map projectRes, Logger logger) throws Exception { - if (CollectionUtils.isEmpty(projectRes)){ + if (MapUtils.isEmpty(projectRes)){ return; } - for (String resource : projectRes) { - File resFile = new File(execLocalPath, resource); + Set> resEntries = projectRes.entrySet(); + + for (Map.Entry resource : resEntries) { + String fullName = resource.getKey(); + String tenantCode = resource.getValue(); + File resFile = new File(execLocalPath, fullName); if (!resFile.exists()) { try { // query the tenant code of the resource according to the name of the resource - String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tenantCode, resource); + String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tenantCode, fullName); logger.info("get resource file from hdfs :{}", resHdfsPath); - HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + resource, false, true); + HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + fullName, false, true); }catch (Exception e){ logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 0afeb8a3c8dff5e1ac8aefd8fbb91167b12770f2..acc75d70d2e199668987196332a419fc940b3d32 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -131,8 +131,7 @@ public class SqlTask extends AbstractTask { .map(this::getSqlAndSqlParamsMap) .collect(Collectors.toList()); - List createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncList(), - taskExecutionContext.getTenantCode(), + List createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncTenantCodeMap(), logger); // execute sql task diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 1b5b95363e38d9ca9d4bebcba2558f4ced3c664a..f0e8f408d7f149aa80879f2bce8c94c0df586941 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 8baeb38a777b19a1dbfbb71271bd3a94302c94a4..13644bad91f178f0e8dbe082dbbcc4df2496acc0 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index c8449cb21b19ebe6b91918250c842d99e59e3dc9..09636e4eab1a819abf44f9d95aa8ac3cfff01a85 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.dolphinscheduler dolphinscheduler - 1.3.0-SNAPSHOT + 1.3.2-SNAPSHOT pom ${project.artifactId} http://dolphinscheduler.apache.org @@ -916,7 +916,7 @@ **/dist/** **/licenses/** .github/** - sql/soft_version + **/sql/soft_version **/common/utils/ScriptRunner.java **/*.json diff --git a/sql/soft_version b/sql/soft_version index 589268e6fedb18e0dcdb97be4f19d569c5878d2b..6261a05bb0ef615cd09cf9ad6f8eb85f8f260084 100644 --- a/sql/soft_version +++ b/sql/soft_version @@ -1 +1 @@ -1.3.0 \ No newline at end of file +1.3.1 \ No newline at end of file