Skip to content

DolphinScheduler
DolphinScheduler

DolphinScheduler社区 / DolphinScheduler
与 Fork 源项目一致

Fork自 apache / DolphinScheduler

通知 67
Star 1
Fork 0
DolphinScheduler
DolphinScheduler

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 ac4ed940 编写于 作者: Y Yichao Yang 提交者： GitHub
浏览文件
操作

[Improvement][common] Add UnsupportedOperationException for utils construct (#3381) 

* [Improvement][common] Add UnsupportedOperationException for utils construct

* Fix checkstyle
上级 632d52cd
展开全部 隐藏空白更改
内联 并排
Showing with 1313 addition and 1204 deletion
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
浏览文件 @ ac4ed940
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
......@@ -27,7 +28,7 @@ import java.util.regex.Pattern;
public final class Constants {
private Constants() {
throw new IllegalStateException("Constants class");
throw new UnsupportedOperationException("Construct Constants");
}
/**
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
浏览文件 @ ac4ed940
......@@ -14,13 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Provides utility methods and decorators for {@link Collection} instances.
......@@ -37,8 +43,9 @@ import java.util.*;
public class CollectionUtils {
private CollectionUtils() {
throw new IllegalStateException("CollectionUtils class");
throw new UnsupportedOperationException("Construct CollectionUtils");
}
/**
* Returns a new {@link Collection} containing <i>a</i> minus a subset of
* <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate
......@@ -71,7 +78,7 @@ public class CollectionUtils {
/**
* String to map
*
* @param str string
* @param str string
* @param separator separator
* @return string to map
*/
......@@ -82,7 +89,7 @@ public class CollectionUtils {
/**
* String to map
*
* @param str string
* @param str string
* @param separator separator
* @param keyPrefix prefix
* @return string to map
......@@ -112,7 +119,6 @@ public class CollectionUtils {
return map;
}
/**
* Helper class to easily access cardinality properties of two collections.
*
......@@ -137,8 +143,8 @@ public class CollectionUtils {
* @param b the second collection
*/
public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) {
cardinalityA = CollectionUtils.<O>getCardinalityMap(a);
cardinalityB = CollectionUtils.<O>getCardinalityMap(b);
cardinalityA = CollectionUtils.getCardinalityMap(a);
cardinalityB = CollectionUtils.getCardinalityMap(b);
}
/**
......@@ -227,7 +233,7 @@ public class CollectionUtils {
* Only those elements present in the collection will appear as
* keys in the map.
*
* @param <O> the type of object in the returned {@link Map}. This is a super type of O
* @param <O> the type of object in the returned {@link Map}. This is a super type of O
* @param coll the collection to get the cardinality map for, must not be null
* @return the populated cardinality map
*/
......@@ -239,9 +245,9 @@ public class CollectionUtils {
return count;
}
/**
* Removes certain attributes of each object in the list
*
* @param originList origin list
* @param exclusionSet exclusion set
* @param <T> T
......@@ -258,8 +264,8 @@ public class CollectionUtils {
Map<String, Object> instanceMap;
for (T instance : originList) {
Map<String, Object> dataMap = new BeanMap(instance);
instanceMap = new LinkedHashMap<>(16,0.75f,true);
for (Map.Entry<String, Object> entry: dataMap.entrySet()) {
instanceMap = new LinkedHashMap<>(16, 0.75f, true);
for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
if (exclusionSet.contains(entry.getKey())) {
continue;
}
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
浏览文件 @ ac4ed940
......@@ -14,124 +14,129 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* common utils
*/
public class CommonUtils {
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
private static final Base64 BASE64 = new Base64();
private CommonUtils() {
throw new IllegalStateException("CommonUtils class");
}
/**
* @return get the path of system environment variables
*/
public static String getSystemEnvPath() {
String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
if (StringUtils.isEmpty(envPath)) {
URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
if (envDefaultPath != null){
envPath = envDefaultPath.getPath();
logger.debug("env path :{}", envPath);
}else{
envPath = "/etc/profile";
}
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
private static final Base64 BASE64 = new Base64();
private CommonUtils() {
throw new UnsupportedOperationException("Construct CommonUtils");
}
return envPath;
}
/**
*
* @return is develop mode
*/
public static boolean isDevelopMode() {
return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
}
/**
* if upload resource is HDFS and kerberos startup is true , else false
* @return true if upload resource is HDFS and kerberos startup
*/
public static boolean getKerberosStartupState(){
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
}
/**
* load kerberos configuration
* @throws Exception errors
*/
public static void loadKerberosConf()throws Exception{
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration();
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
/**
* @return get the path of system environment variables
*/
public static String getSystemEnvPath() {
String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
if (StringUtils.isEmpty(envPath)) {
URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
if (envDefaultPath != null) {
envPath = envDefaultPath.getPath();
logger.debug("env path :{}", envPath);
} else {
envPath = "/etc/profile";
}
}
return envPath;
}
}
/**
* encode password
* @param password
* @return
*/
public static String encodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ;
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
}
/**
* decode password
* @param password
* @return
*/
public static String decodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
if(!passwordWithSalt.startsWith(salt)){
logger.warn("There is a password and salt mismatch: {} ",password);
return password;
/**
* @return is develop mode
*/
public static boolean isDevelopMode() {
return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
}
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
}
/**
* if upload resource is HDFS and kerberos startup is true , else false
*
* @return true if upload resource is HDFS and kerberos startup
*/
public static boolean getKerberosStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
}
/**
* load kerberos configuration
*
* @throws Exception errors
*/
public static void loadKerberosConf() throws Exception {
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration();
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
}
/**
* encode password
*/
public static String encodePassword(String password) {
if (StringUtils.isEmpty(password)) {
return StringUtils.EMPTY;
}
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
if (!encryptionEnable) {
return password;
}
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)));
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
}
/**
* decode password
*/
public static String decodePassword(String password) {
if (StringUtils.isEmpty(password)) {
return StringUtils.EMPTY;
}
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
if (!encryptionEnable) {
return password;
}
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8);
if (!passwordWithSalt.startsWith(salt)) {
logger.warn("There is a password and salt mismatch: {} ", password);
return password;
}
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8);
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
浏览文件 @ ac4ed940
......@@ -14,37 +14,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.Arrays;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConnectionUtils {
public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class);
public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class);
private ConnectionUtils() {
throw new IllegalStateException("ConnectionUtils class");
}
private ConnectionUtils() {
throw new UnsupportedOperationException("Construct ConnectionUtils");
}
/**
* release resource
* @param resources resources
*/
public static void releaseResource(AutoCloseable... resources) {
/**
* release resource
*
* @param resources resources
*/
public static void releaseResource(AutoCloseable... resources) {
if (resources == null || resources.length == 0) {
return;
if (resources == null || resources.length == 0) {
return;
}
Arrays.stream(resources).filter(Objects::nonNull)
.forEach(resource -> {
try {
resource.close();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});
}
Arrays.stream(resources).filter(Objects::nonNull)
.forEach(resource -> {
try {
resource.close();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
浏览文件 @ ac4ed940
......@@ -36,6 +36,10 @@ public class DateUtils {
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils");
}
/**
* date to local datetime
*
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java
浏览文件 @ ac4ed940
......@@ -20,8 +20,6 @@ import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.DependentRelation;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
......@@ -29,32 +27,35 @@ import java.util.List;
public class DependentUtils {
private static final Logger logger = LoggerFactory.getLogger(DependentUtils.class);
private DependentUtils() {
throw new UnsupportedOperationException("Construct DependentUtils");
}
public static DependResult getDependResultForRelation(DependentRelation relation,
List<DependResult> dependResultList){
List<DependResult> dependResultList) {
DependResult dependResult = DependResult.SUCCESS;
switch (relation){
switch (relation) {
case AND:
if(dependResultList.contains(DependResult.FAILED)){
if (dependResultList.contains(DependResult.FAILED)) {
dependResult = DependResult.FAILED;
} if(dependResultList.contains(DependResult.WAITING)){
}
if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING;
}
break;
case OR:
if(dependResultList.contains(DependResult.SUCCESS)){
if (dependResultList.contains(DependResult.SUCCESS)) {
dependResult = DependResult.SUCCESS;
}else if(dependResultList.contains(DependResult.WAITING)){
} else if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING;
}else{
} else {
dependResult = DependResult.FAILED;
}
break;
default:
break;
break;
}
return dependResult;
}
......@@ -62,36 +63,37 @@ public class DependentUtils {
/**
* get date interval list by business date and date value.
*
* @param businessDate business date
* @param dateValue date value
* @return date interval list by business date and date value.
*/
public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue){
public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue) {
List<DateInterval> result = new ArrayList<>();
switch (dateValue){
switch (dateValue) {
case "currentHour":
result = DependentDateUtils.getLastHoursInterval(businessDate, 0);
break;
case "last1Hour":
result = DependentDateUtils.getLastHoursInterval(businessDate, 1);
result = DependentDateUtils.getLastHoursInterval(businessDate, 1);
break;
case "last2Hours":
result = DependentDateUtils.getLastHoursInterval(businessDate, 2);
result = DependentDateUtils.getLastHoursInterval(businessDate, 2);
break;
case "last3Hours":
result = DependentDateUtils.getLastHoursInterval(businessDate, 3);
result = DependentDateUtils.getLastHoursInterval(businessDate, 3);
break;
case "last24Hours":
result = DependentDateUtils.getSpecialLastDayInterval(businessDate);
break;
case "today":
result = DependentDateUtils.getTodayInterval(businessDate);
result = DependentDateUtils.getTodayInterval(businessDate);
break;
case "last1Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 1);
result = DependentDateUtils.getLastDayInterval(businessDate, 1);
break;
case "last2Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 2);
result = DependentDateUtils.getLastDayInterval(businessDate, 2);
break;
case "last3Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 3);
......@@ -144,5 +146,4 @@ public class DependentUtils {
return result;
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
浏览文件 @ ac4ed940
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.digest.DigestUtils;
......@@ -23,14 +24,16 @@ import org.apache.commons.codec.digest.DigestUtils;
*/
public class EncryptionUtils {
private EncryptionUtils() {
throw new UnsupportedOperationException("Construct EncryptionUtils");
}
/**
*
* @param rawStr raw string
* @return md5(rawStr)
*/
public static String getMd5(String rawStr) {
return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr);
return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr);
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
浏览文件 @ ac4ed940
......@@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
public class EnumUtils {
private EnumUtils() {
throw new UnsupportedOperationException("Construct EnumUtils");
}
public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) {
if (enumName == null) {
return null;
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
浏览文件 @ ac4ed940
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
......@@ -21,6 +22,9 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
......@@ -36,9 +40,6 @@ import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -46,12 +47,17 @@ import org.slf4j.LoggerFactory;
* file utils
*/
public class FileUtils {
public static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler");
public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH, "/tmp/dolphinscheduler");
public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
private FileUtils() {
throw new UnsupportedOperationException("Construct FileUtils");
}
/**
* get file suffix
*
......@@ -80,7 +86,7 @@ public class FileUtils {
String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename);
File file = new File(fileName);
if (!file.getParentFile().exists()){
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
......@@ -97,7 +103,7 @@ public class FileUtils {
public static String getUploadFilename(String tenantCode, String filename) {
String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename);
File file = new File(fileName);
if (!file.getParentFile().exists()){
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
......@@ -106,6 +112,7 @@ public class FileUtils {
/**
* directory of process execution
*
* @param projectId project id
* @param processDefineId process definition id
* @param processInstanceId process instance id
......@@ -114,9 +121,9 @@ public class FileUtils {
*/
public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) {
String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId),
Integer.toString(processDefineId), Integer.toString(processInstanceId),Integer.toString(taskInstanceId));
Integer.toString(processDefineId), Integer.toString(processInstanceId), Integer.toString(taskInstanceId));
File file = new File(fileName);
if (!file.getParentFile().exists()){
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
......@@ -125,6 +132,7 @@ public class FileUtils {
/**
* directory of process instances
*
* @param projectId project id
* @param processDefineId process definition id
* @param processInstanceId process instance id
......@@ -150,6 +158,7 @@ public class FileUtils {
/**
* create directory and user
*
* @param execLocalPath execute local path
* @param userName user name
* @throws IOException errors
......@@ -190,12 +199,11 @@ public class FileUtils {
OSUtils.taskLoggerThreadLocal.remove();
}
/**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir
*
* @param content content
* @param filePath target file path
* @param content content
* @param filePath target file path
* @return true if write success
*/
public static boolean writeContent2File(String content, String filePath) {
......@@ -231,13 +239,13 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
*
* <p>
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
* @since 2.4
......@@ -248,13 +256,13 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
*
* <p>
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
*/
......@@ -265,9 +273,9 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
......@@ -287,15 +295,14 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @throws UnsupportedCharsetException
* thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* supported by the VM
* @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* supported by the VM
* @since 2.1
*/
public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException {
......@@ -305,8 +312,8 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
* @param file the file to write
* @param data the content to write to the file
* @param file the file to write
* @param data the content to write to the file
* @throws IOException in case of an I/O error
*/
public static void writeStringToFile(File file, String data) throws IOException {
......@@ -316,8 +323,8 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
* @param file the file to write
* @param data the content to write to the file
* @param file the file to write
* @param data the content to write to the file
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
......@@ -340,7 +347,7 @@ public class FileUtils {
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
* @param file the file to open for output, must not be {@code null}
* @param file the file to open for output, must not be {@code null}
* @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to
......@@ -364,7 +371,7 @@ public class FileUtils {
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
* @param file the file to open for output, must not be {@code null}
* @param file the file to open for output, must not be {@code null}
* @param append if {@code true}, then bytes will be added to the
* end of the file rather than overwriting
* @return a new {@link FileOutputStream} for the specified file
......@@ -384,15 +391,15 @@ public class FileUtils {
} else {
File parent = file.getParentFile();
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
throw new IOException("Directory '" + parent + "' could not be created");
throw new IOException("Directory '" + parent + "' could not be created");
}
}
return new FileOutputStream(file, append);
}
/**
* deletes a directory recursively
*
* @param dir directory
* @throws IOException in case deletion is unsuccessful
*/
......@@ -420,17 +427,18 @@ public class FileUtils {
/**
* Gets all the parent subdirectories of the parentDir directory
*
* @param parentDir parent dir
* @return all dirs
*/
public static File[] getAllDir(String parentDir){
if(parentDir == null || "".equals(parentDir)) {
public static File[] getAllDir(String parentDir) {
if (parentDir == null || "".equals(parentDir)) {
throw new RuntimeException("parentDir can not be empty");
}
File file = new File(parentDir);
if(!file.exists() || !file.isDirectory()) {
throw new RuntimeException("parentDir not exist, or is not a directory:"+parentDir);
if (!file.exists() || !file.isDirectory()) {
throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
}
return file.listFiles(File::isDirectory);
......@@ -438,6 +446,7 @@ public class FileUtils {
/**
* Get Content
*
* @param inputStream input stream
* @return string of input stream
*/
......@@ -447,15 +456,14 @@ public class FileUtils {
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length= inputStream.read(buffer)) != -1) {
output.write(buffer,0,length);
while ((length = inputStream.read(buffer)) != -1) {
output.write(buffer, 0, length);
}
return output.toString();
} catch (Exception e) {
logger.error(e.getMessage(),e);
logger.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
浏览文件 @ ac4ed940
......@@ -14,9 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
......@@ -30,148 +32,148 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.Arrays;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* http utils
*/
public class HttpUtils {
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
private HttpUtils() {
}
public static CloseableHttpClient getInstance(){
return HttpClientInstance.httpClient;
}
private static class HttpClientInstance{
private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
}
private static PoolingHttpClientConnectionManager cm;
private static SSLContext ctx = null;
private static SSLConnectionSocketFactory socketFactory;
private static RequestConfig requestConfig;
private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
private static X509TrustManager xtm = new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) {
}
private HttpUtils() {
throw new UnsupportedOperationException("Construct HttpUtils");
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
public static CloseableHttpClient getInstance() {
return HttpClientInstance.httpClient;
}
static {
try {
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
ctx.init(null, new TrustManager[] { xtm }, null);
} catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) {
logger.error("SSLContext init with KeyManagementException", e);
}
socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
/** set timeout、request time、socket timeout */
requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setExpectContinueEnabled(Boolean.TRUE)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
.build();
socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setDefaultMaxPerRoute(60);
cm.setMaxTotal(100);
private static class HttpClientInstance {
private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
}
}
private static PoolingHttpClientConnectionManager cm;
private static SSLContext ctx = null;
private static SSLConnectionSocketFactory socketFactory;
private static RequestConfig requestConfig;
private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
private static X509TrustManager xtm = new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
static {
try {
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
ctx.init(null, new TrustManager[]{xtm}, null);
} catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) {
logger.error("SSLContext init with KeyManagementException", e);
}
socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
/** set timeout、request time、socket timeout */
requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setExpectContinueEnabled(Boolean.TRUE)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
.build();
socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setDefaultMaxPerRoute(60);
cm.setMaxTotal(100);
}
/**
* get http request content
* @param url url
* @return http get request response content
*/
public static String get(String url){
CloseableHttpClient httpclient = HttpUtils.getInstance();
/**
* get http request content
*
* @param url url
* @return http get request response content
*/
public static String get(String url) {
CloseableHttpClient httpclient = HttpUtils.getInstance();
HttpGet httpget = new HttpGet(url);
return getResponseContentString(httpget,httpclient);
HttpGet httpget = new HttpGet(url);
return getResponseContentString(httpget, httpclient);
}
/**
* get http response content
*
* @param httpget httpget
* @param httpget httpget
* @param httpClient httpClient
* @return http get request response content
*/
public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
String responseContent = null;
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpget);
// check response status is 200
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
if (entity != null) {
responseContent = EntityUtils.toString(entity, Constants.UTF_8);
} else {
logger.warn("http entity is null");
}
} else {
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
}
} catch (IOException ioe) {
logger.error(ioe.getMessage(), ioe);
} finally {
try {
if (response != null) {
EntityUtils.consume(response.getEntity());
response.close();
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
if (!httpget.isAborted()) {
httpget.releaseConnection();
httpget.abort();
}
}
return responseContent;
String responseContent = null;
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpget);
// check response status is 200
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
if (entity != null) {
responseContent = EntityUtils.toString(entity, Constants.UTF_8);
} else {
logger.warn("http entity is null");
}
} else {
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
}
} catch (IOException ioe) {
logger.error(ioe.getMessage(), ioe);
} finally {
try {
if (response != null) {
EntityUtils.consume(response.getEntity());
response.close();
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
if (!httpget.isAborted()) {
httpget.releaseConnection();
httpget.abort();
}
}
return responseContent;
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
浏览文件 @ ac4ed940
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
......@@ -18,14 +17,17 @@
package org.apache.dolphinscheduler.common.utils;
import java.io.Closeable;
import java.io.IOException;
public class IOUtils {
public static void closeQuietly(Closeable closeable){
if(closeable != null){
private IOUtils() {
throw new UnsupportedOperationException("Construct IOUtils");
}
public static void closeQuietly(Closeable closeable) {
if (closeable != null) {
try {
closeable.close();
} catch (IOException ignore) {
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
浏览文件 @ ac4ed940
......@@ -14,46 +14,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
/**
* http utils
*/
public class IpUtils {
public static final String DOT = ".";
/**
* ip str to long <p>
*
* @param ipStr ip string
* @return ip to long
*/
public static Long ipToLong(String ipStr) {
String[] ipSet = ipStr.split("\\" + DOT);
return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
}
/**
* long to ip
* @param ipLong the long number converted from IP
* @return String
*/
public static String longToIp(long ipLong) {
long[] ipNumbers = new long[4];
long tmp = 0xFF;
ipNumbers[0] = ipLong >> 24 & tmp;
ipNumbers[1] = ipLong >> 16 & tmp;
ipNumbers[2] = ipLong >> 8 & tmp;
ipNumbers[3] = ipLong & tmp;
String sb = ipNumbers[0] + DOT +
ipNumbers[1] + DOT +
ipNumbers[2] + DOT +
ipNumbers[3];
return sb;
}
private IpUtils() {
throw new UnsupportedOperationException("Construct IpUtils");
}
public static final String DOT = ".";
/**
* ip str to long <p>
*
* @param ipStr ip string
* @return ip to long
*/
public static Long ipToLong(String ipStr) {
String[] ipSet = ipStr.split("\\" + DOT);
return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
}
/**
* long to ip
*
* @param ipLong the long number converted from IP
* @return String
*/
public static String longToIp(long ipLong) {
long[] ipNumbers = new long[4];
long tmp = 0xFF;
ipNumbers[0] = ipLong >> 24 & tmp;
ipNumbers[1] = ipLong >> 16 & tmp;
ipNumbers[2] = ipLong >> 8 & tmp;
ipNumbers[3] = ipLong & tmp;
return ipNumbers[0] + DOT
+ ipNumbers[1] + DOT
+ ipNumbers[2] + DOT
+ ipNumbers[3];
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
浏览文件 @ ac4ed940
......@@ -14,25 +14,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.fasterxml.jackson.databind.type.CollectionType;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
import static com.fasterxml.jackson.databind.DeserializationFeature.*;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
/**
* json utils
......@@ -49,13 +63,12 @@ public class JSONUtils {
.configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
.configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.configure(REQUIRE_SETTERS_FOR_GETTERS, true)
.setTimeZone(TimeZone.getDefault())
;
.setTimeZone(TimeZone.getDefault());
private JSONUtils() {
throw new UnsupportedOperationException("Construct JSONUtils");
}
public static ArrayNode createArrayNode() {
return objectMapper.createArrayNode();
}
......@@ -94,9 +107,9 @@ public class JSONUtils {
* the fields of the specified object are generics, just the object itself should not be a
* generic type.
*
* @param json the string from which the object is to be deserialized
* @param json the string from which the object is to be deserialized
* @param clazz the class of T
* @param <T> T
* @param <T> T
* @return an object of type T from the string
* classOfT
*/
......@@ -116,9 +129,9 @@ public class JSONUtils {
/**
* json to list
*
* @param json json string
* @param json json string
* @param clazz class
* @param <T> T
* @param <T> T
* @return list
*/
public static <T> List<T> toList(String json, Class<T> clazz) {
......@@ -137,7 +150,6 @@ public class JSONUtils {
return Collections.emptyList();
}
/**
* check json object valid
*
......@@ -160,13 +172,12 @@ public class JSONUtils {
return false;
}
/**
* Method for finding a JSON Object field with specified name in this
* node or its child nodes, and returning value it has.
* If no matching field is found in this node or its descendants, returns null.
*
* @param jsonNode json node
* @param jsonNode json node
* @param fieldName Name of field to look for
* @return Value of first matching node found, if any; null if none
*/
......@@ -180,7 +191,6 @@ public class JSONUtils {
return node.toString();
}
/**
* json to map
* <p>
......@@ -195,7 +205,8 @@ public class JSONUtils {
}
try {
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {});
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {
});
} catch (Exception e) {
logger.error("json to map exception!", e);
}
......@@ -206,11 +217,11 @@ public class JSONUtils {
/**
* json to map
*
* @param json json
* @param json json
* @param classK classK
* @param classV classV
* @param <K> K
* @param <V> V
* @param <K> K
* @param <V> V
* @return to map
*/
public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) {
......@@ -258,7 +269,6 @@ public class JSONUtils {
}
}
/**
* json serializer
*/
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
浏览文件 @ ac4ed940
......@@ -14,22 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger;
/**
* logger utils
* logger utils
*/
public class LoggerUtils {
private LoggerUtils() {
throw new UnsupportedOperationException("Construct LoggerUtils");
}
/**
* rules for extracting application ID
*/
......@@ -53,29 +59,29 @@ public class LoggerUtils {
/**
* build job id
*
* @param affix Task Logger's prefix
* @param processDefId process define id
* @param affix Task Logger's prefix
* @param processDefId process define id
* @param processInstId process instance id
* @param taskId task id
* @param taskId task id
* @return task id format
*/
public static String buildTaskId(String affix,
int processDefId,
int processInstId,
int taskId){
int processDefId,
int processInstId,
int taskId) {
// - [taskAppId=TASK_79_4084_15210]
return String.format(" - %s%s-%s-%s-%s]",TASK_APPID_LOG_FORMAT,affix,
return String.format(" - %s%s-%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix,
processDefId,
processInstId,
taskId);
}
/**
* processing log
* get yarn application id list
* @param log log content
* @param logger logger
*
* @param log log content
* @param logger logger
* @return app id list
*/
public static List<String> getAppIds(String log, Logger logger) {
......@@ -87,7 +93,7 @@ public class LoggerUtils {
// analyse logs to get all submit yarn application id
while (matcher.find()) {
String appId = matcher.group();
if(!appIds.contains(appId)){
if (!appIds.contains(appId)) {
logger.info("find app id: {}", appId);
appIds.add(appId);
}
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
浏览文件 @ ac4ed940
......@@ -14,27 +14,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE;
import static java.util.Collections.emptyList;
import java.io.IOException;
import java.net.*;
import java.util.*;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;
import static java.util.Collections.emptyList;
import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* NetUtils
*/
public class NetUtils {
private NetUtils() {
throw new IllegalStateException("Utility class");
throw new UnsupportedOperationException("Construct NetUtils");
}
private static Logger logger = LoggerFactory.getLogger(NetUtils.class);
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
浏览文件 @ ac4ed940
......@@ -14,10 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType;
......@@ -25,231 +24,228 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.PreparedStatement;
import java.text.ParseException;
import java.util.*;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* parameter parse utils
*/
public class ParameterUtils {
private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
/**
* convert parameters place holders
*
* @param parameterString parameter
* @param parameterMap parameter map
* @return convert parameters place holders
*/
public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
if (StringUtils.isEmpty(parameterString) || parameterMap == null) {
return parameterString;
}
//Get current time, schedule execute time
String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) {
try {
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
} catch (ParseException e) {
logger.error("parse {} exception", cronTimeStr, e);
}
} else {
cronTime = new Date();
}
// replace variable ${} form,refers to the replacement of system variables and custom variables
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
// replace time $[...] form, eg. $[yyyyMMdd]
if (cronTime != null) {
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
private ParameterUtils() {
throw new UnsupportedOperationException("Construct ParameterUtils");
}
return parameterString;
}
/**
* new
* convert parameters place holders
*
* @param parameterString parameter
* @param parameterMap parameter map
* @return convert parameters place holders
*/
public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
if (StringUtils.isEmpty(parameterString)) {
return parameterString;
}
//Get current time, schedule execute time
String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) {
try {
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
} catch (ParseException e) {
logger.error(String.format("parse %s exception", cronTimeStr), e);
}
} else {
cronTime = new Date();
/**
* convert parameters place holders
*
* @param parameterString parameter
* @param parameterMap parameter map
* @return convert parameters place holders
*/
public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
if (StringUtils.isEmpty(parameterString) || parameterMap == null) {
return parameterString;
}
//Get current time, schedule execute time
String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) {
cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
} else {
cronTime = new Date();
}
// replace variable ${} form,refers to the replacement of system variables and custom variables
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
// replace time $[...] form, eg. $[yyyyMMdd]
if (cronTime != null) {
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
}
return parameterString;
}
// replace variable ${} form,refers to the replacement of system variables and custom variables
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
// replace time $[...] form, eg. $[yyyyMMdd]
if (cronTime != null) {
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
}
return parameterString;
}
/**
* set in parameter
* @param index index
* @param stmt preparedstatement
* @param dataType data type
* @param value value
* @throws Exception errors
*/
public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{
if (dataType.equals(DataType.VARCHAR)){
stmt.setString(index,value);
}else if (dataType.equals(DataType.INTEGER)){
stmt.setInt(index, Integer.parseInt(value));
}else if (dataType.equals(DataType.LONG)){
stmt.setLong(index, Long.parseLong(value));
}else if (dataType.equals(DataType.FLOAT)){
stmt.setFloat(index, Float.parseFloat(value));
}else if (dataType.equals(DataType.DOUBLE)){
stmt.setDouble(index, Double.parseDouble(value));
}else if (dataType.equals(DataType.DATE)){
stmt.setDate(index, java.sql.Date.valueOf(value));
}else if (dataType.equals(DataType.TIME)){
stmt.setString(index, value);
}else if (dataType.equals(DataType.TIMESTAMP)){
stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
}else if (dataType.equals(DataType.BOOLEAN)){
stmt.setBoolean(index,Boolean.parseBoolean(value));
}
}
/**
* curing user define parameters
*
* @param globalParamMap global param map
* @param globalParamList global param list
* @param commandType command type
* @param scheduleTime schedule time
* @return curing user define parameters
*/
public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime){
if (globalParamList == null || globalParamList.isEmpty()) {
return null;
/**
* new
* convert parameters place holders
*
* @param parameterString parameter
* @param parameterMap parameter map
* @return convert parameters place holders
*/
public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
if (StringUtils.isEmpty(parameterString)) {
return parameterString;
}
//Get current time, schedule execute time
String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) {
cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
} else {
cronTime = new Date();
}
// replace variable ${} form,refers to the replacement of system variables and custom variables
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
// replace time $[...] form, eg. $[yyyyMMdd]
if (cronTime != null) {
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
}
return parameterString;
}
Map<String, String> globalMap = new HashMap<>();
if (globalParamMap!= null){
globalMap.putAll(globalParamMap);
/**
* set in parameter
*
* @param index index
* @param stmt preparedstatement
* @param dataType data type
* @param value value
* @throws Exception errors
*/
public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception {
if (dataType.equals(DataType.VARCHAR)) {
stmt.setString(index, value);
} else if (dataType.equals(DataType.INTEGER)) {
stmt.setInt(index, Integer.parseInt(value));
} else if (dataType.equals(DataType.LONG)) {
stmt.setLong(index, Long.parseLong(value));
} else if (dataType.equals(DataType.FLOAT)) {
stmt.setFloat(index, Float.parseFloat(value));
} else if (dataType.equals(DataType.DOUBLE)) {
stmt.setDouble(index, Double.parseDouble(value));
} else if (dataType.equals(DataType.DATE)) {
stmt.setDate(index, java.sql.Date.valueOf(value));
} else if (dataType.equals(DataType.TIME)) {
stmt.setString(index, value);
} else if (dataType.equals(DataType.TIMESTAMP)) {
stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
} else if (dataType.equals(DataType.BOOLEAN)) {
stmt.setBoolean(index, Boolean.parseBoolean(value));
}
}
Map<String,String> allParamMap = new HashMap<>();
//If it is a complement, a complement time needs to be passed in, according to the task type
Map<String,String> timeParams = BusinessTimeUtils
.getBusinessTime(commandType, scheduleTime);
if (timeParams != null) {
allParamMap.putAll(timeParams);
/**
* curing user define parameters
*
* @param globalParamMap global param map
* @param globalParamList global param list
* @param commandType command type
* @param scheduleTime schedule time
* @return curing user define parameters
*/
public static String curingGlobalParams(Map<String, String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime) {
if (globalParamList == null || globalParamList.isEmpty()) {
return null;
}
Map<String, String> globalMap = new HashMap<>();
if (globalParamMap != null) {
globalMap.putAll(globalParamMap);
}
Map<String, String> allParamMap = new HashMap<>();
//If it is a complement, a complement time needs to be passed in, according to the task type
Map<String, String> timeParams = BusinessTimeUtils
.getBusinessTime(commandType, scheduleTime);
if (timeParams != null) {
allParamMap.putAll(timeParams);
}
allParamMap.putAll(globalMap);
Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
Map<String, String> resolveMap = new HashMap<>();
for (Map.Entry<String, String> entry : entries) {
String val = entry.getValue();
if (val.startsWith("$")) {
String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
resolveMap.put(entry.getKey(), str);
}
}
globalMap.putAll(resolveMap);
for (Property property : globalParamList) {
String val = globalMap.get(property.getProp());
if (val != null) {
property.setValue(val);
}
}
return JSONUtils.toJsonString(globalParamList);
}
allParamMap.putAll(globalMap);
Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
Map<String,String> resolveMap = new HashMap<>();
for (Map.Entry<String,String> entry : entries){
String val = entry.getValue();
if (val.startsWith("$")){
String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
resolveMap.put(entry.getKey(),str);
}
/**
* handle escapes
*
* @param inputString input string
* @return string filter escapes
*/
public static String handleEscapes(String inputString) {
if (StringUtils.isNotEmpty(inputString)) {
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
}
return inputString;
}
globalMap.putAll(resolveMap);
for (Property property : globalParamList){
String val = globalMap.get(property.getProp());
if (val != null){
property.setValue(val);
}
/**
* $[yyyyMMdd] replace schedule time
*/
public static String replaceScheduleTime(String text, Date scheduleTime) {
Map<String, Property> paramsMap = new HashMap<>();
//if getScheduleTime null ,is current date
if (null == scheduleTime) {
scheduleTime = new Date();
}
String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
Property p = new Property();
p.setValue(dateTime);
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
return text;
}
return JSONUtils.toJsonString(globalParamList);
}
/**
* handle escapes
* @param inputString input string
* @return string filter escapes
*/
public static String handleEscapes(String inputString){
if(StringUtils.isNotEmpty(inputString)){
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
}
return inputString;
}
/**
* $[yyyyMMdd] replace schedule time
* @param text
* @param scheduleTime
* @return
*/
public static String replaceScheduleTime(String text, Date scheduleTime) {
Map<String, Property> paramsMap = new HashMap<>();
//if getScheduleTime null ,is current date
if (null == scheduleTime) {
scheduleTime = new Date();
}
String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
Property p = new Property();
p.setValue(dateTime);
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
return text;
}
/**
* format convert
* @param paramsMap params map
* @return Map of converted
* see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
*/
public static Map<String,String> convert(Map<String,Property> paramsMap){
Map<String,String> map = new HashMap<>();
Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
while (iter.hasNext()){
Map.Entry<String, Property> en = iter.next();
map.put(en.getKey(),en.getValue().getValue());
/**
* format convert
*
* @param paramsMap params map
* @return Map of converted
* see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
*/
public static Map<String, String> convert(Map<String, Property> paramsMap) {
Map<String, String> map = new HashMap<>();
Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<String, Property> en = iter.next();
map.put(en.getKey(), en.getValue().getValue());
}
return map;
}
return map;
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java
浏览文件 @ ac4ed940
......@@ -14,16 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
/**
* utility methods for validating input
*
* utility methods for validating input
*/
public final class Preconditions {
private Preconditions() {}
private Preconditions() {
throw new UnsupportedOperationException("Construct Preconditions");
}
/**
* if obj is null will throw NPE
......@@ -41,32 +42,30 @@ public final class Preconditions {
/**
* if obj is null will throw NullPointerException with error message
*
* @param obj obj
* @param errorMsg error message
* @param <T> T
* @return T
*/
public static <T> T checkNotNull(T obj, String errorMsg) {
public static <T> T checkNotNull(T obj, String errorMsg) {
if (obj == null) {
throw new NullPointerException(errorMsg);
}
return obj;
}
/**
* if condition is false will throw an IllegalArgumentException with the given message
*
* @param condition condition
* @param errorMsg error message
*
* @param errorMsg error message
* @throws IllegalArgumentException Thrown, if the condition is violated.
*/
public static void checkArgument(boolean condition, Object errorMsg) {
public static void checkArgument(boolean condition, Object errorMsg) {
if (!condition) {
throw new IllegalArgumentException(String.valueOf(errorMsg));
}
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
浏览文件 @ ac4ed940
......@@ -14,13 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
......@@ -28,7 +30,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* property utils
......@@ -44,7 +47,7 @@ public class PropertyUtils {
private static final Properties properties = new Properties();
private PropertyUtils() {
throw new IllegalStateException("PropertyUtils class");
throw new UnsupportedOperationException("Construct PropertyUtils");
}
static {
......@@ -68,10 +71,9 @@ public class PropertyUtils {
}
/**
*
* @return judge whether resource upload startup
* @return judge whether resource upload startup
*/
public static Boolean getResUploadStartupState(){
public static Boolean getResUploadStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3;
......@@ -113,14 +115,13 @@ public class PropertyUtils {
* get property value
*
* @param key property name
* @return get property int value , if key == null, then return -1
* @return get property int value , if key == null, then return -1
*/
public static int getInt(String key) {
return getInt(key, -1);
}
/**
*
* @param key key
* @param defaultValue default value
* @return property value
......@@ -134,7 +135,7 @@ public class PropertyUtils {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return defaultValue;
}
......@@ -147,7 +148,7 @@ public class PropertyUtils {
*/
public static boolean getBoolean(String key) {
String value = properties.getProperty(key.trim());
if(null != value){
if (null != value) {
return Boolean.parseBoolean(value);
}
......@@ -163,7 +164,7 @@ public class PropertyUtils {
*/
public static Boolean getBoolean(String key, boolean defaultValue) {
String value = properties.getProperty(key.trim());
if(null != value){
if (null != value) {
return Boolean.parseBoolean(value);
}
......@@ -172,6 +173,7 @@ public class PropertyUtils {
/**
* get property long value
*
* @param key key
* @param defaultVal default value
* @return property value
......@@ -182,16 +184,14 @@ public class PropertyUtils {
}
/**
*
* @param key key
* @return property value
*/
public static long getLong(String key) {
return getLong(key,-1);
return getLong(key, -1);
}
/**
*
* @param key key
* @param defaultVal default value
* @return property value
......@@ -201,11 +201,11 @@ public class PropertyUtils {
return val == null ? defaultVal : Double.parseDouble(val);
}
/**
* get array
* @param key property name
* @param splitStr separator
* get array
*
* @param key property name
* @param splitStr separator
* @return property value through array
*/
public static String[] getArray(String key, String splitStr) {
......@@ -217,18 +217,17 @@ public class PropertyUtils {
String[] propertyArray = value.split(splitStr);
return propertyArray;
} catch (NumberFormatException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return new String[0];
}
/**
*
* @param key key
* @param type type
* @param defaultValue default value
* @param <T> T
* @return get enum value
* @return get enum value
*/
public <T extends Enum<T>> T getEnum(String key, Class<T> type,
T defaultValue) {
......@@ -238,6 +237,7 @@ public class PropertyUtils {
/**
* get all properties with specified prefix, like: fs.
*
* @param prefix prefix to search
* @return all properties with specified prefix
*/
......@@ -253,11 +253,9 @@ public class PropertyUtils {
/**
*
* @param key
* @param value
*/
public static void setValue(String key, String value) {
properties.setProperty(key,value);
properties.setProperty(key, value);
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java
浏览文件 @ ac4ed940
......@@ -17,15 +17,21 @@
package org.apache.dolphinscheduler.common.utils;
import com.github.rholder.retry.*;
import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
/**
* The Retryer util.
*/
......@@ -35,7 +41,7 @@ public class RetryerUtils {
private static Retryer<Boolean> defaultRetryerResultNoCheck;
private RetryerUtils() {
throw new UnsupportedOperationException("Construct RetryerUtils");
}
private static Retryer<Boolean> getDefaultRetryerResultNoCheck() {
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
浏览文件 @ ac4ed940
......@@ -14,10 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
package org.apache.dolphinscheduler.common.utils;
import java.io.File;
import java.io.FileInputStream;
......@@ -29,113 +27,123 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Metadata related common classes
*
*/
public class SchemaUtils {
private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
private static Pattern p = Pattern.compile("\\s*|\t|\r|\n");
/**
* Gets upgradable schemas for all upgrade directories
* @return all schema list
*/
@SuppressWarnings("unchecked")
public static List<String> getAllSchemaList() {
List<String> schemaDirList = new ArrayList<>();
File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade");
if(schemaDirArr == null || schemaDirArr.length == 0) {
return null;
}
for(File file : schemaDirArr) {
schemaDirList.add(file.getName());
}
Collections.sort(schemaDirList , new Comparator() {
@Override
public int compare(Object o1 , Object o2){
try {
String dir1 = String.valueOf(o1);
String dir2 = String.valueOf(o2);
String version1 = dir1.split("_")[0];
String version2 = dir2.split("_")[0];
if(version1.equals(version2)) {
return 0;
}
if(SchemaUtils.isAGreatVersion(version1, version2)) {
return 1;
}
return -1;
} catch (Exception e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e);
}
}
});
return schemaDirList;
}
/**
* Determine whether schemaVersion is higher than version
* @param schemaVersion schema version
* @param version version
* @return Determine whether schemaVersion is higher than version
*/
public static boolean isAGreatVersion(String schemaVersion, String version) {
if(StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) {
throw new RuntimeException("schemaVersion or version is empty");
}
String[] schemaVersionArr = schemaVersion.split("\\.");
String[] versionArr = version.split("\\.");
int arrLength = Math.min(schemaVersionArr.length, versionArr.length);
for(int i = 0 ; i < arrLength ; i++) {
if(Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) {
return true;
}else if(Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) {
return false;
}
}
// If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number
return schemaVersionArr.length > versionArr.length;
}
/**
* Gets the current software version number of the system
* @return current software version
*/
public static String getSoftVersion() {
String soft_version;
try {
soft_version = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version")));
soft_version = replaceBlank(soft_version);
} catch (FileNotFoundException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException("Failed to get the product version description file. The file could not be found", e);
}
return soft_version;
}
/**
* Strips the string of space carriage returns and tabs
* @param str string
* @return string removed blank
*/
public static String replaceBlank(String str) {
String dest = "";
if (str!=null) {
Matcher m = p.matcher(str);
dest = m.replaceAll("");
}
return dest;
}
private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
private static Pattern p = Pattern.compile("\\s*|\t|\r|\n");
private SchemaUtils() {
throw new UnsupportedOperationException("Construct SchemaUtils");
}
/**
* Gets upgradable schemas for all upgrade directories
*
* @return all schema list
*/
@SuppressWarnings("unchecked")
public static List<String> getAllSchemaList() {
List<String> schemaDirList = new ArrayList<>();
File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade");
if (schemaDirArr == null || schemaDirArr.length == 0) {
return null;
}
for (File file : schemaDirArr) {
schemaDirList.add(file.getName());
}
Collections.sort(schemaDirList, new Comparator() {
@Override
public int compare(Object o1, Object o2) {
try {
String dir1 = String.valueOf(o1);
String dir2 = String.valueOf(o2);
String version1 = dir1.split("_")[0];
String version2 = dir2.split("_")[0];
if (version1.equals(version2)) {
return 0;
}
if (SchemaUtils.isAGreatVersion(version1, version2)) {
return 1;
}
return -1;
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
});
return schemaDirList;
}
/**
* Determine whether schemaVersion is higher than version
*
* @param schemaVersion schema version
* @param version version
* @return Determine whether schemaVersion is higher than version
*/
public static boolean isAGreatVersion(String schemaVersion, String version) {
if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) {
throw new RuntimeException("schemaVersion or version is empty");
}
String[] schemaVersionArr = schemaVersion.split("\\.");
String[] versionArr = version.split("\\.");
int arrLength = Math.min(schemaVersionArr.length, versionArr.length);
for (int i = 0; i < arrLength; i++) {
if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) {
return true;
} else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) {
return false;
}
}
// If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number
return schemaVersionArr.length > versionArr.length;
}
/**
* Gets the current software version number of the system
*
* @return current software version
*/
public static String getSoftVersion() {
String softVersion;
try {
softVersion = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version")));
softVersion = replaceBlank(softVersion);
} catch (FileNotFoundException e) {
logger.error(e.getMessage(), e);
throw new RuntimeException("Failed to get the product version description file. The file could not be found", e);
}
return softVersion;
}
/**
* Strips the string of space carriage returns and tabs
*
* @param str string
* @return string removed blank
*/
public static String replaceBlank(String str) {
String dest = "";
if (str != null) {
Matcher m = p.matcher(str);
dest = m.replaceAll("");
}
return dest;
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
浏览文件 @ ac4ed940
......@@ -14,21 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
/**
* sensitive log Util
* sensitive log Util
*/
public class SensitiveLogUtils {
private SensitiveLogUtils() {
throw new UnsupportedOperationException("Construct SensitiveLogUtils");
}
/**
* @param dataSourcePwd data source password
* @return String
*/
public static String maskDataSourcePwd(String dataSourcePwd){
public static String maskDataSourcePwd(String dataSourcePwd) {
if (StringUtils.isNotEmpty(dataSourcePwd)) {
dataSourcePwd = Constants.PASSWORD_DEFAULT;
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java
浏览文件 @ ac4ed940
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.Iterator;
......@@ -22,7 +23,9 @@ import java.util.stream.StreamSupport;
public class StreamUtils {
private StreamUtils() { }
private StreamUtils() {
throw new UnsupportedOperationException("Construct StreamUtils");
}
public static <T> Stream<T> asStream(Iterator<T> sourceIterator) {
return asStream(sourceIterator, false);
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
浏览文件 @ ac4ed940
......@@ -14,11 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
public class StringUtils {
public static final String EMPTY = "";
private StringUtils() {
throw new UnsupportedOperationException("Construct StringUtils");
}
public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
......@@ -27,18 +33,18 @@ public class StringUtils {
return !isEmpty(cs);
}
public static boolean isBlank(String s){
public static boolean isBlank(String s) {
if (isEmpty(s)) {
return true;
}
return s.trim().length() == 0;
}
public static boolean isNotBlank(String s){
public static boolean isNotBlank(String s) {
return !isBlank(s);
}
public static String replaceNRTtoUnderline(String src){
public static String replaceNRTtoUnderline(String src) {
return src.replaceAll("[\n|\r|\t]", "_");
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java
浏览文件 @ ac4ed940
......@@ -14,13 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters;
......@@ -31,60 +32,65 @@ import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* task parameters utils
*/
public class TaskParametersUtils {
private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class);
private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class);
private TaskParametersUtils() {
throw new UnsupportedOperationException("Construct TaskParametersUtils");
}
/**
* get task parameters
* @param taskType task type
* @param parameter parameter
* @return task parameters
*/
public static AbstractParameters getParameters(String taskType, String parameter) {
try {
switch (EnumUtils.getEnum(TaskType.class,taskType)) {
case SUB_PROCESS:
return JSONUtils.parseObject(parameter, SubProcessParameters.class);
case WATERDROP:
return JSONUtils.parseObject(parameter, ShellParameters.class);
case SHELL:
return JSONUtils.parseObject(parameter, ShellParameters.class);
case PROCEDURE:
return JSONUtils.parseObject(parameter, ProcedureParameters.class);
case SQL:
return JSONUtils.parseObject(parameter, SqlParameters.class);
case MR:
return JSONUtils.parseObject(parameter, MapreduceParameters.class);
case SPARK:
return JSONUtils.parseObject(parameter, SparkParameters.class);
case PYTHON:
return JSONUtils.parseObject(parameter, PythonParameters.class);
case DEPENDENT:
return JSONUtils.parseObject(parameter, DependentParameters.class);
case FLINK:
return JSONUtils.parseObject(parameter, FlinkParameters.class);
case HTTP:
return JSONUtils.parseObject(parameter, HttpParameters.class);
case DATAX:
return JSONUtils.parseObject(parameter, DataxParameters.class);
case CONDITIONS:
return JSONUtils.parseObject(parameter, ConditionsParameters.class);
case SQOOP:
return JSONUtils.parseObject(parameter, SqoopParameters.class);
default:
return null;
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
/**
* get task parameters
*
* @param taskType task type
* @param parameter parameter
* @return task parameters
*/
public static AbstractParameters getParameters(String taskType, String parameter) {
try {
switch (EnumUtils.getEnum(TaskType.class, taskType)) {
case SUB_PROCESS:
return JSONUtils.parseObject(parameter, SubProcessParameters.class);
case WATERDROP:
return JSONUtils.parseObject(parameter, ShellParameters.class);
case SHELL:
return JSONUtils.parseObject(parameter, ShellParameters.class);
case PROCEDURE:
return JSONUtils.parseObject(parameter, ProcedureParameters.class);
case SQL:
return JSONUtils.parseObject(parameter, SqlParameters.class);
case MR:
return JSONUtils.parseObject(parameter, MapreduceParameters.class);
case SPARK:
return JSONUtils.parseObject(parameter, SparkParameters.class);
case PYTHON:
return JSONUtils.parseObject(parameter, PythonParameters.class);
case DEPENDENT:
return JSONUtils.parseObject(parameter, DependentParameters.class);
case FLINK:
return JSONUtils.parseObject(parameter, FlinkParameters.class);
case HTTP:
return JSONUtils.parseObject(parameter, HttpParameters.class);
case DATAX:
return JSONUtils.parseObject(parameter, DataxParameters.class);
case CONDITIONS:
return JSONUtils.parseObject(parameter, ConditionsParameters.class);
case SQOOP:
return JSONUtils.parseObject(parameter, SqoopParameters.class);
default:
return null;
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return null;
}
return null;
}
}
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或