diff --git a/README.md b/README.md index 96cef561d71c50cc6688757e7f7c8b8cb6a9790a..23fbd6ee7db9a8a3658a275d85046f4dc5714f16 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Easy Scheduler +Easy Scheduler ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java b/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java index 5cb3b338535461635f691351383fd9f97b0e6436..916e4f37922abc91330a699cd7b73481e666b5f3 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java @@ -230,6 +230,8 @@ public class QuartzExecutors { if(scheduler.checkExists(jobKey)){ logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); return scheduler.deleteJob(jobKey); + }else { + return true; } } catch (SchedulerException e) { diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 291c59ea22348ef89c29f0ed2193e6feb4699f67..f1c850a87a862e33d3bbcbf68f58608960abda08 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -569,6 +569,7 @@ public class ResourcesService extends BaseService { * @param resourceId * @return */ + @Transactional(value = "TransactionManager",rollbackFor = Exception.class) public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); @@ -597,6 +598,10 @@ public class ResourcesService extends BaseService { } } + resource.setSize(content.getBytes().length); + resource.setUpdateTime(new Date()); + resourcesMapper.update(resource); + User user = userMapper.queryDetailsById(resource.getUserId()); String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); @@ -643,6 +648,7 @@ public class ResourcesService extends BaseService { logger.error("{} is not exist", resourcePath); result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); result.setMsg(String.format("%s is not exist", resourcePath)); + return result; } } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java index 4e589c37fa9f0f993af76e5d2d15a6f6600105b8..0b4f566ae529df46acc29cc22d942a63089e01c4 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java @@ -95,6 +95,9 @@ public class DependentUtils { case "last7Days": result = DependentDateUtils.getLastDayInterval(businessDate, 7); break; + case "thisWeek": + result = DependentDateUtils.getThisWeekInterval(businessDate); + break; case "lastWeek": result = DependentDateUtils.getLastWeekInterval(businessDate); break; @@ -119,6 +122,9 @@ public class DependentUtils { case "lastSunday": result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 7); break; + case "thisMonth": + result = DependentDateUtils.getThisMonthInterval(businessDate); + break; case "lastMonth": result = DependentDateUtils.getLastMonthInterval(businessDate); break; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java index d8dcc621a8d9f7fa242c3c596e3142610dac6613..0cf06d3b02a63eef7106412a3aa55339b68d902a 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java @@ -220,7 +220,7 @@ public class OSUtils { * @throws IOException */ public static String exeShell(String command) throws IOException { - return ShellExecutor.execCommand("groups"); + return ShellExecutor.execCommand(command); } /** diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java index df79dd163af6d5b0f77ed2bdc682a3f8e2e48d89..0127fe2ca98300cbf5283142d9cb35099e204a2d 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java @@ -76,6 +76,16 @@ public class DependentDateUtils { return dateIntervals; } + /** + * get interval between this month first day and businessDate + * @param businessDate + * @return + */ + public static List getThisMonthInterval(Date businessDate) { + Date firstDay = DateUtils.getFirstDayOfMonth(businessDate); + return getDateIntervalListBetweenTwoDates(firstDay, businessDate); + } + /** * get interval between last month first day and last day * @param businessDate @@ -108,6 +118,16 @@ public class DependentDateUtils { } } + /** + * get interval between monday to businessDate of this week + * @param businessDate + * @return + */ + public static List getThisWeekInterval(Date businessDate) { + Date mondayThisWeek = DateUtils.getMonday(businessDate); + return getDateIntervalListBetweenTwoDates(mondayThisWeek, businessDate); + } + /** * get interval between monday to sunday of last week * default set monday the first day of week diff --git a/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java b/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java index a0569b02ee4c1eab2390dcb48260291455601c2e..8680ab0cb4c7a838cb02bd4c9aedc302c465fd0f 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java @@ -80,6 +80,26 @@ public class DependentUtilsTest { Assert.assertEquals(dateIntervals.get(0), diCur); + dateValue = "thisWeek"; + Date firstWeekDay = DateUtils.getMonday(curDay); + dateIntervals = DependentUtils.getDateIntervalList(curDay, dateValue); + + DateInterval weekHead = new DateInterval(DateUtils.getStartOfDay(firstWeekDay), DateUtils.getEndOfDay(firstWeekDay)); + DateInterval weekThis = new DateInterval(DateUtils.getStartOfDay(curDay), DateUtils.getEndOfDay(curDay)); + + Assert.assertEquals(dateIntervals.get(0), weekHead); + Assert.assertEquals(dateIntervals.get(dateIntervals.size() - 1), weekThis); + + + dateValue = "thisMonth"; + Date firstMonthDay = DateUtils.getFirstDayOfMonth(curDay); + dateIntervals = DependentUtils.getDateIntervalList(curDay, dateValue); + + DateInterval monthHead = new DateInterval(DateUtils.getStartOfDay(firstMonthDay), DateUtils.getEndOfDay(firstMonthDay)); + DateInterval monthThis = new DateInterval(DateUtils.getStartOfDay(curDay), DateUtils.getEndOfDay(curDay)); + + Assert.assertEquals(dateIntervals.get(0), monthHead); + Assert.assertEquals(dateIntervals.get(dateIntervals.size() - 1), monthThis); } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java index 4122c7722c9d32146f34a1d0a71b8fd754df5b11..4314b8f58436906b442d643aa7cbe7ac5ff33001 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java @@ -118,6 +118,7 @@ public class ResourceMapperProvider { SET("`alias` = #{resource.alias}"); SET("`desc` = #{resource.desc}"); SET("`update_time` = #{resource.updateTime}"); + SET("`size` = #{resource.size}"); WHERE("`id` = #{resource.id}"); }}.toString(); } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index baf82de0dfdb745f0e4fbcc6ab18c4d53ced18f4..0c3aec602bdbddce9e82dcbe9e536cdbfd6b7606 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -18,6 +18,7 @@ package cn.escheduler.server.utils; import cn.escheduler.common.Constants; import cn.escheduler.common.utils.CommonUtils; +import cn.escheduler.common.utils.OSUtils; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.rpc.LogClient; import org.apache.commons.io.FileUtils; @@ -33,6 +34,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; + /** * mainly used to get the start command line of a process */ @@ -139,6 +141,8 @@ public class ProcessUtils { {' ', '\t', '<', '>'}, {' ', '\t'}}; + private static Matcher matcher; + private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) { StringBuilder cmdbuf = new StringBuilder(80); @@ -256,11 +260,11 @@ public class ProcessUtils { return ; } - String cmd = String.format("sudo kill -9 %d", processId); + String cmd = String.format("sudo kill -9 %s", getPidsStr(processId)); logger.info("process id:{}, cmd:{}", processId, cmd); - Runtime.getRuntime().exec(cmd); + OSUtils.exeCmd(cmd); // find log and kill yarn job killYarnJob(taskInstance); @@ -270,6 +274,23 @@ public class ProcessUtils { } } + /** + * get pids str + * @param processId + * @return + * @throws Exception + */ + private static String getPidsStr(int processId)throws Exception{ + StringBuilder sb = new StringBuilder(); + // pstree -p pid get sub pids + String pids = OSUtils.exeCmd("pstree -p " +processId+ ""); + Matcher mat = Pattern.compile("(\\d+)").matcher(pids); + while (mat.find()){ + sb.append(mat.group()+" "); + } + return sb.toString().trim(); + } + /** * find logs and kill yarn tasks * @param taskInstance diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java index 36c32c9713242faf1c7f13dd98914efb758af205..e3bd4013935c573a3ac930c9c237cffca6ac1133 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java @@ -213,7 +213,7 @@ public abstract class AbstractCommandExecutor { */ private int updateState(ProcessDao processDao, int exitStatusCode, int pid, int taskInstId) { //get yarn state by log - if (exitStatusCode != -1) { + if (exitStatusCode != 0) { TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); logger.info("process id is {}", pid); @@ -556,10 +556,4 @@ public abstract class AbstractCommandExecutor { protected abstract boolean checkShowLog(String line); protected abstract boolean checkFindApp(String line); protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException; - - - -// if(line.contains(taskAppId) || !line.contains("cn.escheduler.server.worker.log.TaskLogger")){ -// logs.add(line); -// } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 3e85b55b918f1132b604109b408e3d622a87e13e..bb57f60b6ab0377b28004fd8ad528c72991ab374 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -347,14 +347,14 @@ public class SqlTask extends AbstractTask { // receiving group list List receviersList = new ArrayList(); for(User user:users){ - receviersList.add(user.getEmail()); + receviersList.add(user.getEmail().trim()); } // custom receiver String receivers = sqlParameters.getReceivers(); if (StringUtils.isNotEmpty(receivers)){ String[] splits = receivers.split(Constants.COMMA); for (String receiver : splits){ - receviersList.add(receiver); + receviersList.add(receiver.trim()); } } @@ -365,7 +365,7 @@ public class SqlTask extends AbstractTask { if (StringUtils.isNotEmpty(receiversCc)){ String[] splits = receiversCc.split(Constants.COMMA); for (String receiverCc : splits){ - receviersCcList.add(receiverCc); + receviersCcList.add(receiverCc.trim()); } } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js index 6274533c1a9631444ad3337035f3abe6cb8da4d2..18979400648bd69bf0713fcbf0f87fa6455862ac 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js @@ -64,6 +64,10 @@ const dateValueList = { } ], 'week': [ + { + value: 'thisWeek', + label: `${i18n.$t('ThisWeek')}` + }, { value: 'lastWeek', label: `${i18n.$t('LastWeek')}` @@ -98,6 +102,10 @@ const dateValueList = { } ], 'month': [ + { + value: 'thisMonth', + label: `${i18n.$t('ThisMonth')}` + }, { value: 'lastMonth', label: `${i18n.$t('LastMonth')}` diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index 50c33c061b5feeba66b9915ef65ac831f106086c..fbc621e18ba3b59719228f000e5a416ca5ac76d5 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -395,6 +395,7 @@ export default { 'Last2Days': 'Last2Days', 'Last3Days': 'Last3Days', 'Last7Days': 'Last7Days', + 'ThisWeek': 'ThisWeek', 'LastWeek': 'LastWeek', 'LastMonday': 'LastMonday', 'LastTuesday': 'LastTuesday', @@ -403,6 +404,7 @@ export default { 'LastFriday': 'LastFriday', 'LastSaturday': 'LastSaturday', 'LastSunday': 'LastSunday', + 'ThisMonth': 'ThisMonth', 'LastMonth': 'LastMonth', 'LastMonthBegin': 'LastMonthBegin', 'LastMonthEnd': 'LastMonthEnd', diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index d8ea8230480c4b326fe3297c319e5b3ddd2b8fe0..d77c55715affb7897c7ab742a403299430eae369 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -395,6 +395,7 @@ export default { 'Last2Days': '前两天', 'Last3Days': '前三天', 'Last7Days': '前七天', + 'ThisWeek': '本周', 'LastWeek': '上周', 'LastMonday': '上周一', 'LastTuesday': '上周二', @@ -403,6 +404,7 @@ export default { 'LastFriday': '上周五', 'LastSaturday': '上周六', 'LastSunday': '上周日', + 'ThisMonth': '本月', 'LastMonth': '上月', 'LastMonthBegin': '上月初', 'LastMonthEnd': '上月末',