[1.3.7-prepare][Improvement] Impove text description, OSUtilsTest and reduce...

[1.3.7-prepare][Improvement] Impove text description, OSUtilsTest and reduce heap memory for create/upgrade-dolphinscheduler.sh (#5890)

* Rename NORAML_NODE_STATUS to NORMAL_NODE_STATUS

* Fix OSUtilsTest

* Optimize comments in properties

* Reduce heap size to 64m in create-dolphinscheduler.sh and upgrade-dolphinscheduler.sh

* [Improvement][License] Update oshi version in release-docs/LICENSE

* Add test prefix for test method

ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml

@@ -30,7 +30,7 @@
 		<value-attributes>
 			<type>int</type>
 		</value-attributes>
-		<description>master execute thread number to limit process instances</description>
+		<description>master execute thread number to limit process instances in parallel</description>
 		<on-ambari-upgrade add="true"/>
 	</property>
 	<property>

ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml

@@ -30,7 +30,7 @@
 		<value-attributes>
 			<type>int</type>
 		</value-attributes>
-		<description>worker execute thread number to limit task instances</description>
+		<description>worker execute thread number to limit task instances in parallel</description>
 		<on-ambari-upgrade add="true"/>
 	</property>
 	<property>

docker/build/conf/dolphinscheduler/master.properties.tpl

@@ -18,7 +18,7 @@
 # master listen port
 #master.listen.port=5678
 
-# master execute thread number to limit process instances
+# master execute thread number to limit process instances in parallel
 master.exec.threads=${MASTER_EXEC_THREADS}
 
 # master execute task number in parallel per process instance

docker/build/conf/dolphinscheduler/worker.properties.tpl

@@ -18,7 +18,7 @@
 # worker listener port
 #worker.listen.port=1234
 
-# worker execute thread number to limit task instances
+# worker execute thread number to limit task instances in parallel
 worker.exec.threads=${WORKER_EXEC_THREADS}
 
 # worker heartbeat interval, the unit is second

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -995,7 +995,7 @@ public final class Constants {
      */
     public static final String PLUGIN_JAR_SUFFIX = ".jar";
 
-    public static final int NORAML_NODE_STATUS = 0;
+    public static final int NORMAL_NODE_STATUS = 0;
     public static final int ABNORMAL_NODE_STATUS = 1;
 
     public static final String START_TIME = "start time";

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@@ -88,7 +88,7 @@ public class OSUtils {
    */
   public static double availablePhysicalMemorySize() {
     GlobalMemory memory = hal.getMemory();
-    double  availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
+    double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
 
     DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
     df.setRoundingMode(RoundingMode.HALF_UP);
@@ -101,9 +101,9 @@ public class OSUtils {
    * Keep 2 decimal
    * @return  available Physical Memory Size, unit: G
    */
-  public static double totalMemorySize() {
+  public static double totalPhysicalMemorySize() {
     GlobalMemory memory = hal.getMemory();
-    double  totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
+    double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
 
     DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
     df.setRoundingMode(RoundingMode.HALF_UP);

dolphinscheduler-common/src/main/resources/common.properties

@@ -39,7 +39,7 @@ login.user.keytab.path=/opt/hdfs.headless.keytab
 # kerberos expire time, the unit is hour
 kerberos.expire.time=2
 
-# resource view suffixss
+# resource view suffixs
 #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
 
 # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java

@@ -19,15 +19,11 @@ package org.apache.dolphinscheduler.common.os;
 
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 
-import java.math.RoundingMode;
-import java.text.DecimalFormat;
-
+import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import oshi.hardware.GlobalMemory;
-
 /**
  * OSUtilsTest
  */
@@ -36,40 +32,33 @@ public class OSUtilsTest {
     private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class);
 
     @Test
-    public void memoryUsage() {
-        logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239
+    public void testMemoryUsage() {
+        double memoryUsage = OSUtils.memoryUsage();
+        logger.info("memoryUsage : {}", memoryUsage);
+        Assert.assertTrue(memoryUsage >= 0.0);
     }
 
     @Test
-    public void availablePhysicalMemorySize() {
-        logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize());
-        logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10);
+    public void testPhysicalMemorySize() {
+        double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
+        double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
+        logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize);
+        logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize);
+        Assert.assertTrue(availablePhysicalMemorySize >= 0.0);
+        Assert.assertTrue(totalPhysicalMemorySize >= 0.0);
     }
 
-
     @Test
-    public void loadAverage() {
-        logger.info("memoryUsage : {}", OSUtils.loadAverage());
-    }
-
-
-    private void printMemory(GlobalMemory memory) {
-        logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() );
+    public void testLoadAverage() {
+        double loadAverage = OSUtils.loadAverage();
+        logger.info("loadAverage : {}", loadAverage);
+        Assert.assertTrue(loadAverage >= 0.0);
     }
 
-
     @Test
-    public void cpuUsage() throws Exception {
-        logger.info("cpuUsage : {}", OSUtils.cpuUsage());
-        Thread.sleep(1000L);
-        logger.info("cpuUsage : {}", OSUtils.cpuUsage());
-
+    public void testCpuUsage() {
         double cpuUsage = OSUtils.cpuUsage();
-
-        DecimalFormat df = new DecimalFormat("0.00");
-
-        df.setRoundingMode(RoundingMode.HALF_UP);
-
-        logger.info("cpuUsage1 : {}", df.format(cpuUsage));
+        logger.info("cpuUsage : {}", cpuUsage);
+        Assert.assertTrue(cpuUsage >= 0.0);
     }
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

@@ -38,8 +38,8 @@ public class OSUtilsTest {
     public void testOSMetric(){
         double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
         Assert.assertTrue(availablePhysicalMemorySize > 0.0f);
-        double totalMemorySize = OSUtils.totalMemorySize();
-        Assert.assertTrue(totalMemorySize > 0.0f);
+        double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
+        Assert.assertTrue(totalPhysicalMemorySize > 0.0f);
         double loadAverage = OSUtils.loadAverage();
         logger.info("loadAverage {}", loadAverage);
         double memoryUsage = OSUtils.memoryUsage();

dolphinscheduler-dist/release-docs/LICENSE

@@ -443,7 +443,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     core-3.1.1 https://mvnrepository.com/artifact/org.eclipse.jdt/core/3.1.1 Eclipse Public License v1.0
     logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1
     logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1
-    oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0
+    oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0
     junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0
     h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java

@@ -75,7 +75,7 @@ public class HeartBeatTask implements Runnable {
 
             double loadAverage = OSUtils.loadAverage();
             double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
-            int status = Constants.NORAML_NODE_STATUS;
+            int status = Constants.NORMAL_NODE_STATUS;
             if (loadAverage > maxCpuloadAvg || availablePhysicalMemorySize < reservedMemory) {
                 logger.warn("current cpu load average {} is too high or available memory {}G is too low, under max.cpuload.avg={} and reserved.memory={}G",
                         loadAverage, availablePhysicalMemorySize, maxCpuloadAvg, reservedMemory);

dolphinscheduler-server/src/main/resources/master.properties

@@ -18,7 +18,7 @@
 # master listen port
 #master.listen.port=5678
 
-# master execute thread number to limit process instances
+# master execute thread number to limit process instances in parallel
 #master.exec.threads=100
 
 # master execute task number in parallel per process instance

dolphinscheduler-server/src/main/resources/worker.properties

@@ -18,7 +18,7 @@
 # worker listener port
 #worker.listen.port=1234
 
-# worker execute thread number to limit task instances
+# worker execute thread number to limit task instances in parallel
 #worker.exec.threads=100
 
 # worker heartbeat interval, the unit is second

script/create-dolphinscheduler.sh

@@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
 export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
 export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
 
-export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
 export STOP_TIMEOUT=5
 
 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler

script/upgrade-dolphinscheduler.sh

@@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
 export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
 export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
 
-export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
 export STOP_TIMEOUT=5
 
 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler