Merge pull request #4006 from taosdata/release/s104

Release/s104

Jenkinsfile

@@ -9,7 +9,7 @@ pipeline {
       stage('Parallel test stage') {
       parallel {
         stage('pytest') {
-          agent{label 'master'}
+          agent{label '184'}
           steps {
             sh '''
             date
@@ -34,7 +34,7 @@ pipeline {
           }
         }
         stage('test_b1') {
-          agent{label '184'}
+          agent{label 'master'}
           steps {
             sh '''
             cd ${WKC}

alert/README.md

@@ -61,7 +61,7 @@ The use of each configuration item is:
 
 * **port**: This is the `http` service port which enables other application to manage rules by `restful API`.
 * **database**: rules are stored in a `sqlite` database, this is the path of the database file (if the file does not exist, the alert application creates it automatically).
-* **tdengine**: connection string of `TDEngine` server, note in most cases the database information should be put in a rule, thus it should NOT be included here.
+* **tdengine**: connection string of `TDEngine` server, note the database name should be put in the `sql` field of a rule in most cases, thus it should NOT be included in the string.
 * **log > level**: log level, could be `production` or `debug`.
 * **log > path**: log output file path.
 * **receivers > alertManager**: the alert application pushes alerts to `AlertManager` at this URL.

alert/README_cn.md

@@ -58,7 +58,7 @@ $ go build
 
 * **port**：报警监测程序支持使用 `restful API` 对规则进行管理，这个参数用于配置 `http` 服务的侦听端口。
 * **database**：报警监测程序将规则保存到了一个 `sqlite` 数据库中，这个参数用于指定数据库文件的路径（不需要提前创建这个文件，如果它不存在，程序会自动创建它）。
-* **tdengine**：`TDEngine` 的连接信息，一般来说，数据库信息应该在报警规则中指定，所以这里 **不** 应包含这一部分信息。
+* **tdengine**：`TDEngine` 的连接字符串，一般来说，数据库名应该在报警规则的 `sql` 语句中指定，所以这个字符串中 **不** 应包含数据库名。
 * **log > level**：日志的记录级别，可选 `production` 或 `debug`。
 * **log > path**：日志文件的路径。
 * **receivers > alertManager**：报警监测程序会将报警推送到 `AlertManager`，在这里指定 `AlertManager` 的接收地址。

documentation20/webdocs/markdowndocs/architecture-ch.md

@@ -228,7 +228,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久
 
 为充分利用时序数据特点，TDengine将一个vnode保存在持久化存储的数据切分成多个文件，每个文件只保存固定天数的数据，这个天数由系统配置参数days决定。切分成多个文件后，给定查询的起止日期，无需任何索引，就可以立即定位需要打开哪些数据文件，大大加快读取速度。
 
-对于采集的数据，一般有保留时长，这个时长由系统配置参数keep决定。超过这个设置天数的数据文件，将被系统将自动删除，释放存储空间。
+对于采集的数据，一般有保留时长，这个时长由系统配置参数keep决定。超过这个设置天数的数据文件，将被系统自动删除，释放存储空间。
 
 给定days与keep两个参数，一个vnode总的数据文件数为：keep/days。总的数据文件个数不宜过大，也不宜过小。10到100以内合适。基于这个原则，可以设置合理的days。 目前的版本，参数keep可以修改，但对于参数days，一但设置后，不可修改。
 

packaging/tools/makeclient_power.sh

@@ -123,7 +123,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp -r ${examples_dir}/R      ${install_dir}/examples
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/R/command.txt
   cp -r ${examples_dir}/go     ${install_dir}/examples  
-  sed -i '/root/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/go/src/taosapp/taosapp.go
+  sed -i '/root/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/go/taosdemo.go
 fi
 # Copy driver
 mkdir -p ${install_dir}/driver 

packaging/tools/makepkg_power.sh

@@ -146,7 +146,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp -r ${examples_dir}/R      ${install_dir}/examples
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/R/command.txt  
   cp -r ${examples_dir}/go     ${install_dir}/examples  
-  sed -i '/root/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/go/src/taosapp/taosapp.go
+  sed -i '/root/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/go/taosdemo.go
 fi
 # Copy driver
 mkdir -p ${install_dir}/driver 

packaging/tools/post.sh

@@ -10,6 +10,7 @@ data_dir="/var/lib/taos"
 log_dir="/var/log/taos"
 data_link_dir="/usr/local/taos/data"
 log_link_dir="/usr/local/taos/log"
+install_main_dir="/usr/local/taos"
 
 # static directory
 cfg_dir="/usr/local/taos/cfg"
@@ -134,6 +135,29 @@ function install_config() {
         else
             break
         fi
+    done		
+
+    # user email 
+    #EMAIL_PATTERN='^[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+$'
+    #EMAIL_PATTERN='^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$'
+    #EMAIL_PATTERN="^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$"
+    echo
+    echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: "
+    read emailAddr
+    while true; do
+        if [ ! -z "$emailAddr" ]; then
+            # check the format of the emailAddr
+            #if [[ "$emailAddr" =~ $EMAIL_PATTERN ]]; then
+                # Write the email address to temp file                    
+                email_file="${install_main_dir}/email" 
+                ${csudo} bash -c "echo $emailAddr > ${email_file}"
+                break         
+            #else
+            #    read -p "Please enter the correct email address: " emailAddr   
+            #fi
+        else
+            break
+        fi
     done	
 }
 

snap/snapcraft.yaml

 name: tdengine
 base: core18
-version: '2.0.5.1'
+version: '2.0.6.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.5.1
+      - usr/lib/libtaos.so.2.0.6.0
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/dnode/src/dnodeModule.c

@@ -97,7 +97,7 @@ void dnodeCleanupModules() {
     }
   }
 
-  if (tsModule[TSDB_MOD_MNODE].enable && tsModule[TSDB_MOD_MNODE].cleanUpFp) {
+  if (tsModule[TSDB_MOD_MNODE].cleanUpFp) {
     (*tsModule[TSDB_MOD_MNODE].cleanUpFp)();
   }
 }

src/dnode/src/dnodeVWrite.c

@@ -190,6 +190,7 @@ void dnodeFreeVnodeWqueue(void *wqueue) {
 
 void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
   SWriteMsg *pWrite = (SWriteMsg *)param;
+  if (pWrite == NULL) return;
 
   if (code < 0) pWrite->code = code;
   int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);

src/mnode/src/mnodeMain.c

@@ -123,16 +123,18 @@ int32_t mnodeInitSystem() {
 }
 
 void mnodeCleanupSystem() {
-  mInfo("starting to clean up mnode");
-  tsMgmtIsRunning = false;
-
-  dnodeFreeMnodeWqueue();
-  dnodeFreeMnodeRqueue();
-  dnodeFreeMnodePqueue();
-  mnodeCleanupTimer();
-  mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
-  
-  mInfo("mnode is cleaned up");
+  if (tsMgmtIsRunning) {
+    mInfo("starting to clean up mnode");
+    tsMgmtIsRunning = false;
+
+    dnodeFreeMnodeWqueue();
+    dnodeFreeMnodeRqueue();
+    dnodeFreeMnodePqueue();
+    mnodeCleanupTimer();
+    mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
+
+    mInfo("mnode is cleaned up");
+  }
 }
 
 void mnodeStopSystem() {

src/query/src/qExecutor.c

@@ -6771,7 +6771,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
 
   *buildRes = false;
   if (IS_QUERY_KILLED(pQInfo)) {
-    qDebug("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code);
+    qDebug("QInfo:%p query is killed, code:0x%08x", pQInfo, pQInfo->code);
     return pQInfo->code;
   }
 
@@ -7149,13 +7149,20 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
 void** qAcquireQInfo(void* pMgmt, uint64_t _key) {
   SQueryMgmt *pQueryMgmt = pMgmt;
 
-  if (pQueryMgmt->qinfoPool == NULL || pQueryMgmt->closed) {
+  if (pQueryMgmt->closed) {
+    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+    return NULL;
+  }
+
+  if (pQueryMgmt->qinfoPool == NULL) {
+    terrno = TSDB_CODE_QRY_INVALID_QHANDLE;
     return NULL;
   }
 
   TSDB_CACHE_PTR_TYPE key = (TSDB_CACHE_PTR_TYPE)_key;
   void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(TSDB_CACHE_PTR_TYPE));
   if (handle == NULL || *handle == NULL) {
+    terrno = TSDB_CODE_QRY_INVALID_QHANDLE;
     return NULL;
   } else {
     return handle;

src/rpc/src/rpcTcp.c

@@ -174,12 +174,15 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   pThreadObj->stop = true;
   eventfd_t fd = -1;
 
+  // save thread into local variable since pThreadObj is freed when thread exits
+  pthread_t thread = pThreadObj->thread; 
+
   if (taosComparePthread(pThreadObj->thread, pthread_self())) {
     pthread_detach(pthread_self());
     return;
   }
 
-  if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) {
+  if (taosCheckPthreadValid(pThreadObj->thread)) {
     // signal the thread to stop, try graceful method first,
     // and use pthread_cancel when failed
     struct epoll_event event = { .events = EPOLLIN };
@@ -196,8 +199,9 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
     }
   }
 
-  if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) {
-     pthread_join(pThreadObj->thread, NULL);
+  // at this step, pThreadObj has already been released
+  if (taosCheckPthreadValid(thread)) {
+     pthread_join(thread, NULL);
   }
 
   if (fd != -1) taosCloseSocket(fd);

src/vnode/src/vnodeRead.c

@@ -287,16 +287,22 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   memset(pRet, 0, sizeof(SRspRet));
 
+  terrno = TSDB_CODE_SUCCESS;
   int32_t code = TSDB_CODE_SUCCESS;
   void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
-  if (handle == NULL || (*handle) != (void *)pRetrieve->qhandle) {
+  if (handle == NULL) {
+    code = terrno;
+    terrno = TSDB_CODE_SUCCESS;
+  } else if ((*handle) != (void *)pRetrieve->qhandle) {
     code = TSDB_CODE_QRY_INVALID_QHANDLE;
-    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void *)pRetrieve->qhandle);
+  }
 
+  if (code != TSDB_CODE_SUCCESS) {
+    vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
     vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
-
+  
   if (pRetrieve->free == 1) {
     vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
     qKillQuery(*handle);

src/wal/src/walMain.c

@@ -60,6 +60,7 @@ static int  walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp);
 static int  walRemoveWalFiles(const char *path);
 static void walProcessFsyncTimer(void *param, void *tmrId);
 static void walRelease(SWal *pWal);
+static int walGetMaxOldFileId(char *odir);
 
 static void walModuleInitFunc() {
   walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL");
@@ -312,7 +313,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
     for (index = minId; index <= maxId; ++index) {
       snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index);
       terrno = walRestoreWalFile(pWal, pVnode, writeFp);
-      if (terrno < 0) break;
+      if (terrno < 0) continue;
     }
   }
 
@@ -476,31 +477,26 @@ int walHandleExistingFiles(const char *path) {
   int    plen = strlen(walPrefix);
   terrno = 0;
 
-  if (access(opath, F_OK) == 0) {
-    // old directory is there, it means restore process is not finished
-    walRemoveWalFiles(path);
-
-  } else {
-    // move all files to old directory
-    int count = 0;
-    while ((ent = readdir(dir)) != NULL) {
-      if (strncmp(ent->d_name, walPrefix, plen) == 0) {
-        snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
-        snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
-        if (taosMkDir(opath, 0755) != 0) {
-          wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
-          terrno = TAOS_SYSTEM_ERROR(errno);
-          break;
-        }
-
-        if (rename(oname, nname) < 0) {
-          wError("wal:%s, failed to move to new:%s", oname, nname);
-          terrno = TAOS_SYSTEM_ERROR(errno);
-          break;
-        }
+  int midx = walGetMaxOldFileId(opath);
+  int count = 0;
+  while ((ent = readdir(dir)) != NULL) {
+    if (strncmp(ent->d_name, walPrefix, plen) == 0) {
+      midx++;
+      snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
+      snprintf(nname, sizeof(nname), "%s/old/wal%d", path, midx);
+      if (taosMkDir(opath, 0755) != 0) {
+        wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        break;
+      }
 
-        count++;
+      if (rename(oname, nname) < 0) {
+        wError("wal:%s, failed to move to new:%s", oname, nname);
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        break;
       }
+
+      count++;
     }
 
     wDebug("wal:%s, %d files are moved for restoration", path, count);
@@ -563,4 +559,30 @@ int64_t walGetVersion(twalh param) {
   if (pWal == 0) return 0;
 
   return pWal->version;
+}
+
+static int walGetMaxOldFileId(char *odir) {
+  int            midx = 0;
+  DIR *          dir = NULL;
+  struct dirent *dp = NULL;
+  int            plen = strlen(walPrefix);
+
+  if (access(odir, F_OK) != 0) return midx;
+
+  dir = opendir(odir);
+  if (dir == NULL) {
+    wError("failed to open directory %s since %s", odir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  while ((dp = readdir(dir)) != NULL) {
+    if (strncmp(dp->d_name, walPrefix, plen) == 0) {
+      int idx = atol(dp->d_name + plen);
+      if (midx < idx) midx = idx;
+    }
+  }
+
+  closedir(dir);
+  return midx;
 }
\ No newline at end of file

tests/pytest/fulltest.sh

@@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 
+python3 ./test.py -f table/alter_wal0.py
 python3 ./test.py -f table/column_name.py
 python3 ./test.py -f table/column_num.py
 python3 ./test.py -f table/db_table.py

tests/pytest/insert/insertIntoTwoTables.py

@@ -39,7 +39,7 @@ class TDTestCase:
 
         tdSql.query("select * from t0")
         tdSql.checkRows(1)
-        tdSql.checkData(0, 0, 1)
+        tdSql.checkData(0, 1, 1)
 
         tdSql.query("select * from t1")
         tdSql.checkRows(5)

tests/pytest/tag_lite/datatype.py

@@ -103,7 +103,7 @@ class TDTestCase:
         tdSql.execute('alter table stb add tag tnc nchar(10)')
         for tid in range(1, self.ntables + 1):
             tdSql.execute('alter table tb%d set tag tnc=\"%s\"' %
-                          (tid, str(tid * 1.2)))
+                          (tid, str(tid + 1000000000)))
         tdLog.info("insert %d data in to each %d tables" % (2, self.ntables))
         for rid in range(self.rowsPerTable + 1, self.rowsPerTable + 3):
             sqlcmd = ['insert into']

tests/script/general/insert/insert_drop.sim

@@ -52,10 +52,10 @@ sleep 1000
 
 sql use $db
 sql drop table tb5
-
+$i = 0
 while $i < 4
-  $tbId = $i + $halfNum
-  $tb = $tbPrefix . $i
+  
+  $tb = tb . $i
   $x = 0
   while $x < $rowNum
     $xs = $x * $delta