diff --git a/Jenkinsfile b/Jenkinsfile index 08b6121f12916a8be712dbc4585f167d4dc679f0..ea50d6ef5a13fed2114868f8a36cd0acace78dd5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ pipeline { stage('Parallel test stage') { parallel { stage('pytest') { - agent{label 'master'} + agent{label '184'} steps { sh ''' date @@ -34,7 +34,7 @@ pipeline { } } stage('test_b1') { - agent{label '184'} + agent{label 'master'} steps { sh ''' cd ${WKC} diff --git a/alert/README.md b/alert/README.md index 23179669349ebf94f3774c6101c06c2e375ef059..547f3a0381a74714b1f6c8c74b861678b3805619 100644 --- a/alert/README.md +++ b/alert/README.md @@ -61,7 +61,7 @@ The use of each configuration item is: * **port**: This is the `http` service port which enables other application to manage rules by `restful API`. * **database**: rules are stored in a `sqlite` database, this is the path of the database file (if the file does not exist, the alert application creates it automatically). -* **tdengine**: connection string of `TDEngine` server, note in most cases the database information should be put in a rule, thus it should NOT be included here. +* **tdengine**: connection string of `TDEngine` server, note the database name should be put in the `sql` field of a rule in most cases, thus it should NOT be included in the string. * **log > level**: log level, could be `production` or `debug`. * **log > path**: log output file path. * **receivers > alertManager**: the alert application pushes alerts to `AlertManager` at this URL. diff --git a/alert/README_cn.md b/alert/README_cn.md index ec6e4566c8eb9fa2142c37b2b3e1b1a04f783a9a..938b23a58406f5d6f279191a47dc957c446911ce 100644 --- a/alert/README_cn.md +++ b/alert/README_cn.md @@ -58,7 +58,7 @@ $ go build * **port**:报警监测程序支持使用 `restful API` 对规则进行管理,这个参数用于配置 `http` 服务的侦听端口。 * **database**:报警监测程序将规则保存到了一个 `sqlite` 数据库中,这个参数用于指定数据库文件的路径(不需要提前创建这个文件,如果它不存在,程序会自动创建它)。 -* **tdengine**:`TDEngine` 的连接信息,一般来说,数据库信息应该在报警规则中指定,所以这里 **不** 应包含这一部分信息。 +* **tdengine**:`TDEngine` 的连接字符串,一般来说,数据库名应该在报警规则的 `sql` 语句中指定,所以这个字符串中 **不** 应包含数据库名。 * **log > level**:日志的记录级别,可选 `production` 或 `debug`。 * **log > path**:日志文件的路径。 * **receivers > alertManager**:报警监测程序会将报警推送到 `AlertManager`,在这里指定 `AlertManager` 的接收地址。 diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index ba45bc4796e6f0fb544f7383f74528ee486d94e5..d4705ccb05c092d8da38072368a167466bd78968 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -228,7 +228,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久 为充分利用时序数据特点,TDengine将一个vnode保存在持久化存储的数据切分成多个文件,每个文件只保存固定天数的数据,这个天数由系统配置参数days决定。切分成多个文件后,给定查询的起止日期,无需任何索引,就可以立即定位需要打开哪些数据文件,大大加快读取速度。 -对于采集的数据,一般有保留时长,这个时长由系统配置参数keep决定。超过这个设置天数的数据文件,将被系统将自动删除,释放存储空间。 +对于采集的数据,一般有保留时长,这个时长由系统配置参数keep决定。超过这个设置天数的数据文件,将被系统自动删除,释放存储空间。 给定days与keep两个参数,一个vnode总的数据文件数为:keep/days。总的数据文件个数不宜过大,也不宜过小。10到100以内合适。基于这个原则,可以设置合理的days。 目前的版本,参数keep可以修改,但对于参数days,一但设置后,不可修改。 diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index b4416a68bb30751d5e9b02f5e83186d750d5a935..faa5a03f52b4c9b56981a6b1c0918e543262b3bb 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -123,7 +123,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/R ${install_dir}/examples sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/R/command.txt cp -r ${examples_dir}/go ${install_dir}/examples - sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/src/taosapp/taosapp.go + sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/taosdemo.go fi # Copy driver mkdir -p ${install_dir}/driver diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 3d625900c9d912ff835092c7c5675d618b42b06d..2c02b99787c6d5ad6234de2319bf78b0b09d7e8a 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -146,7 +146,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/R ${install_dir}/examples sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/R/command.txt cp -r ${examples_dir}/go ${install_dir}/examples - sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/src/taosapp/taosapp.go + sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/taosdemo.go fi # Copy driver mkdir -p ${install_dir}/driver diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 0feb64c795159b67920c8a39b53b0125dfb565bf..d91daaa5c44488e34dea7ec2ddec0863699446f2 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -10,6 +10,7 @@ data_dir="/var/lib/taos" log_dir="/var/log/taos" data_link_dir="/usr/local/taos/data" log_link_dir="/usr/local/taos/log" +install_main_dir="/usr/local/taos" # static directory cfg_dir="/usr/local/taos/cfg" @@ -134,6 +135,29 @@ function install_config() { else break fi + done + + # user email + #EMAIL_PATTERN='^[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+$' + #EMAIL_PATTERN='^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$' + #EMAIL_PATTERN="^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$" + echo + echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: " + read emailAddr + while true; do + if [ ! -z "$emailAddr" ]; then + # check the format of the emailAddr + #if [[ "$emailAddr" =~ $EMAIL_PATTERN ]]; then + # Write the email address to temp file + email_file="${install_main_dir}/email" + ${csudo} bash -c "echo $emailAddr > ${email_file}" + break + #else + # read -p "Please enter the correct email address: " emailAddr + #fi + else + break + fi done } diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b0d8d8983bab473b8711d19805cea0fef2f10ee7..233b7a15b4a9383ee65dac8c9c0dc107fb66dd0a 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.5.1' +version: '2.0.6.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.5.1 + - usr/lib/libtaos.so.2.0.6.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 001c73eb3946fe46984c25758d9e0ac2a678dd37..46376159c6782efde7adbd19c75af83aa4cde397 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -97,7 +97,7 @@ void dnodeCleanupModules() { } } - if (tsModule[TSDB_MOD_MNODE].enable && tsModule[TSDB_MOD_MNODE].cleanUpFp) { + if (tsModule[TSDB_MOD_MNODE].cleanUpFp) { (*tsModule[TSDB_MOD_MNODE].cleanUpFp)(); } } diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index e56bae0d7e84953af493176a585ba3e7060b751b..f2740bf6b810197283926a602db4fc423067e231 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -190,6 +190,7 @@ void dnodeFreeVnodeWqueue(void *wqueue) { void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) { SWriteMsg *pWrite = (SWriteMsg *)param; + if (pWrite == NULL) return; if (code < 0) pWrite->code = code; int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index ea2ac9bf90ed6669c01402c78e11c23bd200dc3d..2bb8a810566a676a4ae991bc79e5eb78234747c7 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -123,16 +123,18 @@ int32_t mnodeInitSystem() { } void mnodeCleanupSystem() { - mInfo("starting to clean up mnode"); - tsMgmtIsRunning = false; - - dnodeFreeMnodeWqueue(); - dnodeFreeMnodeRqueue(); - dnodeFreeMnodePqueue(); - mnodeCleanupTimer(); - mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); - - mInfo("mnode is cleaned up"); + if (tsMgmtIsRunning) { + mInfo("starting to clean up mnode"); + tsMgmtIsRunning = false; + + dnodeFreeMnodeWqueue(); + dnodeFreeMnodeRqueue(); + dnodeFreeMnodePqueue(); + mnodeCleanupTimer(); + mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); + + mInfo("mnode is cleaned up"); + } } void mnodeStopSystem() { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3cb8db9fafe4ea8fa985377424152c7d0daecf12..5ad52ef29e6ba0a4b47129db891c0d1318e76c5f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6771,7 +6771,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex *buildRes = false; if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); + qDebug("QInfo:%p query is killed, code:0x%08x", pQInfo, pQInfo->code); return pQInfo->code; } @@ -7149,13 +7149,20 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { void** qAcquireQInfo(void* pMgmt, uint64_t _key) { SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL || pQueryMgmt->closed) { + if (pQueryMgmt->closed) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + if (pQueryMgmt->qinfoPool == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; return NULL; } TSDB_CACHE_PTR_TYPE key = (TSDB_CACHE_PTR_TYPE)_key; void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(TSDB_CACHE_PTR_TYPE)); if (handle == NULL || *handle == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; return NULL; } else { return handle; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 0b9bbae92eef3942b7a930f30650c412ce2eba46..bc8d360d39509ce9a2fdbe6a9dd883c5c5c99190 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -174,12 +174,15 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) { pThreadObj->stop = true; eventfd_t fd = -1; + // save thread into local variable since pThreadObj is freed when thread exits + pthread_t thread = pThreadObj->thread; + if (taosComparePthread(pThreadObj->thread, pthread_self())) { pthread_detach(pthread_self()); return; } - if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) { + if (taosCheckPthreadValid(pThreadObj->thread)) { // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed struct epoll_event event = { .events = EPOLLIN }; @@ -196,8 +199,9 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) { } } - if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) { - pthread_join(pThreadObj->thread, NULL); + // at this step, pThreadObj has already been released + if (taosCheckPthreadValid(thread)) { + pthread_join(thread, NULL); } if (fd != -1) taosCloseSocket(fd); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 1da84bf9b41c1bfc011a97cb0c06a71795a204b0..99aed03e54ccd069e5879104f62eb01ff7bb3d05 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -287,16 +287,22 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { memset(pRet, 0, sizeof(SRspRet)); + terrno = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS; void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle); - if (handle == NULL || (*handle) != (void *)pRetrieve->qhandle) { + if (handle == NULL) { + code = terrno; + terrno = TSDB_CODE_SUCCESS; + } else if ((*handle) != (void *)pRetrieve->qhandle) { code = TSDB_CODE_QRY_INVALID_QHANDLE; - vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void *)pRetrieve->qhandle); + } + if (code != TSDB_CODE_SUCCESS) { + vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); vnodeBuildNoResultQueryRsp(pRet); return code; } - + if (pRetrieve->free == 1) { vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index d7fd1b84c97b94730971277a699c40e3252c335f..182600204259e703d171d6598f46a2a16cdcb27b 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -60,6 +60,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp); static int walRemoveWalFiles(const char *path); static void walProcessFsyncTimer(void *param, void *tmrId); static void walRelease(SWal *pWal); +static int walGetMaxOldFileId(char *odir); static void walModuleInitFunc() { walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL"); @@ -312,7 +313,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) for (index = minId; index <= maxId; ++index) { snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index); terrno = walRestoreWalFile(pWal, pVnode, writeFp); - if (terrno < 0) break; + if (terrno < 0) continue; } } @@ -476,31 +477,26 @@ int walHandleExistingFiles(const char *path) { int plen = strlen(walPrefix); terrno = 0; - if (access(opath, F_OK) == 0) { - // old directory is there, it means restore process is not finished - walRemoveWalFiles(path); - - } else { - // move all files to old directory - int count = 0; - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name); - snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name); - if (taosMkDir(opath, 0755) != 0) { - wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (rename(oname, nname) < 0) { - wError("wal:%s, failed to move to new:%s", oname, nname); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } + int midx = walGetMaxOldFileId(opath); + int count = 0; + while ((ent = readdir(dir)) != NULL) { + if (strncmp(ent->d_name, walPrefix, plen) == 0) { + midx++; + snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name); + snprintf(nname, sizeof(nname), "%s/old/wal%d", path, midx); + if (taosMkDir(opath, 0755) != 0) { + wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } - count++; + if (rename(oname, nname) < 0) { + wError("wal:%s, failed to move to new:%s", oname, nname); + terrno = TAOS_SYSTEM_ERROR(errno); + break; } + + count++; } wDebug("wal:%s, %d files are moved for restoration", path, count); @@ -563,4 +559,30 @@ int64_t walGetVersion(twalh param) { if (pWal == 0) return 0; return pWal->version; +} + +static int walGetMaxOldFileId(char *odir) { + int midx = 0; + DIR * dir = NULL; + struct dirent *dp = NULL; + int plen = strlen(walPrefix); + + if (access(odir, F_OK) != 0) return midx; + + dir = opendir(odir); + if (dir == NULL) { + wError("failed to open directory %s since %s", odir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + while ((dp = readdir(dir)) != NULL) { + if (strncmp(dp->d_name, walPrefix, plen) == 0) { + int idx = atol(dp->d_name + plen); + if (midx < idx) midx = idx; + } + } + + closedir(dir); + return midx; } \ No newline at end of file diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index bdebe9a1ade22ae362961508666a808af8d1b1c4..39d0fa3d94d36aab88366cc9f428a08f6fd3d1dc 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py +python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py diff --git a/tests/pytest/insert/insertIntoTwoTables.py b/tests/pytest/insert/insertIntoTwoTables.py index dc985fbe12efb5cfb7e511336e573a18abcaa358..8b4f423c3dae38a0d8218f7b113f2784259300b4 100644 --- a/tests/pytest/insert/insertIntoTwoTables.py +++ b/tests/pytest/insert/insertIntoTwoTables.py @@ -39,7 +39,7 @@ class TDTestCase: tdSql.query("select * from t0") tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) tdSql.query("select * from t1") tdSql.checkRows(5) diff --git a/tests/pytest/tag_lite/datatype.py b/tests/pytest/tag_lite/datatype.py index f7fa9fa3a2dddaa3f352d02f74872ba42bd32f44..40f0ed7a090683faf149e56620ff63db7a52acb3 100644 --- a/tests/pytest/tag_lite/datatype.py +++ b/tests/pytest/tag_lite/datatype.py @@ -103,7 +103,7 @@ class TDTestCase: tdSql.execute('alter table stb add tag tnc nchar(10)') for tid in range(1, self.ntables + 1): tdSql.execute('alter table tb%d set tag tnc=\"%s\"' % - (tid, str(tid * 1.2))) + (tid, str(tid + 1000000000))) tdLog.info("insert %d data in to each %d tables" % (2, self.ntables)) for rid in range(self.rowsPerTable + 1, self.rowsPerTable + 3): sqlcmd = ['insert into'] diff --git a/tests/script/general/insert/insert_drop.sim b/tests/script/general/insert/insert_drop.sim index 80c16ff8e4c0de45bbc6e43991a96b75b6c924d3..9b68e5a6a6c7e77fc0e591e4950500369a88a650 100644 --- a/tests/script/general/insert/insert_drop.sim +++ b/tests/script/general/insert/insert_drop.sim @@ -52,10 +52,10 @@ sleep 1000 sql use $db sql drop table tb5 - +$i = 0 while $i < 4 - $tbId = $i + $halfNum - $tb = $tbPrefix . $i + + $tb = tb . $i $x = 0 while $x < $rowNum $xs = $x * $delta