Skip to content

Incubator Pegasus
Incubator Pegasus

apache / Incubator Pegasus

通知 9
Star 5
Fork 0
Incubator Pegasus
Incubator Pegasus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 7cb96b40 编写于 作者: Q QinZuoyan 提交者： GitHub
浏览文件
操作

*: fix compile error of -Wuninitialized; add --compiler option for build; add... 

*: fix compile error of -Wuninitialized; add --compiler option for build; add --wait_healthy option for start_onebox; speed up build time; update rdsn (#83)
上级 1a2c4eb5
隐藏空白更改
内联 并排
Showing with 91 addition and 43 deletion
.travis.yml
浏览文件 @ 7cb96b40
......@@ -8,6 +8,9 @@ os: linux
compiler:
- gcc
cache:
- ccache
before_install:
- wget https://raw.githubusercontent.com/xiaomi/pegasus-common/master/build-depends.tar.gz
- tar xf build-depends.tar.gz
......
rdsn @ f7cad36a
比较 b35cf269...f7cad36a
Subproject commit b35cf26999c3155aeec7aacaac7b03f22ae35c44
Subproject commit f7cad36ad67bb0788cd77f481fbc6f8956c4165a
run.sh
浏览文件 @ 7cb96b40
......@@ -61,6 +61,9 @@ function usage_build()
echo " -c|--clear clear rdsn/rocksdb/pegasus before building, not clear thirdparty"
echo " -cc|--half-clear clear pegasus before building, not clear thirdparty/rdsn/rocksdb"
echo " --clear_thirdparty clear thirdparty/rdsn/rocksdb/pegasus before building"
echo " --compiler specify c and cxx compiler, sperated by ','"
echo " e.g., \"gcc,g++\" or \"clang-3.9,clang++-3.9\""
echo " default is \"gcc,g++\""
echo " -j|--jobs <num> the number of jobs to run simultaneously, default 8"
echo " -b|--boost_dir <dir> specify customized boost directory, use system boost if not set"
echo " -w|--warning_all open all warnings when building, default no"
......@@ -69,6 +72,8 @@ function usage_build()
}
function run_build()
{
C_COMPILER="gcc"
CXX_COMPILER="g++"
BUILD_TYPE="release"
CLEAR=NO
PART_CLEAR=NO
......@@ -99,6 +104,17 @@ function run_build()
--clear_thirdparty)
CLEAR_THIRDPARTY=YES
;;
--compiler)
C_COMPILER=`echo $2 | awk -F',' '{print $1}'`
CXX_COMPILER=`echo $2 | awk -F',' '{print $2}'`
if [ "x"$C_COMPILER == "x" -o "x"$CXX_COMPILER == "x" ]; then
echo "ERROR: invalid compiler option: $2"
echo
usage_build
exit 1
fi
shift
;;
-j|--jobs)
JOB_NUM="$2"
shift
......@@ -142,9 +158,12 @@ function run_build()
ln -sf $DSN_ROOT $ROOT/DSN_ROOT
fi
echo "Build start time: `date`"
start_time=`date +%s`
echo "INFO: start build rdsn..."
cd $ROOT/rdsn
OPT="-t $BUILD_TYPE -j $JOB_NUM"
OPT="-t $BUILD_TYPE -j $JOB_NUM --compiler $C_COMPILER,$CXX_COMPILER"
if [ "$BOOST_DIR" != "" ]; then
OPT="$OPT -b $BOOST_DIR"
fi
......@@ -162,7 +181,8 @@ function run_build()
echo "INFO: start build pegasus..."
cd $ROOT/src
BUILD_TYPE="$BUILD_TYPE" CLEAR="$CLEAR" PART_CLEAR="$PART_CLEAR" JOB_NUM="$JOB_NUM" \
C_COMPILER="$C_COMPILER" CXX_COMPILER="$CXX_COMPILER" BUILD_TYPE="$BUILD_TYPE" \
CLEAR="$CLEAR" PART_CLEAR="$PART_CLEAR" JOB_NUM="$JOB_NUM" \
BOOST_DIR="$BOOST_DIR" WARNING_ALL="$WARNING_ALL" ENABLE_GCOV="$ENABLE_GCOV" \
RUN_VERBOSE="$RUN_VERBOSE" TEST_MODULE="$TEST_MODULE" ./build.sh
if [ $? -ne 0 ]; then
......@@ -172,6 +192,11 @@ function run_build()
cd $ROOT
chmod +x scripts/*.sh
echo "Build finish time: `date`"
finish_time=`date +%s`
used_time=$((finish_time-start_time))
echo "Build elapsed time: $((used_time/60))m $((used_time%60))s"
}
#####################
......@@ -216,10 +241,11 @@ function run_test()
test_modules="pegasus_unit_test pegasus_function_test"
fi
echo "Test start time: `date`"
start_time=`date +%s`
./run.sh clear_onebox #clear the onebox before test
./run.sh start_onebox
echo "sleep 30 to wait for the onebox to start all partitions ..."
sleep 30
./run.sh start_onebox -w
for module in `echo $test_modules`; do
pushd $ROOT/src/builder/bin/$module
......@@ -234,6 +260,11 @@ function run_test()
if [ "$clear_flags" == "1" ]; then
./run.sh clear_onebox
fi
echo "Test finish time: `date`"
finish_time=`date +%s`
used_time=$((finish_time-start_time))
echo "Test elapsed time: $((used_time/60))m $((used_time%60))s"
}
#####################
......@@ -383,6 +414,8 @@ function usage_start_onebox()
echo " default app name, default is temp"
echo " -p|--partition_count <num>"
echo " default app partition count, default is 8"
echo " -w|--wait_healthy"
echo " wait cluster to become healthy, default not wait"
echo " -s|--server_path <str>"
echo " server binary path, default is ${DSN_ROOT}/bin/pegasus_server"
echo " --use_product_config"
......@@ -396,6 +429,7 @@ function run_start_onebox()
COLLECTOR_COUNT=0
APP_NAME=temp
PARTITION_COUNT=8
WAIT_HEALHY=false
SERVER_PATH=${DSN_ROOT}/bin/pegasus_server
USE_PRODUCT_CONFIG=false
while [[ $# > 0 ]]; do
......@@ -424,6 +458,9 @@ function run_start_onebox()
PARTITION_COUNT="$2"
shift
;;
-w|--wait_healthy)
WAIT_HEALHY=true
;;
-s|--server_path)
SERVER_PATH="$2"
shift
......@@ -523,6 +560,22 @@ function run_start_onebox()
ps -ef | grep '/pegasus_server config.ini' | grep "\<$PID\>"
cd ..
fi
if [ $WAIT_HEALHY == "true" ]; then
cd $ROOT
echo "Wait cluster to become healthy..."
sleeped=0
while true; do
sleep 1
sleeped=$((sleeped+1))
echo "Sleeped for $sleeped seconds"
unhealthy_count=`echo "ls -d" | ./run.sh shell | awk 'f{ if(NF<7){f=0} else if($3!=$4){print} } /fully_healthy/{print;f=1}' | wc -l`
if [ $unhealthy_count -eq 1 ]; then
echo "Cluster becomes healthy."
break
fi
done
fi
}
#####################
......@@ -615,6 +668,7 @@ function run_clear_onebox()
run_stop_onebox
run_clear_zk
rm -rf onebox *.log *.data config-*.ini &>/dev/null
sleep 1
}
#####################
......
src/build.sh
浏览文件 @ 7cb96b40
......@@ -5,6 +5,8 @@
# PART_CLEAR YES|NO
# JOB_NUM <num>
# BUILD_TYPE debug|release
# C_COMPILER <str>
# CXX_COMPILER <str>
# RUN_VERBOSE YES|NO
# WARNING_ALL YES|NO
# ENABLE_GCOV YES|NO
......@@ -12,8 +14,8 @@
# TEST_MODULE "<module1> <module2> ..."
#
# CMake options:
# -DCMAKE_C_COMPILER=gcc
# -DCMAKE_CXX_COMPILER=g++
# -DCMAKE_C_COMPILER=gcc|clang
# -DCMAKE_CXX_COMPILER=g++|clang++
# [-DCMAKE_BUILD_TYPE=Debug]
# [-DWARNING_ALL=TRUE]
# [-DENABLE_GCOV=TRUE]
......@@ -21,8 +23,13 @@
ROOT=`pwd`
BUILD_DIR="$ROOT/builder"
CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++"
MAKE_OPTIONS="$MAKE_OPTIONS"
echo "C_COMPILER=$C_COMPILER"
echo "CXX_COMPILER=$CXX_COMPILER"
CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER"
echo "JOB_NUM=$JOB_NUM"
MAKE_OPTIONS="$MAKE_OPTIONS -j$JOB_NUM"
if [ "$CLEAR" == "YES" ]
then
......@@ -38,15 +45,6 @@ else
echo "PART_CLEAR=NO"
fi
if [ -n "$JOB_NUM" ]
then
echo "JOB_NUM=$JOB_NUM"
MAKE_OPTIONS="$MAKE_OPTIONS -j$JOB_NUM"
else
echo "JOB_NUM=8"
MAKE_OPTIONS="$MAKE_OPTIONS -j8"
fi
if [ "$BUILD_TYPE" == "debug" ]
then
echo "BUILD_TYPE=debug"
......
src/server/config-server.ini
浏览文件 @ 7cb96b40
......@@ -245,7 +245,7 @@ log_shared_force_flush = false
config_sync_disabled = false
config_sync_interval_ms = 30000
lb_interval_ms = 10000
lb_interval_ms = 3000
;; the prefix of the path that to save backup-data on cold backup media
;; recommand using cluster name as the root
......
src/server/pegasus_server_impl.cpp
浏览文件 @ 7cb96b40
......@@ -1024,7 +1024,7 @@ void pegasus_server_impl::on_ttl(const ::dsn::blob &key,
std::string value;
rocksdb::Status status = _db->Get(_rd_opts, skey, &value);
uint32_t expire_ts;
uint32_t expire_ts = 0;
uint32_t now_ts = ::pegasus::utils::epoch_now();
if (status.ok()) {
expire_ts = pegasus_extract_expire_ts(_value_schema_version, value);
......
src/shell/commands.h
浏览文件 @ 7cb96b40
......@@ -3277,8 +3277,8 @@ inline bool restore(command_executor *e, shell_context *sc, arguments args)
std::string old_cluster_name, old_policy_name;
std::string old_app_name, new_app_name;
std::string backup_provider_type;
int32_t old_app_id;
int64_t timestamp;
int32_t old_app_id = 0;
int64_t timestamp = 0;
bool skip_bad_partition = false;
optind = 0;
......
src/test/function_test/config.ini
浏览文件 @ 7cb96b40
......@@ -65,6 +65,9 @@ rpc_call_header_format = NET_HDR_DSN
fast_execution_in_network_thread = false
rpc_timeout_milliseconds = 5000
[replication]
lb_interval_ms = 3000
[uri-resolver.dsn://mycluster]
factory = partition_resolver_simple
arguments = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603
......
src/test/function_test/test_recovery.cpp
浏览文件 @ 7cb96b40
......@@ -38,11 +38,9 @@ protected:
system("sed -i \"/^perf_counter_enable_logging/c perf_counter_enable_logging = false\" "
"src/server/config-server.ini");
std::cout << "sleep for a while to wait the onebox cleaned" << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(10));
system("./run.sh start_onebox -m 1 -r 3");
std::cout << "sleep for a while to wait the new onebox start" << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(5));
std::this_thread::sleep_for(std::chrono::seconds(3));
chdir(global_env::instance()._working_dir.c_str());
......@@ -78,9 +76,7 @@ protected:
chdir(global_env::instance()._pegasus_root.c_str());
system("./run.sh clear_onebox");
system("git checkout -- src/server/config-server.ini");
system("./run.sh start_onebox");
std::cout << "sleep for a while to wait the onebox to restart" << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(20));
system("./run.sh start_onebox -w");
chdir(global_env::instance()._working_dir.c_str());
}
......
src/test/function_test/test_restore.cpp
浏览文件 @ 7cb96b40
......@@ -29,9 +29,7 @@ public:
std::string cmd = "sed -i \"/^cold_backup_root/c cold_backup_root = " + cluster_name;
cmd = cmd + std::string("\" src/server/config-server.ini");
system(cmd.c_str());
std::this_thread::sleep_for(std::chrono::seconds(3));
system("./run.sh clear_onebox");
std::this_thread::sleep_for(std::chrono::seconds(3));
system("./run.sh start_onebox");
std::this_thread::sleep_for(std::chrono::seconds(3));
......@@ -72,11 +70,8 @@ public:
{
chdir(global_env::instance()._pegasus_root.c_str());
system("./run.sh clear_onebox");
std::this_thread::sleep_for(std::chrono::seconds(3));
system("git checkout -- src/server/config-server.ini");
system("./run.sh start_onebox");
std::cout << "sleep 10s to restart onebox" << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(10));
system("./run.sh start_onebox -w");
std::string cmd = "rm -rf " + backup_data_dir;
system(cmd.c_str());
chdir(global_env::instance()._working_dir.c_str());
......@@ -136,7 +131,6 @@ public:
bool restore()
{
system("./run.sh clear_onebox");
std::this_thread::sleep_for(std::chrono::seconds(3));
system("./run.sh start_onebox");
std::this_thread::sleep_for(std::chrono::seconds(3));
time_stamp = get_first_backup_timestamp();
......@@ -153,8 +147,8 @@ public:
std::cout << "restore failed, err = " << err.to_string() << std::endl;
return false;
} else {
// sleep for at most 2min to wait app is fully healthy
bool ret = wait_app_healthy(120);
// sleep for at most 3 min to wait app is fully healthy
bool ret = wait_app_healthy(180);
return ret;
}
}
......@@ -167,8 +161,8 @@ public:
error_code err = ERR_OK;
while (seconds > 0 && !is_app_full_healthy) {
int64_t sleep_time = 0;
if (seconds >= 10) {
sleep_time = 10;
if (seconds >= 3) {
sleep_time = 3;
} else {
sleep_time = seconds;
}
......@@ -216,8 +210,8 @@ public:
bool is_backup_complete = false;
while (seconds > 0 && !is_backup_complete) {
sleep_time = 0;
if (seconds >= 10) {
sleep_time = 10;
if (seconds >= 3) {
sleep_time = 3;
} else {
sleep_time = seconds;
}
......@@ -313,7 +307,7 @@ TEST_F(restore_test, restore)
{
std::cout << "start testing restore..." << std::endl;
// step1: wait backup complete
ASSERT_TRUE(wait_backup_complete(120));
ASSERT_TRUE(wait_backup_complete(180));
// step2: test restore
ASSERT_TRUE(restore());
// step3: verify_data
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或