Skip to content

Incubator Pegasus
Incubator Pegasus

apache / Incubator Pegasus

通知 9
Star 5
Fork 0
Incubator Pegasus
Incubator Pegasus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 ad2fffcf 编写于 作者: W Wu Tao 提交者： neverchanje
浏览文件
操作

chore: remove unmaintained scripts (#634)

上级 03e02d31
展开全部 隐藏空白更改
内联 并排
Showing with 0 addition and 2748 deletion
scripts/carrot 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/bin/bash
staging_branch="master"
project_name="pegasus"
command_decorator="verify"
function git_current_branch()
{
echo `git branch | fgrep "*" | cut -d " " -f 2`
}
function get_next_version()
{
versions=(`grep "PEGASUS_VERSION" src/include/pegasus/version.h | cut -d"\"" -f 2 | sed 's/\./ /g'`)
case $1 in
major)
versions[0]=$[ ${versions[0]} + 1 ]
;;
minor)
versions[1]=$[ ${versions[1]} + 1 ]
;;
patch)
if [ ${versions[2]} == "SNAPSHOT" ]; then
versions[2]="0"
else
versions[2]=$[ ${versions[2]} + 1 ]
fi
;;
*)
echo "Invalid next version type"
exit -1
;;
esac
echo ${versions[*]} | sed 's/ /\./g'
}
function get_current_version()
{
versions=(`grep "PEGASUS_VERSION" src/include/pegasus/version.h | cut -d"\"" -f 2 | sed 's/\./ /g'`)
case $1 in
major)
echo ${versions[0]}
;;
minor)
echo ${versions[0]}.${versions[1]}
;;
patch)
echo ${versions[*]} | sed 's/ /\./g'
;;
*)
echo "Invalid current version type"
exit -1
;;
esac
}
function get_branch_type()
{
if [ $1 = $staging_branch ]; then
echo "staging"
else
echo "release"
fi
}
function verify_command()
{
answer=""
echo -n -e "\033[31mExecuting command: $@, y/N?\033[0m"
read answer
if [ -z $answer ] || [ $answer = "y" ]; then
eval "$@"
else
return -1
fi
return $?
}
function verbose_command()
{
echo -e "\033[31mExec Command: $@ \033[0m"
eval "$@"
return $?
}
function carrot_execute()
{
case $command_decorator in
silence)
eval $1
;;
verbose)
verbose_command $1
;;
verify)
verify_command $1
;;
simulate)
echo -e "\033[32m$1\033[0m"
;;
*)
echo "invalid command decorator"
exit -1
;;
esac
if [ $? -ne 0 ]; then
echo "error in execute command $1, simulate the remaining commands"
command_decorator="simulate"
fi
}
#
# patch -b|--branch branch_name -p|--commit_point commit_point -s|--start_from_this -d|--decorate decorate_type
#
function usage_patch
{
echo "carrot patch -- apply patch to specific branch, and release a new patch version"
echo " -h|--help, print this help"
echo " -b|--branch BRANCH_NAME, the target branch. For current branch if not set"
echo " -p|--commit_point GIT_COMMIT_ID, cherry-pick this to the target"
echo " -s|--start_from_this. If set, cherry-pick from [GIT_COMMIT_ID, HEAD] to the target"
echo " -d|--decorate TYPE. [silence|verbose|verify|simulate], default is verify"
}
function make_patch
{
branch_name=""
commit_point=""
recent_commit=""
starting_flag="false"
while [[ $# > 0 ]]; do
key="$1"
case $key in
-h|--help)
usage_patch
exit 0
;;
-b|--branch)
branch_name=$2
shift
;;
-p|--commit_point)
commit_point=$2
shift;;
-s|--start_from_this)
starting_flag="true"
;;
-d|--decorate)
command_decorator=$2
shift
;;
*)
usage_patch
exit -1
;;
esac
shift
done
old_branch=`git_current_branch`
old_branch_type=`get_branch_type $old_branch`
# only in staging branch, we try to calcuate the -s flag, AND
# only in staging branch, we try to get the recent commit point in log
if [ $old_branch_type == "staging" ]; then
if [ ! -z $commit_point ]; then
if [ $starting_flag == "true" ]; then
recent_commit=`git log | sed -n "1p" | cut -d" " -f 2`
fi
else
commit_point=`git log | sed -n "1p" | cut -d" " -f 2`
fi
fi
current_branch=$old_branch
# we don't apply the patch unless we are in a release tag
if [ ! -z $branch_name ]; then
carrot_execute "git checkout $branch_name"
current_branch=$branch_name
if [ ! -z $recent_commit ]; then
carrot_execute "git cherry-pick $commit_point^..$recent_commit"
elif [ -n $commit_point ]; then
carrot_execute "git cherry-pick $commit_point"
fi
elif [ $old_branch_type == "staging" ]; then
echo "Please checkout to a release branch, or give a release branch name by -b"
exit -1
fi
new_version=`get_next_version patch`
carrot_execute "./run.sh bump_version $new_version"
carrot_execute "git commit -am \"Release $project_name $new_version\""
carrot_execute "git tag -a v$new_version -m \"Release $project_name $new_version\""
carrot_execute "git push -u origin $current_branch"
carrot_execute "git push origin v$new_version"
if [ $current_branch != $old_branch ]; then
carrot_execute "git checkout $old_branch"
fi
}
#
# minor-release -d|--decorate decorate_type
#
function usage_release_minor
{
echo "carrot minor-release"
echo " -h|--help, print this help "
echo " -d|--decorate TYPE. [silence|verbose|verify|simulate], default is verify"
}
function release_minor
{
while [[ $# > 0 ]]; do
key="$1"
case $key in
-h|--help)
usage_release_minor
exit 0
;;
-d|--decorate)
command_decorator=$2
shift
;;
esac
shift
done
this_branch=`git_current_branch`
branch_type=`get_branch_type $this_branch`
if [ $branch_type != "staging" ]; then
echo "when release minor, we need to be in staging branch, currently in a $branch_type branch $this_branch"
exit -1
fi
this_version=`get_current_version minor`
# create new branch and push
carrot_execute "git checkout -b v$this_version"
# from a.b.SNAPSHOT -> a.b.0
new_version=`get_next_version patch`
# commit the release version
carrot_execute "./run.sh bump_version $new_version"
carrot_execute "git commit -am \"Release $project_name $new_version\""
carrot_execute "git push -u origin v$this_version"
# then make tag
carrot_execute "git tag -a v$new_version -m \"Release $project_name $new_version\""
carrot_execute "git push origin v$new_version"
# update the staging branch's version
carrot_execute "git checkout $this_branch"
# from a.b.SNAPSHOT -> a.b+1.SNAPSHOT
new_version=`get_next_version minor`
carrot_execute "./run.sh bump_version $new_version"
carrot_execute "git commit -am \"Bump version to $new_version\""
carrot_execute "git push -u origin $this_branch"
}
function usage_carrot
{
echo "carrot -- Carrot is A Release veRsiOn Tool"
echo " help print the help"
echo " patch Make patch"
echo " minor-release Release a minor version"
}
pwd="$( cd "$( dirname "$0" )" && pwd )"
shell_dir="$( cd $pwd/.. && pwd )"
cd $shell_dir
action=$1
case $action in
help)
usage_carrot ;;
patch)
shift
make_patch $*
;;
minor-release)
shift
release_minor $*
;;
*)
echo "ERROR: unknown command $cmd"
echo
usage_carrot
exit -1
esac
scripts/cluster_check.in 已删除 100644 → 0
浏览文件 @ 03e02d31
cluster_info
server_info
ls -d
nodes -d
app_stat
query_backup_policy -p every_day
scripts/clusters_show.sh 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
if [ $# -lt 2 ]; then
echo "USAGE: $0 <cluster-list-file> <result-format>"
echo
echo "The result format must be 'table' or 'csv'."
echo
echo "For example:"
echo " $0 \"clusters.txt\" \"table\""
echo
exit 1
fi
PID=$$
clusters_file=$1
format=$2
if [ "$format" != "table" -a "$format" != "csv" ]; then
echo "ERROR: invalid result format, should be 'table' or 'csv'."
exit 1
fi
pwd="$( cd "$( dirname "$0" )" && pwd )"
shell_dir="$( cd $pwd/.. && pwd )"
cd $shell_dir
echo "show_time = `date`"
echo
echo "Columns:"
echo " - cluster: name of the cluster"
echo " - rs_count: current count of replica servers"
echo " - version: current version of replica servers"
echo " - lb_op_count: current count of load balance operations to make cluster balanced"
echo " - app_count: current count of tables in the cluster"
echo " - storage_gb: current total data size in GB of tables in the cluster"
echo
if [ "$format" == "table" ]; then
printf '%-30s%-12s%-12s%-12s%-12s%-12s\n' cluster rs_count version lb_op_count app_count storage_gb
elif [ "$format" == "csv" ]; then
echo "cluster,rs_count,version,lb_op_count,app_count,storage_gb"
else
echo "ERROR: invalid format: $format"
exit -1
fi
cluster_count=0
rs_count_sum=0
app_count_sum=0
data_size_sum=0
lb_op_count_sum=0
while read cluster
do
tmp_file="/tmp/$UID.$PID.pegasus.clusters_status.cluster_info"
echo "cluster_info" | ./run.sh shell -n $cluster &>$tmp_file
cluster_info_fail=`grep "\<failed\>" $tmp_file | wc -l`
if [ $cluster_info_fail -eq 1 ]; then
echo "ERROR: get cluster info failed, refer error to $tmp_file"
exit 1
fi
lb_op_count=`cat $tmp_file | grep 'balance_operation_count' | grep -o 'total=[0-9]*' | cut -d= -f2`
if [ -z $lb_op_count ]; then
lb_op_count="-"
else
lb_op_count_sum=$((lb_op_count_sum + lb_op_count))
fi
tmp_file="/tmp/$UID.$PID.pegasus.clusters_status.server_info"
echo "server_info" | ./run.sh shell -n $cluster &>$tmp_file
rs_count=`cat $tmp_file | grep 'replica-server' | wc -l`
rs_version=`cat $tmp_file | grep 'replica-server' | grep -o 'Pegasus Server [^ ]*' | head -n 1 | sed 's/SNAPSHOT/SN/' | awk '{print $3}'`
app_stat_result="/tmp/$UID.$PID.pegasus.clusters_status.app_stat_result"
tmp_file="/tmp/$UID.$PID.pegasus.clusters_status.app_stat"
echo "app_stat -o $app_stat_result" | ./run.sh shell -n $cluster &>$tmp_file
app_stat_fail=`grep "\<failed\>" $tmp_file | wc -l`
if [ $app_stat_fail -eq 1 ]; then
sleep 1
echo "app_stat -o $app_stat_result" | ./run.sh shell -n $cluster &>$tmp_file
app_stat_fail=`grep "\<failed\>" $tmp_file | wc -l`
if [ $app_stat_fail -eq 1 ]; then
echo "ERROR: app stat failed, refer error to $tmp_file"
exit 1
fi
fi
app_count=`cat $app_stat_result | wc -l`
app_count=$((app_count-2))
data_size_column=`cat $app_stat_result | awk '/file_mb/{ for(i = 1; i <= NF; i++) { if ($i == "file_mb") print i; } }'`
data_size=`cat $app_stat_result | tail -n 1 | awk '{print $'$data_size_column'}' | sed 's/\.00$//'`
data_size=$(((data_size+1023)/1024))
if [ "$format" == "table" ]; then
printf '%-30s%-12s%-12s%-12s%-12s%-12s\n' $cluster $rs_count $rs_version $lb_op_count $app_count $data_size
elif [ "$format" == "csv" ]; then
echo -e "$cluster,$rs_count,$rs_version,$lb_op_count,$app_count,$data_size"
else
echo "ERROR: invalid format: $format"
exit -1
fi
cluster_count=$((cluster_count + 1))
rs_count_sum=$((rs_count_sum + rs_count))
app_count_sum=$((app_count_sum + app_count))
data_size_sum=$((data_size_sum + data_size))
done <clusters
if [ "$format" == "table" ]; then
printf '%-30s%-12s%-12s%-12s%-12s%-12s\n' "(total:$cluster_count)" $rs_count_sum "-" $lb_op_count_sum $app_count_sum $data_size_sum
elif [ "$format" == "csv" ]; then
echo -e "(total:$cluster_count),$rs_count_sum,,$lb_op_count_sum,$app_count_sum,$data_size_sum"
else
echo "ERROR: invalid format: $format"
exit -1
fi
rm -rf /tmp/$UID.$PID.pegasus.* &>/dev/null
scripts/clusters_stat.sh 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
if [ $# -lt 3 ]; then
echo "USAGE: $0 <cluster-list-file> <month-list> <result-format>"
echo
echo "The result format must be 'table' or 'csv'."
echo
echo "For example:"
echo " $0 \"clusters.txt\" \"2019-01\" \"table\""
echo " $0 \"clusters.txt\" \"2019-01 2019-02\" \"csv\""
echo
exit 1
fi
clusters_file=$1
months=$2
format=$3
if [ "$format" != "table" -a "$format" != "csv" ]; then
echo "ERROR: invalid result format, should be 'table' or 'csv'."
exit 1
fi
pwd="$( cd "$( dirname "$0" )" && pwd )"
shell_dir="$( cd $pwd/.. && pwd )"
cd $shell_dir
all_result="/tmp/pegasus.stat_available.all_result"
rm $all_result &>/dev/null
echo "stat_time = `date`"
echo "month_list = $months"
echo
echo "Stat method:"
echo " - for each cluster, there is a collector which sends get/set requests to detect table every 3 seconds."
echo " - every minute, the collector will write a record of Send and Succeed count into detect table."
echo " - to stat cluster availability, we scan all the records for the months from detect table, calculate the"
echo " total Send count and total Succeed count, and calculate the availability by:"
echo " Available = TotalSucceedCount / TotalSendCount"
echo
echo "Columns:"
echo " - cluster: name of the cluster"
echo " - rs_count: current count of replica servers"
echo " - version: current version of replica servers"
echo " - minutes: record count in detect table for the months"
echo " - available: cluster availability"
echo " - app_count: current count of tables in the cluster"
echo " - storage_gb: current total data size in GB of tables in the cluster"
echo
if [ "$format" == "table" ]; then
printf '%-30s%-12s%-12s%-12s%-12s%-12s%-12s\n' cluster rs_count version minutes available app_count storage_gb
elif [ "$format" == "csv" ]; then
echo "cluster,rs_count,version,minutes,available,table_count,storage_gb"
else
echo "ERROR: invalid format: $format"
exit 1
fi
cluster_count=0
rs_count_sum=0
app_count_sum=0
data_size_sum=0
while read cluster
do
rs_count=`echo server_info | ./run.sh shell -n $cluster 2>&1 | grep 'replica-server' | wc -l`
rs_version=`echo server_info | ./run.sh shell -n $cluster 2>&1 | grep 'replica-server' | \
grep -o 'Pegasus Server [^ ]*' | head -n 1 | sed 's/SNAPSHOT/SN/' | awk '{print $3}'`
result=`./scripts/pegasus_stat_available.sh $cluster $months`
if echo $result | grep '^ERROR'; then
echo "ERROR: process cluster $cluster failed"
continue
fi
minutes=`echo $result | awk '{print $2}'`
available=`echo $result | awk '{print $3}' | sed 's/data/-/'`
app_count=`echo $result | awk '{print $4}'`
data_size=`echo $result | awk '{print $5}'`
if [ "$available" == "1.000000" ]; then
available_str="99.9999%"
elif [ "$available" == "0" ]; then
available_str="00.0000%"
else
available_str="${available:2:2}.${available:4:4}%"
fi
if [ "$format" == "table" ]; then
printf '%-30s%-12s%-12s%-12s%-12s%-12s%-12s\n' $cluster $rs_count $rs_version $minutes $available $app_count $data_size
elif [ "$format" == "csv" ]; then
echo -e "$cluster,$rs_count,$rs_version,$minutes,=\"$available_str\",$app_count,$data_size"
else
echo "ERROR: invalid format: $format"
exit 1
fi
cluster_count=$((cluster_count + 1))
rs_count_sum=$((rs_count_sum + rs_count))
app_count_sum=$((app_count_sum + app_count))
data_size_sum=$((data_size_sum + data_size))
done <$clusters_file
minutes=`cat $all_result | wc -l`
if [ $minutes -eq 0 ]; then
available="0.000000"
else
available=`cat $all_result | grep -o '[0-9]*,[0-9]*,[0-9]*' | awk -F, '{a+=$1;b+=$2}END{printf("%f\n",(double)b/a);}'`
fi
if [ "$available" == "1.000000" ]; then
available_str="99.9999%"
elif [ "$available" == "0" ]; then
available_str="00.0000%"
else
available_str="${available:2:2}.${available:4:4}%"
fi
if [ "$format" == "table" ]; then
printf '%-30s%-12s%-12s%-12s%-12s%-12s%-12s\n' "(total:$cluster_count)" $rs_count_sum "-" $minutes $available $app_count_sum $data_size_sum
echo
elif [ "$format" == "csv" ]; then
echo -e "(total:$cluster_count),$rs_count_sum,,$minutes,=\"$available_str\",$app_count_sum,$data_size_sum"
else
echo "ERROR: invalid format: $format"
exit 1
fi
rm $all_result &>/dev/null
scripts/create_table.py 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
HOWTO
=====
./scripts/create_table.py --table ai_user_info \
--depart 云平台部-存储平台-KV系统组 \
--user wutao1&qinzuoyan \
--cluster bj1-ai \
--write_throttling "2000*delay*100" \
--partition_count 16
OR
./scripts/create_table.py -t ai_user_info \
-d 云平台部-存储平台-KV系统组 \
-u wutao1&qinzuoyan \
-c bj1-ai \
-w "2000*delay*100" \
-p 16
DEVLOPER GUIDE
==============
The source code is formatted using autopep8.
Ensure you have run formatter before committing changes.
```
autopep8 -i --aggressive --aggressive scripts/create_table.py
```
TODO(wutao1): automatically set write throttling according to the given
estimated QPS on the table.
"""
import os
import click
import py_utils
import re
import json
import math
def validate_param_table(ctx, param, value):
# TODO(wutao1): check illegal characters
return value.encode('utf-8')
def validate_param_depart(ctx, param, value):
return value.encode('utf-8')
def validate_param_user(ctx, param, value):
return value.encode('utf-8')
def validate_param_cluster(ctx, param, value):
return value.encode('utf-8')
def validate_param_partition_count(ctx, param, value):
if value == 0:
raise click.BadParameter("Cannot create table with 0 partition")
if math.log(value, 2) != math.floor(math.log(value, 2)):
raise click.BadParameter(
"Partition count {} should be a power of 2".format(value))
return value
def validate_param_write_throttling(ctx, param, value):
if value == '':
return None
pattern = re.compile(r'^\d+\*delay\*\d+(,\d+\*reject\*\d+)?$')
match = pattern.match(value)
if match is not None:
return value.encode('utf-8')
else:
raise click.BadParameter(
'invalid value of throttle \'%s\'' % value)
def create_table_if_needed(cluster, table, partition_count):
if not cluster.has_table(table):
try:
# TODO(wutao1): Outputs progress while polling.
py_utils.echo("Creating table {}...".format(table))
cluster.create_table(table, partition_count)
except Exception as err:
py_utils.echo(err, "red")
exit(1)
else:
py_utils.echo("Success: table \"{}\" exists".format(table))
def set_business_info_if_needed(cluster, table, depart, user):
new_business_info = "depart={},user={}".format(depart, user)
set_app_envs_if_needed(cluster, table, 'business.info', new_business_info)
def set_write_throttling_if_needed(cluster, table, new_throttle):
if new_throttle is None:
return
set_app_envs_if_needed(
cluster, table, 'replica.write_throttling', new_throttle)
def set_app_envs_if_needed(cluster, table, env_name, new_env_value):
py_utils.echo("New value of {}={}".format(env_name, new_env_value))
envs = cluster.get_app_envs(table)
if envs is not None and envs.get(env_name) is not None:
old_env_value = envs.get(env_name).encode('utf-8')
if old_env_value is not None:
py_utils.echo("Old value of {}={}".format(env_name, old_env_value))
if old_env_value == new_env_value:
py_utils.echo("Success: {} keeps unchanged".format(env_name))
return
cluster.set_app_envs(table, env_name,
new_env_value)
def all_arguments_to_string(
table,
depart,
user,
cluster,
partition_count,
write_throttling):
return json.dumps({
'table': table,
'depart': depart,
'user': user,
'cluster': cluster,
'partition_count': partition_count,
'write_throttling': write_throttling,
}, sort_keys=True, indent=4, ensure_ascii=False, encoding='utf-8')
@click.command()
@click.option("--table", "-t",
required=True,
callback=validate_param_table,
help="Name of the table you want to create.")
@click.option(
"--depart", "-d",
required=True,
callback=validate_param_depart,
help="Department of the table owner. If there are more than one levels of department, use '-' to concatenate them.")
@click.option(
"--user", "-u",
required=True,
callback=validate_param_user,
help="The table owner. If there are more than one owners, use '&' to concatenate them.")
@click.option("--cluster", "-c",
required=True,
callback=validate_param_cluster,
help="The cluster name. Where you want to place the table.")
@click.option("--partition_count", "-p",
callback=validate_param_partition_count,
help="The partition count of the table. Empty means no create.",
type=int)
@click.option(
"--write_throttling", "-w",
default="",
callback=validate_param_write_throttling,
help="{delay_qps_threshold}*delay*{delay_ms},{reject_qps_threshold}*reject*{delay_ms_before_reject}")
def main(table, depart, user, cluster, partition_count, write_throttling):
if not click.confirm(
"Confirm to create table:\n{}\n".format(
all_arguments_to_string(
table,
depart,
user,
cluster,
partition_count,
write_throttling))):
return
c = py_utils.PegasusCluster(cluster_name=cluster)
create_table_if_needed(c, table, partition_count)
set_business_info_if_needed(c, table, depart, user)
set_write_throttling_if_needed(c, table, write_throttling)
if __name__ == "__main__":
main()
scripts/falcon_screen.json 已删除 100644 → 0
浏览文件 @ 03e02d31
此差异已折叠。
scripts/falcon_screen.py 已删除 100755 → 0
浏览文件 @ 03e02d31
此差异已折叠。
scripts/pegasus_check_clusters.py 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
Basic usage:
> vim ~/.bashrc
export PYTHONPATH=$PYTHONPATH:$HOME/.local/lib/python2.7/site-packages/
export PEGASUS_CONFIG_PATH=$HOME/work/conf_pegasus
export PEGASUS_SHELL_PATH=$HOME/work/pegasus
> pip install --user click
> ./pegasus_check_clusters.py --env c3srv
"""
import os
import click
from py_utils import *
@click.command()
@click.option(
"--env", default="", help="Env of pegasus cluster, eg. c3srv or c4tst")
@click.option('-v', '--verbose', count=True)
def main(env, verbose):
pegasus_config_path = os.getenv("PEGASUS_CONFIG_PATH")
if pegasus_config_path is None:
echo(
"Please configure environment variable PEGASUS_CONFIG_PATH in your bashrc or zshrc",
"red")
exit(1)
if env != "":
echo("env = " + env)
set_global_verbose(verbose)
clusters = list_pegasus_clusters(pegasus_config_path, env)
for cluster in clusters:
echo("=== " + cluster.name())
try:
cluster.print_imbalance_nodes()
cluster.print_unhealthy_partitions()
except RuntimeError as e:
echo(str(e), "red")
return
echo("===")
if __name__ == "__main__":
main()
scripts/pegasus_check_ports.py 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
Basic usage:
> vim ~/.bashrc
export PYTHONPATH=$PYTHONPATH:$HOME/.local/lib/python2.7/site-packages/
export PEGASUS_CONFIG_PATH=$HOME/work/conf_pegasus
export PEGASUS_SHELL_PATH=$HOME/work/pegasus
> pip install --user click
> ./pegasus_check_posts.py --env c3srv
"""
import os
import click
from py_utils import *
@click.command()
@click.option("--env", help="Env of pegasus cluster, eg. c3srv or c4tst")
def main(env):
pegasus_config_path = os.getenv("PEGASUS_CONFIG_PATH")
if pegasus_config_path is None:
echo(
"Please configure environment variable PEGASUS_CONFIG_PATH in your bashrc or zshrc",
"red")
exit(1)
clusters = list_pegasus_clusters(pegasus_config_path, env)
host_to_ports = {}
for cluster in clusters:
try:
p = cluster.get_meta_port()
h = cluster.get_meta_host()
if not h in host_to_ports:
host_to_ports[h] = set()
if p in host_to_ports[h]:
echo(
"port number conflicted: {0} {1} [{2}]".format(
p, cluster.name(), h), "red")
continue
host_to_ports[h].add(p)
echo("cluster {0}: {1} [{2}]".format(cluster.name(), p, h))
except RuntimeError as e:
echo(str(e), "red")
return
echo("")
for h in host_to_ports:
echo("recommended port number for [{0}] is: {1}".format(
h, str(max(host_to_ports[h]) + 1000)))
echo("host [{0}] has in total {1} clusters on it".format(
h, len(host_to_ports[h])))
echo("")
if __name__ == "__main__":
main()
scripts/pegasus_falcon_screen.sh 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
PID=$$
if [ $# -ne 2 ]
then
echo "This tool is for create or update falcon screen for specified cluster."
echo "USAGE: $0 <create|update> <cluster-name>"
exit 1
fi
pwd="$( cd "$( dirname "$0" )" && pwd )"
shell_dir="$( cd $pwd/.. && pwd )"
cd $shell_dir
operate=$1
cluster=$2
if [ "$operate" != "create" -a "$operate" != "update" ]; then
echo "ERROR: invalid operation type: $operate"
exit 1
fi
echo "UID: $UID"
echo "PID: $PID"
echo "cluster: $cluster"
echo "operate: $operate"
echo "Start time: `date`"
all_start_time=$((`date +%s`))
echo
cd $shell_dir
echo ls | ./run.sh shell -n $cluster &>/tmp/$UID.$PID.pegasus.ls
grep AVAILABLE /tmp/$UID.$PID.pegasus.ls | awk '{print $3}' >/tmp/$UID.$PID.pegasus.table.list
table_count=`cat /tmp/$UID.$PID.pegasus.table.list | wc -l`
if [ $table_count -eq 0 ]; then
echo "ERROR: table list is empty, please check the cluster $cluster"
exit 1
fi
cd $pwd
python falcon_screen.py $cluster falcon_screen.json /tmp/$UID.$PID.pegasus.table.list $operate
if [ $? -ne 0 ]; then
echo "ERROR: falcon screen $operate failed"
exit 1
fi
echo
echo "Finish time: `date`"
all_finish_time=$((`date +%s`))
echo "Falcon screen $operate done, elasped time is $((all_finish_time - all_start_time)) seconds."
rm -f /tmp/$UID.$PID.pegasus.* &>/dev/null
scripts/py_utils/__init__.py 已删除 100644 → 0
浏览文件 @ 03e02d31
#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .lib import set_global_verbose, echo, list_pegasus_clusters, PegasusCluster
__all__ = [
'set_global_verbose', 'echo', 'list_pegasus_clusters', 'PegasusCluster'
]
scripts/py_utils/lib.py 已删除 100644 → 0
浏览文件 @ 03e02d31
#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import click
import commands
import os
import json
_global_verbose = False
def set_global_verbose(val):
_global_verbose = val
def echo(message, color=None):
click.echo(click.style(message, fg=color))
class PegasusCluster(object):
def __init__(self, cfg_file_name=None, cluster_name=None):
if cluster_name is None:
self._cluster_name = os.path.basename(cfg_file_name).replace(
"pegasus-", "").replace(".cfg", "")
else:
self._cluster_name = cluster_name
self._shell_path = os.getenv("PEGASUS_SHELL_PATH")
self._cfg_file_name = cfg_file_name
if self._shell_path is None:
echo(
"Please configure environment variable PEGASUS_SHELL_PATH in your bashrc or zshrc",
"red")
exit(1)
def print_unhealthy_partitions(self):
list_detail = self._run_shell("ls -d -j").strip()
list_detail_json = json.loads(list_detail)
read_unhealthy_app_count = int(
list_detail_json["summary"]["read_unhealthy_app_count"])
write_unhealthy_app_count = int(
list_detail_json["summary"]["write_unhealthy_app_count"])
if write_unhealthy_app_count > 0:
echo("cluster is write unhealthy, write_unhealthy_app_count = " +
str(write_unhealthy_app_count))
return
if read_unhealthy_app_count > 0:
echo("cluster is read unhealthy, read_unhealthy_app_count = " +
str(read_unhealthy_app_count))
return
def print_imbalance_nodes(self):
nodes_detail = self._run_shell("nodes -d -j").strip()
primaries_per_node = {}
min_ = 0
max_ = 0
for ip_port, node_info in json.loads(nodes_detail)["details"].items():
primary_count = int(node_info["primary_count"])
min_ = min(min_, primary_count)
max_ = max(max_, primary_count)
primaries_per_node[ip_port] = primary_count
if float(min_) / float(max_) < 0.8:
print json.dumps(primaries_per_node, indent=4)
def get_meta_port(self):
with open(self._cfg_file_name) as cfg:
for line in cfg.readlines():
if line.strip().startswith("base_port"):
return int(line.split("=")[1])
def get_meta_host(self):
with open(self._cfg_file_name) as cfg:
for line in cfg.readlines():
if line.strip().startswith("host.0"):
return line.split("=")[1].strip()
def create_table(self, table, parts):
create_result = self._run_shell(
"create {} -p {}".format(table, parts)).strip()
if "ERR_INVALID_PARAMETERS" in create_result:
raise ValueError("failed to create table \"{}\"".format(table))
def get_app_envs(self, table):
envs_result = self._run_shell(
"use {} \n get_app_envs".format(table)).strip()[len("OK\n"):]
if "ERR_OBJECT_NOT_FOUND" in envs_result:
raise ValueError("table {} does not exist".format(table))
if envs_result == "":
return None
envs_result = self._run_shell(
"use {} \n get_app_envs -j".format(table)).strip()[len("OK\n"):]
return json.loads(envs_result)['app_envs']
def set_app_envs(self, table, env_name, env_value):
envs_result = self._run_shell(
"use {} \n set_app_envs {} {}".format(
table, env_name, env_value)).strip()[
len("OK\n"):]
if "ERR_OBJECT_NOT_FOUND" in envs_result:
raise ValueError("table {} does not exist".format(table))
def has_table(self, table):
app_result = self._run_shell("app {} ".format(table)).strip()
return "ERR_OBJECT_NOT_FOUND" not in app_result
def _run_shell(self, args):
"""
:param args: arguments passed to ./run.sh shell (type `string`)
:return: shell output
"""
global _global_verbose
cmd = "cd {1}; echo -e \"{0}\" | ./run.sh shell -n {2}".format(
args, self._shell_path, self._cluster_name)
if _global_verbose:
echo("executing command: \"{0}\"".format(cmd))
status, output = commands.getstatusoutput(cmd)
if status != 0:
raise RuntimeError("failed to execute \"{0}\": {1}".format(
cmd, output))
result = ""
result_begin = False
for line in output.splitlines():
if line.startswith("The cluster meta list is:"):
result_begin = True
continue
if line.startswith("dsn exit with code"):
break
if result_begin:
result += line + "\n"
return result
def name(self):
return self._cluster_name
def list_pegasus_clusters(config_path, env):
clusters = []
for fname in os.listdir(config_path):
if not os.path.isfile(config_path + "/" + fname):
continue
if not fname.startswith("pegasus-" + env):
continue
if not fname.endswith(".cfg"):
continue
if fname.endswith("proxy.cfg"):
continue
clusters.append(PegasusCluster(config_path + "/" + fname))
return clusters
scripts/scp-no-interactive 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/usr/bin/expect
# USAGE: scp-no-interactive <host> <username> <password> <src_file> <dest_file>
set timeout 10
set host [lindex $argv 0]
set username [lindex $argv 1]
set password [lindex $argv 2]
set src_file [lindex $argv 3]
set dest_file [lindex $argv 4]
spawn scp $src_file $username@$host:$dest_file
expect {
"(yes/no)?"
{
send "yes\n"
expect "*assword:" { send "$password\n"}
}
"*assword:"
{
send "$password\n"
}
}
expect "100%"
expect eof
scripts/ssh-no-interactive 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/usr/bin/expect
# USAGE: ssh-no-interactive <host> <username> <password> <command>
set timeout 10
set host [lindex $argv 0]
set username [lindex $argv 1]
set password [lindex $argv 2]
set command [lindex $argv 3]
spawn ssh $username@$host "$command"
expect {
"(yes/no)?"
{
send "yes\n"
expect "*assword:" { send "$password\n"}
}
"*assword:"
{
send "$password\n"
}
}
expect eof
scripts/update_qt_config.sh 已删除 100755 → 0
浏览文件 @ 03e02d31
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# This is used for updating the meta-data of Qt Creator IDE.
PREFIX=pegasus
if [ $# -eq 1 ]
then
PREFIX=$1
fi
pwd="$( cd "$( dirname "$0" )" && pwd )"
shell_dir="$( cd $pwd/.. && pwd )"
cd $shell_dir
# config
CONFIG_OUT="${PREFIX}.config"
echo "Generate $CONFIG_OUT"
rm $CONFIG_OUT &>/dev/null
echo "#define __cplusplus 201103L" >>$CONFIG_OUT
echo "#define _DEBUG" >>$CONFIG_OUT
echo "#define DSN_USE_THRIFT_SERIALIZATION" >>$CONFIG_OUT
echo "#define DSN_ENABLE_THRIFT_RPC" >>$CONFIG_OUT
echo "#define DSN_BUILD_TYPE" >>$CONFIG_OUT
echo "#define DSN_BUILD_HOSTNAME" >>$CONFIG_OUT
echo "#define ROCKSDB_PLATFORM_POSIX" >>$CONFIG_OUT
echo "#define OS_LINUX" >>$CONFIG_OUT
echo "#define ROCKSDB_FALLOCATE_PRESENT" >>$CONFIG_OUT
echo "#define GFLAGS google" >>$CONFIG_OUT
echo "#define ZLIB" >>$CONFIG_OUT
echo "#define BZIP2" >>$CONFIG_OUT
echo "#define ROCKSDB_MALLOC_USABLE_SIZE" >>$CONFIG_OUT
#echo "#define __FreeBSD__" >>$CONFIG_OUT
#echo "#define _WIN32" >>$CONFIG_OUT
# includes
INCLUDES_OUT="${PREFIX}.includes"
echo "Generate $INCLUDES_OUT"
rm $INCLUDES_OUT &>/dev/null
echo "/usr/include" >>$INCLUDES_OUT
echo "/usr/include/c++/4.8" >>$INCLUDES_OUT
echo "/usr/include/x86_64-linux-gnu" >>$INCLUDES_OUT
echo "/usr/include/x86_64-linux-gnu/c++/4.8" >>$INCLUDES_OUT
echo "rdsn/include" >>$INCLUDES_OUT
echo "rdsn/thirdparty/output/include" >>$INCLUDES_OUT
echo "rdsn/include/dsn/dist/failure_detector" >>$INCLUDES_OUT
echo "rdsn/src/dist/replication/client_lib" >>$INCLUDES_OUT
echo "rdsn/src/dist/replication/lib" >>$INCLUDES_OUT
echo "rdsn/src/dist/replication/meta_server" >>$INCLUDES_OUT
echo "rdsn/src/dist/replication/zookeeper" >>$INCLUDES_OUT
echo "rdsn/thirdparty/output/include" >>$INCLUDES_OUT
echo "rdsn/src/dist/block_service/fds" >>$INCLUDES_OUT
echo "rdsn/src/dist/block_service/local" >>$INCLUDES_OUT
echo "rdsn/src" >> $INCLUDES_OUT
echo "rocksdb" >>$INCLUDES_OUT
echo "rocksdb/include" >>$INCLUDES_OUT
echo "src" >>$INCLUDES_OUT
echo "src/include" >>$INCLUDES_OUT
echo "src/redis_protocol/proxy_lib" >>$INCLUDES_OUT
# files
FILES_OUT="${PREFIX}.files"
echo "Generate $FILES_OUT"
rm $FILES_OUT >&/dev/null
echo "build.sh" >>$FILES_OUT
echo "rdsn/CMakeLists.txt" >>$FILES_OUT
echo "rdsn/bin/dsn.cmake" >>$FILES_OUT
FILES_DIR="
src rocksdb rdsn scripts
"
for i in $FILES_DIR
do
find $i -name '*.h' -o -name '*.cpp' -o -name '*.c' -o -name '*.cc' \
-o -name '*.thrift' -o -name '*.ini' -o -name '*.act' \
-o -name 'CMakeLists.txt' -o -name '*.sh' \
| grep -v '\<builder\>\|rdsn\/thirdparty\|\.zk_install' >>$FILES_OUT
done
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或