diff --git a/scripts/carrot b/scripts/carrot deleted file mode 100755 index 39fc82f270befcee00d8e006630c05dd57ce05b9..0000000000000000000000000000000000000000 --- a/scripts/carrot +++ /dev/null @@ -1,291 +0,0 @@ -#!/bin/bash - -staging_branch="master" -project_name="pegasus" -command_decorator="verify" - -function git_current_branch() -{ - echo `git branch | fgrep "*" | cut -d " " -f 2` -} - -function get_next_version() -{ - versions=(`grep "PEGASUS_VERSION" src/include/pegasus/version.h | cut -d"\"" -f 2 | sed 's/\./ /g'`) - case $1 in - major) - versions[0]=$[ ${versions[0]} + 1 ] - ;; - minor) - versions[1]=$[ ${versions[1]} + 1 ] - ;; - patch) - if [ ${versions[2]} == "SNAPSHOT" ]; then - versions[2]="0" - else - versions[2]=$[ ${versions[2]} + 1 ] - fi - ;; - *) - echo "Invalid next version type" - exit -1 - ;; - esac - - echo ${versions[*]} | sed 's/ /\./g' -} - -function get_current_version() -{ - versions=(`grep "PEGASUS_VERSION" src/include/pegasus/version.h | cut -d"\"" -f 2 | sed 's/\./ /g'`) - case $1 in - major) - echo ${versions[0]} - ;; - minor) - echo ${versions[0]}.${versions[1]} - ;; - patch) - echo ${versions[*]} | sed 's/ /\./g' - ;; - *) - echo "Invalid current version type" - exit -1 - ;; - esac -} - -function get_branch_type() -{ - if [ $1 = $staging_branch ]; then - echo "staging" - else - echo "release" - fi -} - -function verify_command() -{ - answer="" - echo -n -e "\033[31mExecuting command: $@, y/N?\033[0m" - read answer - if [ -z $answer ] || [ $answer = "y" ]; then - eval "$@" - else - return -1 - fi - return $? -} - -function verbose_command() -{ - echo -e "\033[31mExec Command: $@ \033[0m" - eval "$@" - return $? -} - -function carrot_execute() -{ - case $command_decorator in - silence) - eval $1 - ;; - verbose) - verbose_command $1 - ;; - verify) - verify_command $1 - ;; - simulate) - echo -e "\033[32m$1\033[0m" - ;; - *) - echo "invalid command decorator" - exit -1 - ;; - esac - if [ $? -ne 0 ]; then - echo "error in execute command $1, simulate the remaining commands" - command_decorator="simulate" - fi -} - -# -# patch -b|--branch branch_name -p|--commit_point commit_point -s|--start_from_this -d|--decorate decorate_type -# -function usage_patch -{ - echo "carrot patch -- apply patch to specific branch, and release a new patch version" - echo " -h|--help, print this help" - echo " -b|--branch BRANCH_NAME, the target branch. For current branch if not set" - echo " -p|--commit_point GIT_COMMIT_ID, cherry-pick this to the target" - echo " -s|--start_from_this. If set, cherry-pick from [GIT_COMMIT_ID, HEAD] to the target" - echo " -d|--decorate TYPE. [silence|verbose|verify|simulate], default is verify" -} - -function make_patch -{ - branch_name="" - commit_point="" - recent_commit="" - starting_flag="false" - - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_patch - exit 0 - ;; - -b|--branch) - branch_name=$2 - shift - ;; - -p|--commit_point) - commit_point=$2 - shift;; - -s|--start_from_this) - starting_flag="true" - ;; - -d|--decorate) - command_decorator=$2 - shift - ;; - *) - usage_patch - exit -1 - ;; - esac - shift - done - - old_branch=`git_current_branch` - old_branch_type=`get_branch_type $old_branch` - - # only in staging branch, we try to calcuate the -s flag, AND - # only in staging branch, we try to get the recent commit point in log - if [ $old_branch_type == "staging" ]; then - if [ ! -z $commit_point ]; then - if [ $starting_flag == "true" ]; then - recent_commit=`git log | sed -n "1p" | cut -d" " -f 2` - fi - else - commit_point=`git log | sed -n "1p" | cut -d" " -f 2` - fi - fi - - current_branch=$old_branch - # we don't apply the patch unless we are in a release tag - if [ ! -z $branch_name ]; then - carrot_execute "git checkout $branch_name" - current_branch=$branch_name - if [ ! -z $recent_commit ]; then - carrot_execute "git cherry-pick $commit_point^..$recent_commit" - elif [ -n $commit_point ]; then - carrot_execute "git cherry-pick $commit_point" - fi - elif [ $old_branch_type == "staging" ]; then - echo "Please checkout to a release branch, or give a release branch name by -b" - exit -1 - fi - - new_version=`get_next_version patch` - carrot_execute "./run.sh bump_version $new_version" - carrot_execute "git commit -am \"Release $project_name $new_version\"" - carrot_execute "git tag -a v$new_version -m \"Release $project_name $new_version\"" - carrot_execute "git push -u origin $current_branch" - carrot_execute "git push origin v$new_version" - - if [ $current_branch != $old_branch ]; then - carrot_execute "git checkout $old_branch" - fi -} - -# -# minor-release -d|--decorate decorate_type -# -function usage_release_minor -{ - echo "carrot minor-release" - echo " -h|--help, print this help " - echo " -d|--decorate TYPE. [silence|verbose|verify|simulate], default is verify" -} - -function release_minor -{ - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_release_minor - exit 0 - ;; - -d|--decorate) - command_decorator=$2 - shift - ;; - esac - shift - done - - this_branch=`git_current_branch` - branch_type=`get_branch_type $this_branch` - - if [ $branch_type != "staging" ]; then - echo "when release minor, we need to be in staging branch, currently in a $branch_type branch $this_branch" - exit -1 - fi - - this_version=`get_current_version minor` - - # create new branch and push - carrot_execute "git checkout -b v$this_version" - # from a.b.SNAPSHOT -> a.b.0 - new_version=`get_next_version patch` - # commit the release version - carrot_execute "./run.sh bump_version $new_version" - carrot_execute "git commit -am \"Release $project_name $new_version\"" - carrot_execute "git push -u origin v$this_version" - # then make tag - carrot_execute "git tag -a v$new_version -m \"Release $project_name $new_version\"" - carrot_execute "git push origin v$new_version" - - # update the staging branch's version - carrot_execute "git checkout $this_branch" - # from a.b.SNAPSHOT -> a.b+1.SNAPSHOT - new_version=`get_next_version minor` - carrot_execute "./run.sh bump_version $new_version" - carrot_execute "git commit -am \"Bump version to $new_version\"" - carrot_execute "git push -u origin $this_branch" -} - -function usage_carrot -{ - echo "carrot -- Carrot is A Release veRsiOn Tool" - echo " help print the help" - echo " patch Make patch" - echo " minor-release Release a minor version" -} - -pwd="$( cd "$( dirname "$0" )" && pwd )" -shell_dir="$( cd $pwd/.. && pwd )" -cd $shell_dir - -action=$1 -case $action in - help) - usage_carrot ;; - patch) - shift - make_patch $* - ;; - minor-release) - shift - release_minor $* - ;; - *) - echo "ERROR: unknown command $cmd" - echo - usage_carrot - exit -1 -esac diff --git a/scripts/cluster_check.in b/scripts/cluster_check.in deleted file mode 100644 index e8bb69f8e8fe875914724b9be315cd5e2d75c224..0000000000000000000000000000000000000000 --- a/scripts/cluster_check.in +++ /dev/null @@ -1,7 +0,0 @@ -cluster_info -server_info -ls -d -nodes -d -app_stat -query_backup_policy -p every_day - diff --git a/scripts/clusters_show.sh b/scripts/clusters_show.sh deleted file mode 100755 index b5661f561112721926b3acea8d2f22b1633bfdd6..0000000000000000000000000000000000000000 --- a/scripts/clusters_show.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 2 ]; then - echo "USAGE: $0 " - echo - echo "The result format must be 'table' or 'csv'." - echo - echo "For example:" - echo " $0 \"clusters.txt\" \"table\"" - echo - exit 1 -fi - -PID=$$ -clusters_file=$1 -format=$2 -if [ "$format" != "table" -a "$format" != "csv" ]; then - echo "ERROR: invalid result format, should be 'table' or 'csv'." - exit 1 -fi - -pwd="$( cd "$( dirname "$0" )" && pwd )" -shell_dir="$( cd $pwd/.. && pwd )" -cd $shell_dir - -echo "show_time = `date`" -echo -echo "Columns:" -echo " - cluster: name of the cluster" -echo " - rs_count: current count of replica servers" -echo " - version: current version of replica servers" -echo " - lb_op_count: current count of load balance operations to make cluster balanced" -echo " - app_count: current count of tables in the cluster" -echo " - storage_gb: current total data size in GB of tables in the cluster" -echo -if [ "$format" == "table" ]; then - printf '%-30s%-12s%-12s%-12s%-12s%-12s\n' cluster rs_count version lb_op_count app_count storage_gb -elif [ "$format" == "csv" ]; then - echo "cluster,rs_count,version,lb_op_count,app_count,storage_gb" -else - echo "ERROR: invalid format: $format" - exit -1 -fi -cluster_count=0 -rs_count_sum=0 -app_count_sum=0 -data_size_sum=0 -lb_op_count_sum=0 -while read cluster -do - tmp_file="/tmp/$UID.$PID.pegasus.clusters_status.cluster_info" - echo "cluster_info" | ./run.sh shell -n $cluster &>$tmp_file - cluster_info_fail=`grep "\" $tmp_file | wc -l` - if [ $cluster_info_fail -eq 1 ]; then - echo "ERROR: get cluster info failed, refer error to $tmp_file" - exit 1 - fi - lb_op_count=`cat $tmp_file | grep 'balance_operation_count' | grep -o 'total=[0-9]*' | cut -d= -f2` - if [ -z $lb_op_count ]; then - lb_op_count="-" - else - lb_op_count_sum=$((lb_op_count_sum + lb_op_count)) - fi - - tmp_file="/tmp/$UID.$PID.pegasus.clusters_status.server_info" - echo "server_info" | ./run.sh shell -n $cluster &>$tmp_file - rs_count=`cat $tmp_file | grep 'replica-server' | wc -l` - rs_version=`cat $tmp_file | grep 'replica-server' | grep -o 'Pegasus Server [^ ]*' | head -n 1 | sed 's/SNAPSHOT/SN/' | awk '{print $3}'` - - app_stat_result="/tmp/$UID.$PID.pegasus.clusters_status.app_stat_result" - tmp_file="/tmp/$UID.$PID.pegasus.clusters_status.app_stat" - echo "app_stat -o $app_stat_result" | ./run.sh shell -n $cluster &>$tmp_file - app_stat_fail=`grep "\" $tmp_file | wc -l` - if [ $app_stat_fail -eq 1 ]; then - sleep 1 - echo "app_stat -o $app_stat_result" | ./run.sh shell -n $cluster &>$tmp_file - app_stat_fail=`grep "\" $tmp_file | wc -l` - if [ $app_stat_fail -eq 1 ]; then - echo "ERROR: app stat failed, refer error to $tmp_file" - exit 1 - fi - fi - app_count=`cat $app_stat_result | wc -l` - app_count=$((app_count-2)) - data_size_column=`cat $app_stat_result | awk '/file_mb/{ for(i = 1; i <= NF; i++) { if ($i == "file_mb") print i; } }'` - data_size=`cat $app_stat_result | tail -n 1 | awk '{print $'$data_size_column'}' | sed 's/\.00$//'` - data_size=$(((data_size+1023)/1024)) - - if [ "$format" == "table" ]; then - printf '%-30s%-12s%-12s%-12s%-12s%-12s\n' $cluster $rs_count $rs_version $lb_op_count $app_count $data_size - elif [ "$format" == "csv" ]; then - echo -e "$cluster,$rs_count,$rs_version,$lb_op_count,$app_count,$data_size" - else - echo "ERROR: invalid format: $format" - exit -1 - fi - - cluster_count=$((cluster_count + 1)) - rs_count_sum=$((rs_count_sum + rs_count)) - app_count_sum=$((app_count_sum + app_count)) - data_size_sum=$((data_size_sum + data_size)) -done /dev/null - diff --git a/scripts/clusters_stat.sh b/scripts/clusters_stat.sh deleted file mode 100755 index ef030e6d9a9e570cb5b75555cb3d62a2f53134c5..0000000000000000000000000000000000000000 --- a/scripts/clusters_stat.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 3 ]; then - echo "USAGE: $0 " - echo - echo "The result format must be 'table' or 'csv'." - echo - echo "For example:" - echo " $0 \"clusters.txt\" \"2019-01\" \"table\"" - echo " $0 \"clusters.txt\" \"2019-01 2019-02\" \"csv\"" - echo - exit 1 -fi - -clusters_file=$1 -months=$2 -format=$3 -if [ "$format" != "table" -a "$format" != "csv" ]; then - echo "ERROR: invalid result format, should be 'table' or 'csv'." - exit 1 -fi - -pwd="$( cd "$( dirname "$0" )" && pwd )" -shell_dir="$( cd $pwd/.. && pwd )" -cd $shell_dir - -all_result="/tmp/pegasus.stat_available.all_result" -rm $all_result &>/dev/null -echo "stat_time = `date`" -echo "month_list = $months" -echo -echo "Stat method:" -echo " - for each cluster, there is a collector which sends get/set requests to detect table every 3 seconds." -echo " - every minute, the collector will write a record of Send and Succeed count into detect table." -echo " - to stat cluster availability, we scan all the records for the months from detect table, calculate the" -echo " total Send count and total Succeed count, and calculate the availability by:" -echo " Available = TotalSucceedCount / TotalSendCount" -echo -echo "Columns:" -echo " - cluster: name of the cluster" -echo " - rs_count: current count of replica servers" -echo " - version: current version of replica servers" -echo " - minutes: record count in detect table for the months" -echo " - available: cluster availability" -echo " - app_count: current count of tables in the cluster" -echo " - storage_gb: current total data size in GB of tables in the cluster" -echo -if [ "$format" == "table" ]; then - printf '%-30s%-12s%-12s%-12s%-12s%-12s%-12s\n' cluster rs_count version minutes available app_count storage_gb -elif [ "$format" == "csv" ]; then - echo "cluster,rs_count,version,minutes,available,table_count,storage_gb" -else - echo "ERROR: invalid format: $format" - exit 1 -fi -cluster_count=0 -rs_count_sum=0 -app_count_sum=0 -data_size_sum=0 -while read cluster -do - rs_count=`echo server_info | ./run.sh shell -n $cluster 2>&1 | grep 'replica-server' | wc -l` - rs_version=`echo server_info | ./run.sh shell -n $cluster 2>&1 | grep 'replica-server' | \ - grep -o 'Pegasus Server [^ ]*' | head -n 1 | sed 's/SNAPSHOT/SN/' | awk '{print $3}'` - result=`./scripts/pegasus_stat_available.sh $cluster $months` - if echo $result | grep '^ERROR'; then - echo "ERROR: process cluster $cluster failed" - continue - fi - minutes=`echo $result | awk '{print $2}'` - available=`echo $result | awk '{print $3}' | sed 's/data/-/'` - app_count=`echo $result | awk '{print $4}'` - data_size=`echo $result | awk '{print $5}'` - if [ "$available" == "1.000000" ]; then - available_str="99.9999%" - elif [ "$available" == "0" ]; then - available_str="00.0000%" - else - available_str="${available:2:2}.${available:4:4}%" - fi - if [ "$format" == "table" ]; then - printf '%-30s%-12s%-12s%-12s%-12s%-12s%-12s\n' $cluster $rs_count $rs_version $minutes $available $app_count $data_size - elif [ "$format" == "csv" ]; then - echo -e "$cluster,$rs_count,$rs_version,$minutes,=\"$available_str\",$app_count,$data_size" - else - echo "ERROR: invalid format: $format" - exit 1 - fi - cluster_count=$((cluster_count + 1)) - rs_count_sum=$((rs_count_sum + rs_count)) - app_count_sum=$((app_count_sum + app_count)) - data_size_sum=$((data_size_sum + data_size)) -done <$clusters_file - -minutes=`cat $all_result | wc -l` -if [ $minutes -eq 0 ]; then - available="0.000000" -else - available=`cat $all_result | grep -o '[0-9]*,[0-9]*,[0-9]*' | awk -F, '{a+=$1;b+=$2}END{printf("%f\n",(double)b/a);}'` -fi - -if [ "$available" == "1.000000" ]; then - available_str="99.9999%" -elif [ "$available" == "0" ]; then - available_str="00.0000%" -else - available_str="${available:2:2}.${available:4:4}%" -fi - -if [ "$format" == "table" ]; then - printf '%-30s%-12s%-12s%-12s%-12s%-12s%-12s\n' "(total:$cluster_count)" $rs_count_sum "-" $minutes $available $app_count_sum $data_size_sum - echo -elif [ "$format" == "csv" ]; then - echo -e "(total:$cluster_count),$rs_count_sum,,$minutes,=\"$available_str\",$app_count_sum,$data_size_sum" -else - echo "ERROR: invalid format: $format" - exit 1 -fi - -rm $all_result &>/dev/null - diff --git a/scripts/create_table.py b/scripts/create_table.py deleted file mode 100755 index 557001d80f18f704856e151d1bf176f0c747794d..0000000000000000000000000000000000000000 --- a/scripts/create_table.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -HOWTO -===== - -./scripts/create_table.py --table ai_user_info \ - --depart 云平台部-存储平台-KV系统组 \ - --user wutao1&qinzuoyan \ - --cluster bj1-ai \ - --write_throttling "2000*delay*100" \ - --partition_count 16 - -OR - -./scripts/create_table.py -t ai_user_info \ - -d 云平台部-存储平台-KV系统组 \ - -u wutao1&qinzuoyan \ - -c bj1-ai \ - -w "2000*delay*100" \ - -p 16 - -DEVLOPER GUIDE -============== - -The source code is formatted using autopep8. -Ensure you have run formatter before committing changes. -``` -autopep8 -i --aggressive --aggressive scripts/create_table.py -``` - -TODO(wutao1): automatically set write throttling according to the given - estimated QPS on the table. -""" - -import os -import click -import py_utils -import re -import json -import math - - -def validate_param_table(ctx, param, value): - # TODO(wutao1): check illegal characters - return value.encode('utf-8') - - -def validate_param_depart(ctx, param, value): - return value.encode('utf-8') - - -def validate_param_user(ctx, param, value): - return value.encode('utf-8') - - -def validate_param_cluster(ctx, param, value): - return value.encode('utf-8') - - -def validate_param_partition_count(ctx, param, value): - if value == 0: - raise click.BadParameter("Cannot create table with 0 partition") - if math.log(value, 2) != math.floor(math.log(value, 2)): - raise click.BadParameter( - "Partition count {} should be a power of 2".format(value)) - return value - - -def validate_param_write_throttling(ctx, param, value): - if value == '': - return None - pattern = re.compile(r'^\d+\*delay\*\d+(,\d+\*reject\*\d+)?$') - match = pattern.match(value) - if match is not None: - return value.encode('utf-8') - else: - raise click.BadParameter( - 'invalid value of throttle \'%s\'' % value) - - -def create_table_if_needed(cluster, table, partition_count): - if not cluster.has_table(table): - try: - # TODO(wutao1): Outputs progress while polling. - py_utils.echo("Creating table {}...".format(table)) - cluster.create_table(table, partition_count) - except Exception as err: - py_utils.echo(err, "red") - exit(1) - else: - py_utils.echo("Success: table \"{}\" exists".format(table)) - - -def set_business_info_if_needed(cluster, table, depart, user): - new_business_info = "depart={},user={}".format(depart, user) - set_app_envs_if_needed(cluster, table, 'business.info', new_business_info) - - -def set_write_throttling_if_needed(cluster, table, new_throttle): - if new_throttle is None: - return - set_app_envs_if_needed( - cluster, table, 'replica.write_throttling', new_throttle) - - -def set_app_envs_if_needed(cluster, table, env_name, new_env_value): - py_utils.echo("New value of {}={}".format(env_name, new_env_value)) - envs = cluster.get_app_envs(table) - if envs is not None and envs.get(env_name) is not None: - old_env_value = envs.get(env_name).encode('utf-8') - if old_env_value is not None: - py_utils.echo("Old value of {}={}".format(env_name, old_env_value)) - if old_env_value == new_env_value: - py_utils.echo("Success: {} keeps unchanged".format(env_name)) - return - cluster.set_app_envs(table, env_name, - new_env_value) - - -def all_arguments_to_string( - table, - depart, - user, - cluster, - partition_count, - write_throttling): - return json.dumps({ - 'table': table, - 'depart': depart, - 'user': user, - 'cluster': cluster, - 'partition_count': partition_count, - 'write_throttling': write_throttling, - }, sort_keys=True, indent=4, ensure_ascii=False, encoding='utf-8') - - -@click.command() -@click.option("--table", "-t", - required=True, - callback=validate_param_table, - help="Name of the table you want to create.") -@click.option( - "--depart", "-d", - required=True, - callback=validate_param_depart, - help="Department of the table owner. If there are more than one levels of department, use '-' to concatenate them.") -@click.option( - "--user", "-u", - required=True, - callback=validate_param_user, - help="The table owner. If there are more than one owners, use '&' to concatenate them.") -@click.option("--cluster", "-c", - required=True, - callback=validate_param_cluster, - help="The cluster name. Where you want to place the table.") -@click.option("--partition_count", "-p", - callback=validate_param_partition_count, - help="The partition count of the table. Empty means no create.", - type=int) -@click.option( - "--write_throttling", "-w", - default="", - callback=validate_param_write_throttling, - help="{delay_qps_threshold}*delay*{delay_ms},{reject_qps_threshold}*reject*{delay_ms_before_reject}") -def main(table, depart, user, cluster, partition_count, write_throttling): - if not click.confirm( - "Confirm to create table:\n{}\n".format( - all_arguments_to_string( - table, - depart, - user, - cluster, - partition_count, - write_throttling))): - return - c = py_utils.PegasusCluster(cluster_name=cluster) - create_table_if_needed(c, table, partition_count) - set_business_info_if_needed(c, table, depart, user) - set_write_throttling_if_needed(c, table, write_throttling) - - -if __name__ == "__main__": - main() diff --git a/scripts/falcon_screen.json b/scripts/falcon_screen.json deleted file mode 100644 index 6f051878672d17a3805b36bd846aaa8ef9f465fa..0000000000000000000000000000000000000000 --- a/scripts/falcon_screen.json +++ /dev/null @@ -1,849 +0,0 @@ -{ - "comments": [ - { - "title": "graph名称", - "endpoints": ["机器名或者tag标识，tag之间用空格分隔"], - "counters": ["counter名称"], - "graph_type": "展示类型，endpoint视角为h，counters视角为k,组合视角为a", - "method": "绘图是否进行求和，求和填写sum，不求和填写空字符串", - "timespan": "展示的时间跨度，单位为秒" - } - ], - "version": "20180625", - "graphs": [ - { - "title": "集群可用度（单位：百分比*10000；1M表示100%）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*cluster.available.minute/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各操作总QPS（统计get、multi_get、put、multi_put、remove、multi_remove、scan各操作的总QPS）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.get_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.multi_get_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.put_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.multi_put_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.remove_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.multi_remove_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.incr_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.check_and_set_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.scan_qps#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "集群读写吞吐量（统计最近10s内的读写吞吐量，单位：Capacity Unit）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.recent_read_cu#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.recent_write_cu#_all_/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "集群Load-Balance状态（待执行的balance操作数、已执行的balance操作数等）", - "endpoints": ["cluster=${cluster.name} job=meta service=pegasus"], - "counters": [ - "meta*eon.greedy_balancer*balance_operation_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.greedy_balancer*recent_balance_move_primary_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.greedy_balancer*recent_balance_copy_primary_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.greedy_balancer*recent_balance_copy_secondary_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各ReplicaServer内存用量（单位：MB）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*server*memused.res(MB)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点存储使用率（百分比）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "df.bytes.used.percent/fstype=ext4,mount=/home", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd1", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd2", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd3", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd4", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd5", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd6", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd7", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd8", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd9", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd10", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd11", - "df.bytes.used.percent/fstype=ext4,mount=/home/work/ssd12" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点内存使用率（百分比）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "mem.memused.percent" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各表存储用量（统计各表的单备份数据存储用量；单位：MB）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.storage_mb#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各表RocksDB缓存命中率（统计各表的RocksDB Block Cache命中率；单位：百分比*10000；1M表示100%）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.rdb_block_cache_hit_rate#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Get 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_GET.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Get 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_GET.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 MultiGet 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_GET.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 MultiGet 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_GET.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Set 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_PUT.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Set 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_PUT.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 MultiSet 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 MultiSet 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Del 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_REMOVE.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Del 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_REMOVE.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 MultiDel 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_REMOVE.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 MultiDel 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_REMOVE.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Incr 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_INCR.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Incr 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_INCR.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 CheckAndSet 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_CHECK_AND_SET.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 CheckAndSet 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_CHECK_AND_SET.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 CheckAndMutate 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_CHECK_AND_MUTATE.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 CheckAndMutate 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_CHECK_AND_MUTATE.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Scan 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_SCAN.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Scan 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_SCAN.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Prepare 发送端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_PREPARE_ACK.latency.client(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Prepare 发送端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_PREPARE_ACK.latency.client(ns).p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 Prepare 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_PREPARE.latency.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P999 Prepare 服务端延迟（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_PREPARE.latency.server.p999/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点Replica个数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*replica(Count)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点Commit QPS", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*replicas.commit.qps/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点SharedLog大小（单位：MB）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*shared.log.size(MB)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点SharedLog最近写入字节数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*shared.log.recent.write.size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "集群Partition健康状况（处于heathy、writable_ill、unwritable、unreadable、dead状态的partition个数）", - "endpoints": ["cluster=${cluster.name} job=meta service=pegasus"], - "counters": [ - "meta*eon.server_state*dead_partition_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*unreadable_partition_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*unwritable_partition_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*writable_ill_partition_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*healthy_partition_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "集群Config更新情况（节点失联个数、config变化次数等）", - "endpoints": ["cluster=${cluster.name} job=meta service=pegasus"], - "counters": [ - "meta*eon.meta_service*recent_disconnect_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.meta_service*unalive_nodes/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*recent_update_config_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*recent_partition_change_unwritable_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_state*recent_partition_change_writable_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus", - "meta*eon.server_load_balancer*recent_choose_primary_fail_count/cluster=${cluster.name},job=meta,port=${meta.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点ReplicaServer异常统计（心跳失败次数、Prepare失败次数、Error文件夹个数、Garbage文件夹个数等）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.failure_detector*recent_beacon_fail_count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.error.replica.dir.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.garbage.replica.dir.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.recent.prepare.fail.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.recent.replica.move.error.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.recent.replica.move.garbage.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.recent.replica.remove.dir.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点Learning相关统计（执行次数、执行时间、传输数据量等）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.nfs_client*recent_copy_data_size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.nfs_client*recent_copy_fail_count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.nfs_client*recent_write_data_size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.nfs_client*recent_write_fail_count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.nfs_server*recent_copy_data_size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.nfs_server*recent_copy_fail_count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.max.copy.file.size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.max.duration.time(ms)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.copy.buffer.size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.copy.file.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.copy.file.size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.learn.app.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.learn.cache.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.learn.fail.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.learn.log.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.learn.reset.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.learn.succ.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.round.start.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*replicas.learning.recent.start.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点Cold-Backup相关统计（执行次数、执行时间、上传数据量等）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*cold.backup.max.duration.time.ms/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.max.upload.file.size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.cancel.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.fail.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.pause.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.start.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.succ.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.upload.file.fail.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.upload.file.size/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.recent.upload.file.succ.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*cold.backup.running.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点Manual-Compact相关统计（当前执行个数等）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*app.pegasus*manual.compact.running.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "CPU Busy", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "cpu.busy" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Network Dropped", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "net.if.total.dropped/iface=eth0" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Network In Bytes", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "net.if.in.bytes/iface=eth0" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Network Out Bytes", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "net.if.out.bytes/iface=eth0" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "SSD Util", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "disk.io.util/device=sdb", - "disk.io.util/device=sdc", - "disk.io.util/device=sdd", - "disk.io.util/device=sde", - "disk.io.util/device=sdf", - "disk.io.util/device=sdg", - "disk.io.util/device=sdh", - "disk.io.util/device=sdi", - "disk.io.util/device=sdj", - "disk.io.util/device=sdk", - "disk.io.util/device=sdl", - "disk.io.util/device=sdm", - "disk.io.util/device=vda", - "disk.io.util/device=vdb", - "disk.io.util/device=vdc", - "disk.io.util/device=vdd", - "disk.io.util/device=vde", - "disk.io.util/device=xvda", - "disk.io.util/device=xvdb", - "disk.io.util/device=xvdc", - "disk.io.util/device=xvdd", - "disk.io.util/device=xvde" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "SSD Await", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "disk.io.await/device=sdb", - "disk.io.await/device=sdc", - "disk.io.await/device=sdd", - "disk.io.await/device=sde", - "disk.io.await/device=sdf", - "disk.io.await/device=sdg", - "disk.io.await/device=sdh", - "disk.io.await/device=sdi", - "disk.io.await/device=sdj", - "disk.io.await/device=sdk", - "disk.io.await/device=sdl", - "disk.io.await/device=sdm", - "disk.io.await/device=vda", - "disk.io.await/device=vdb", - "disk.io.await/device=vdc", - "disk.io.await/device=vdd", - "disk.io.await/device=vde", - "disk.io.await/device=xvda", - "disk.io.await/device=xvdb", - "disk.io.await/device=xvdc", - "disk.io.await/device=xvdd", - "disk.io.await/device=xvde" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近Flush次数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*app.pegasus*recent.flush.completed.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近Compaction次数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*app.pegasus*recent.compaction.completed.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近Flush写出字节数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*app.pegasus*recent.flush.output.bytes/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近Compaction写入写出字节数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*app.pegasus*recent.compaction.input.bytes/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*app.pegasus*recent.compaction.output.bytes/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近Emergency Checkpoint触发次数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*recent.trigger.emergency.checkpoint.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近Write Stall触发次数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*app.pegasus*recent.write.change.delayed.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*app.pegasus*recent.write.change.stopped.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 单条读 排队时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_GET.queue(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 单条读 执行时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_GET.exec(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 多条读 排队时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_GET.queue(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 多条读 执行时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_GET.exec(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 单条写 排队时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_PUT.queue(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 单条写 执行时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_PUT.exec(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 多条写 排队时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.queue(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "P99 多条写 执行时间（单位：纳秒）", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.exec(ns)/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点最近读写失败次数", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "replica*eon.replica_stub*recent.read.fail.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "replica*eon.replica_stub*recent.write.fail.count/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "异常查询条数（统计各表最近10秒执行时间超过100毫秒的查询条数）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.recent_abnormal_count#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Expire数据条数（统计各表最近10秒查询的过期数据条数）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.recent_expire_count#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Filter数据条数（统计各表最近10秒过滤的数据条数）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.recent_filter_count#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Delay数据条数（统计各表最近10秒write throttling delay的数据条数）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.recent_write_throttling_delay_count#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "Reject数据条数（统计各表最近10秒write throttling reject的数据条数）", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.recent_write_throttling_reject_count#${for.each.table}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "【${for.each.table}】单表QPS", - "endpoints": ["cluster=${cluster.name} job=collector service=pegasus"], - "counters": [ - "collector*app.pegasus*app.stat.get_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.multi_get_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.put_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.multi_put_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.remove_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.multi_remove_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.incr_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.check_and_set_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus", - "collector*app.pegasus*app.stat.scan_qps#${table.name}/cluster=${cluster.name},job=collector,port=${collector.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - }, - { - "title": "各节点 P99 RPC 报文长度", - "endpoints": ["cluster=${cluster.name} job=replica service=pegasus"], - "counters": [ - "zion*profiler*RPC_RRDB_RRDB_PUT.size.request.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.size.request.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "zion*profiler*RPC_RRDB_RRDB_GET.size.response.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus", - "zion*profiler*RPC_RRDB_RRDB_MULTI_GET.size.response.server/cluster=${cluster.name},job=replica,port=${replica.port},service=pegasus" - ], - "graph_type": "a", - "method": "", - "timespan": 86400 - } - ] -} diff --git a/scripts/falcon_screen.py b/scripts/falcon_screen.py deleted file mode 100755 index 9617e53d86414c2a4dc6247832b063e33cb9ba7f..0000000000000000000000000000000000000000 --- a/scripts/falcon_screen.py +++ /dev/null @@ -1,600 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import requests -import json -import re -import sys -import os -import copy - -# -# RESTful API doc: http://wiki.n.miui.com/pages/viewpage.action?pageId=66037692 -# falcon ctrl api: http://dev.falcon.srv/doc/ -# - -# account info -serviceAccount = "" -serviceSeedMd5 = "" - -############################################################################### - -# global variables -falconServiceUrl = "http://falcon.srv" -pegasusScreenId = 18655 -sessionId = "" -metaPort = "" -replicaPort = "" -collectorPort = "" - -# return: bool -def get_service_port_by_minos(clusterName): - minosEnv = os.environ.get("MINOS_CONFIG_FILE") - if not isinstance(minosEnv, str) or len(minosEnv) == 0: - print "WARN: environment variables 'MINOS_CONFIG_FILE' is not set" - return False - if not os.path.isfile(minosEnv): - print "WARN: environment variables 'MINOS_CONFIG_FILE' is not valid" - return False - minosConfigDir = os.path.dirname(minosEnv) - if not os.path.isdir(minosConfigDir): - print "WARN: environment variables 'MINOS_CONFIG_FILE' is not valid" - return False - clusterConfigFile = minosConfigDir + "/xiaomi-config/conf/pegasus/pegasus-" + clusterName + ".cfg" - if not os.path.isfile(clusterConfigFile): - print "WARN: cluster config file '%s' not exist" % clusterConfigFile - return False - - lines = [line.strip() for line in open(clusterConfigFile)] - mode = '' - global metaPort - global replicaPort - global collectorPort - for line in lines: - if line == '[meta]': - mode = 'meta' - elif line == '[replica]': - mode = 'replica' - elif line == '[collector]': - mode = 'collector' - m = re.search('^base_port *= *([0-9]+)', line) - if m: - basePort = int(m.group(1)) - if mode == 'meta': - metaPort = str(basePort + 1) - elif mode == 'replica': - replicaPort = str(basePort + 1) - elif mode == 'collector': - collectorPort = str(basePort + 1) - mode = '' - - print "INFO: metaPort = %s, replicaPort = %s, collectorPort = %s" % (metaPort, replicaPort, collectorPort) - if metaPort == '' or replicaPort == '' or collectorPort == '': - print "WARN: get port from cluster config file '%s' failed" % clusterConfigFile - return False - return True - - -# return: bool -def get_service_port_by_minos2(clusterName): - minosEnv = os.environ.get("MINOS2_CONFIG_FILE") - if not isinstance(minosEnv, str) or len(minosEnv) == 0: - print "WARN: environment variables 'MINOS2_CONFIG_FILE' is not set" - return False - if not os.path.isfile(minosEnv): - print "WARN: environment variables 'MINOS2_CONFIG_FILE' is not valid" - return False - minosConfigDir = os.path.dirname(minosEnv) - if not os.path.isdir(minosConfigDir): - print "WARN: environment variables 'MINOS2_CONFIG_FILE' is not valid" - return False - clusterConfigFile = minosConfigDir + "/xiaomi-config/conf/pegasus/pegasus-" + clusterName + ".yaml" - if not os.path.isfile(clusterConfigFile): - print "WARN: cluster config file '%s' not exist" % clusterConfigFile - return False - - lines = [line.strip() for line in open(clusterConfigFile)] - mode = '' - global metaPort - global replicaPort - global collectorPort - for line in lines: - if line == 'meta:': - mode = 'meta' - elif line == 'replica:': - mode = 'replica' - elif line == 'collector:': - mode = 'collector' - m = re.search('^base *: *([0-9]+)', line) - if m: - basePort = int(m.group(1)) - if mode == 'meta': - metaPort = str(basePort + 1) - elif mode == 'replica': - replicaPort = str(basePort + 1) - elif mode == 'collector': - collectorPort = str(basePort + 1) - mode = '' - - print "INFO: metaPort = %s, replicaPort = %s, collectorPort = %s" % (metaPort, replicaPort, collectorPort) - if metaPort == '' or replicaPort == '' or collectorPort == '': - print "WARN: get port from cluster config file '%s' failed" % clusterConfigFile - return False - - return True - - -# return: -def get_session_id(): - url = falconServiceUrl + "/v1.0/auth/info" - headers = { - "Accept": "text/plain" - } - - r = requests.get(url, headers=headers) - if r.status_code != 200: - print "ERROR: get_session_id failed, status_code = %s, result:\n%s" % (r.status_code, r.text) - sys.exit(1) - - c = r.headers['Set-Cookie'] - m = re.search('falconSessionId=([^;]+);', c) - if m: - global sessionId - sessionId = m.group(1) - print "INFO: sessionId =", sessionId - else: - print "ERROR: get_session_id failed, cookie not set" - sys.exit(1) - - -# return: -def auth_by_misso(): - url = falconServiceUrl + "/v1.0/auth/callback/misso" - headers = { - "Cookie": "falconSessionId=" + sessionId, - "Authorization": serviceAccount + ";" + serviceSeedMd5 + ";" + serviceSeedMd5 - } - - r = requests.get(url, headers=headers) - if r.status_code != 200: - print "ERROR: auth_by_misso failed, status_code = %s, result:\n%s" % (r.status_code, r.text) - sys.exit(1) - - -# return: -def check_auth_info(): - url = falconServiceUrl + "/v1.0/auth/info" - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.get(url, headers=headers) - if r.status_code != 200: - print "ERROR: check_auth_info failed, status_code = %s, result:\n%s" % (r.status_code, r.text) - sys.exit(1) - - j = json.loads(r.text) - if "user" not in j or j["user"] is None or "name" not in j["user"] or j["user"]["name"] != serviceAccount: - print "ERROR: check_auth_info failed, bad json result:\n%s" % r.text - sys.exit(1) - - -def login(): - get_session_id() - auth_by_misso() - check_auth_info() - print "INFO: login succeed" - - -# return: -def logout(): - url = falconServiceUrl + "/v1.0/auth/logout" - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.get(url, headers=headers) - if r.status_code != 200: - print "ERROR: logout failed, status_code = %s, result:\n%s" % (r.status_code, r.text) - sys.exit(1) - - print "INFO: logout succeed" - - -# return: screenId -def create_screen(screenName): - url = falconServiceUrl + "/v1.0/dashboard/screen" - headers = { - "Cookie": "falconSessionId=" + sessionId - } - req = { - "pid" : pegasusScreenId, - "name" : screenName - } - - r = requests.post(url, headers=headers, data=json.dumps(req)) - if r.status_code != 200: - print "ERROR: create_screen failed, screenName = %s, status_code = %s, result:\n%s" \ - % (screenName, r.status_code, r.text) - sys.exit(1) - - j = json.loads(r.text) - if "id" not in j: - print "ERROR: create_screen failed, screenName = %s, bad json result\n%s" \ - % (screenName, r.text) - sys.exit(1) - - screenId = j["id"] - print "INFO: create_screen succeed, screenName = %s, screenId = %s" % (screenName, screenId) - return screenId - - -# return: screenConfig -def prepare_screen_config(clusterName, screenTemplateFile, tableListFile): - tableList = [] - lines = [line.strip() for line in open(tableListFile)] - for line in lines: - if len(line) > 0: - if line in tableList: - print "ERROR: bad table list file: duplicate table '%s'" % line - sys.exit(1) - tableList.append(line) - if len(tableList) == 0: - print "ERROR: bad table list file: should be non-empty list" - sys.exit(1) - - jsonData = open(screenTemplateFile).read() - screenJson = json.loads(jsonData) - graphsJson = screenJson["graphs"] - if not isinstance(graphsJson, list) or len(graphsJson) == 0: - print "ERROR: bad screen template json: [graphs] should be provided as non-empty list" - sys.exit(1) - - # resolve ${for.each.table} in title and ${table.name} in counters - newGraphsJson = [] - titleSet = [] - for graphJson in graphsJson: - title = graphJson["title"] - if not isinstance(title, (str, unicode)) or len(title) == 0: - print type(title) - print "ERROR: bad screen template json: [graphs]: [title] should be provided as non-empty str" - sys.exit(1) - if title.find("${for.each.table}") != -1: - for table in tableList: - newTitle = title.replace("${for.each.table}", table) - if newTitle in titleSet: - print "ERROR: bad screen template json: [graphs][%s]: duplicate resolved title '%s' " % (title, newTitle) - sys.exit(1) - newGraphJson = copy.deepcopy(graphJson) - counters = newGraphJson["counters"] - if not isinstance(counters, list) or len(counters) == 0: - print "ERROR: bad screen template json: [graphs][%s]: [counters] should be provided as non-empty list" % title - sys.exit(1) - newCounters = [] - for counter in counters: - if len(counter) != 0: - newCounter = counter.replace("${table.name}", table) - newCounters.append(newCounter) - if len(newCounters) == 0: - print "ERROR: bad screen template json: [graphs][%s]: [counters] should be provided as non-empty list" % title - sys.exit(1) - newGraphJson["counters"] = newCounters - newGraphJson["title"] = newTitle - newGraphsJson.append(newGraphJson) - titleSet.append(newTitle) - else: - if title in titleSet: - print "ERROR: bad screen template json: [graphs][%s]: duplicate title" % title - sys.exit(1) - newGraphsJson.append(graphJson) - titleSet.append(title) - - screenConfig = [] - position = 1 - for graphJson in newGraphsJson: - title = graphJson["title"] - - endpoints = graphJson["endpoints"] - if not isinstance(endpoints, list) or len(endpoints) == 0: - print "ERROR: bad screen template json: [graphs][%s]: [endpoints] should be provided as non-empty list" % title - sys.exit(1) - newEndpoints = [] - for endpoint in endpoints: - if len(endpoint) != 0: - newEndpoint = endpoint.replace("${cluster.name}", clusterName).replace("${meta.port}", metaPort) - newEndpoint = newEndpoint.replace("${replica.port}", replicaPort).replace("${collector.port}", collectorPort) - newEndpoints.append(newEndpoint) - if len(newEndpoints) == 0: - print "ERROR: bad screen template json: [graphs][%s]: [endpoints] should be provided as non-empty list" % title - sys.exit(1) - - counters = graphJson["counters"] - if not isinstance(counters, list) or len(counters) == 0: - print "ERROR: bad screen template json: [graphs][%s]: [counters] should be provided as non-empty list" % title - sys.exit(1) - newCounters = [] - for counter in counters: - if len(counter) != 0: - newCounter = counter.replace("${cluster.name}", clusterName).replace("${meta.port}", metaPort) - newCounter = newCounter.replace("${replica.port}", replicaPort).replace("${collector.port}", collectorPort) - if newCounter.find("${for.each.table}") != -1: - for table in tableList: - newCounters.append(newCounter.replace("${for.each.table}", table)) - else: - newCounters.append(newCounter) - if len(newCounters) == 0: - print "ERROR: bad screen template json: [graphs][%s]: [counters] should be provided as non-empty list" % title - sys.exit(1) - - graphType = graphJson["graph_type"] - if not isinstance(graphType, (str, unicode)): - print "ERROR: bad screen template json: [graphs][%s]: [graph_type] should be provided as str" % title - sys.exit(1) - if graphType != "h" and graphType != "k" and graphType != "a": - print "ERROR: bad screen template json: [graphs][%s]: [graph_type] should be 'h' or 'k' or 'a'" % title - sys.exit(1) - - method = graphJson["method"] - if not isinstance(method, (str, unicode)): - print "ERROR: bad screen template json: [graphs][%s]: [method] should be provided as str" % title - sys.exit(1) - if method != "" and method != "sum": - print "ERROR: bad screen template json: [graphs][%s]: [method] should be '' or 'sum'" % title - sys.exit(1) - - timespan = graphJson["timespan"] - if not isinstance(timespan, int) or timespan <= 0: - print "ERROR: bad screen template json: [graphs][%s]: [timespan] should be provided as positive int" % title - sys.exit(1) - - graphConfig = {} - graphConfig["counters"] = newCounters - graphConfig["endpoints"] = newEndpoints - graphConfig["falcon_tags"] = "" - graphConfig["graph_type"] = graphType - graphConfig["method"] = method - graphConfig["position"] = position - graphConfig["timespan"] = timespan - graphConfig["title"] = title - screenConfig.append(graphConfig) - - position += 1 - - return screenConfig - - -# return: graphId -def create_graph(graphConfig): - url = falconServiceUrl + "/v1.0/dashboard/graph" - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.post(url, headers=headers, data=json.dumps(graphConfig)) - if r.status_code != 200: - print "ERROR: create_graph failed, graphTitle = \"%s\", status_code = %s, result:\n%s" \ - % (graphConfig["title"], r.status_code, r.text) - sys.exit(1) - - j = json.loads(r.text) - if "id" not in j: - print "ERROR: create_graph failed, graphTitle = \"%s\", bad json result\n%s" \ - % (graphConfig["title"], r.text) - sys.exit(1) - - graphId = j["id"] - print "INFO: create_graph succeed, graphTitle = \"%s\", graphId = %s" \ - % (graphConfig["title"], graphId) - - # udpate graph position immediately - graphConfig["id"] = graphId - update_graph(graphConfig, "position") - - return graphId - - -# return: screen[] -def get_screens(): - url = falconServiceUrl + "/v1.0/dashboard/screen/pid/" + str(pegasusScreenId) - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.get(url, headers=headers) - if r.status_code != 200: - print "ERROR: get_screens failed, status_code = %s, result:\n%s" % (r.status_code, r.text) - sys.exit(1) - - j = json.loads(r.text) - - print "INFO: get_screens succeed, screenCount = %s" % len(j) - return j - - -# return: graph[] -def get_screen_graphs(screenName, screenId): - url = falconServiceUrl + "/v1.0/dashboard/graph/screen/" + str(screenId) - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.get(url, headers=headers) - if r.status_code != 200: - print "ERROR: get_screen_graphs failed, screenName = %s, screenId = %s, status_code = %s, result:\n%s" \ - % (screenName, screenId, r.status_code, r.text) - sys.exit(1) - - j = json.loads(r.text) - - print "INFO: get_screen_graphs succeed, screenName = %s, screenId = %s, graphCount = %s" \ - % (screenName, screenId, len(j)) - return j - - -# return: -def delete_graph(graphTitle, graphId): - url = falconServiceUrl + "/v1.0/dashboard/graph/" + str(graphId) - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.delete(url, headers=headers) - if r.status_code != 200 or r.text.find("delete success!") == -1: - print "ERROR: delete_graph failed, graphTitle = \"%s\", graphId = %s, status_code = %s, result:\n%s" \ - % (graphTitle, graphId, r.status_code, r.text) - sys.exit(1) - - print "INFO: delete_graph succeed, graphTitle = \"%s\", graphId = %s" % (graphTitle, graphId) - - -# return: -def update_graph(graphConfig, updateReason): - url = falconServiceUrl + "/v1.0/dashboard/graph/" + str(graphConfig["id"]) - headers = { - "Cookie": "falconSessionId=" + sessionId - } - - r = requests.put(url, headers=headers, data=json.dumps(graphConfig)) - if r.status_code != 200: - print "ERROR: update_graph failed, graphTitle = \"%s\", graphId = %s, status_code = %s, result:\n%s" \ - % (graphConfig["title"], graphConfig["id"], r.status_code, r.text) - sys.exit(1) - - j = json.loads(r.text) - if "id" not in j: - print "ERROR: update_graph failed, graphTitle = \"%s\", graphId = %s, bad json result\n%s" \ - % (graphConfig["title"], graphConfig["id"], r.text) - sys.exit(1) - - print "INFO: update_graph succeed, graphTitle = \"%s\", graphId = %s, updateReason = \"%s changed\"" \ - % (graphConfig["title"], graphConfig["id"], updateReason) - - -# return: bool, reason -def is_equal(graph1, graph2): - if graph1["title"] != graph2["title"]: - return False, "title" - if graph1["graph_type"] != graph2["graph_type"]: - return False, "graph_type" - if graph1["method"] != graph2["method"]: - return False, "method" - if graph1["position"] != graph2["position"]: - return False, "position" - if graph1["timespan"] != graph2["timespan"]: - return False, "timespan" - endpoints1 = graph1["endpoints"] - endpoints2 = graph2["endpoints"] - if len(endpoints1) != len(endpoints2): - return False, "endpoints" - for endpoint in endpoints1: - if not endpoint in endpoints2: - return False, "endpoints" - counters1 = graph1["counters"] - counters2 = graph2["counters"] - if len(counters1) != len(counters2): - return False, "counters" - for counter in counters1: - if not counter in counters2: - return False, "counters" - return True, "" - - -if __name__ == '__main__': - if serviceAccount == "" or serviceSeedMd5 == "": - print "ERROR: please set 'serviceAccount' and 'serviceSeedMd5' in %s" % sys.argv[0] - sys.exit(1) - - if len(sys.argv) != 5: - print "USAGE: python %s " % sys.argv[0] - sys.exit(1) - - clusterName = sys.argv[1] - screenTemplateFile = sys.argv[2] - tableListFile = sys.argv[3] - operateType = sys.argv[4] - - if operateType != "create" and operateType != "update": - print "ERROR: argv[4] should be 'create' or 'update', but '%s'" % operateType - sys.exit(1) - - if not get_service_port_by_minos2(clusterName) and not get_service_port_by_minos(clusterName): - print "ERROR: get service ports from minos config failed" - sys.exit(1) - - login() - - if operateType == "create": - screenConfig = prepare_screen_config(clusterName, screenTemplateFile, tableListFile) - screenId = create_screen(screenName=clusterName) - for graphConfig in screenConfig: - graphConfig["screen_id"] = screenId - create_graph(graphConfig) - print "INFO: %s graphs created" % len(screenConfig) - else: # update - screens = get_screens() - screenId = 0 - oldScreenConfig = None - for screen in screens: - if screen["name"] == clusterName: - screenId = screen["id"] - oldScreenConfig = get_screen_graphs(clusterName, screenId) - if oldScreenConfig is None: - print "ERROR: screen '%s' not exit, please create it first" % clusterName - sys.exit(1) - #print "INFO: old screen config:\n%s" % json.dumps(oldScreenConfig, indent=2) - - newScreenConfig = prepare_screen_config(clusterName, screenTemplateFile, tableListFile) - #print "INFO: new screen config:\n%s" % json.dumps(newScreenConfig, indent=2) - - oldScreenMap = {} - newScreenMap = {} - for graph in oldScreenConfig: - oldScreenMap[graph["title"]] = graph - for graph in newScreenConfig: - newScreenMap[graph["title"]] = graph - deleteConfigList = [] - createConfigList = [] - updateConfigList = [] - for graph in oldScreenConfig: - if not graph["title"] in newScreenMap: - deleteConfigList.append(graph) - for graph in newScreenConfig: - if not graph["title"] in oldScreenMap: - graph["screen_id"] = screenId - createConfigList.append(graph) - else: - oldGraph = oldScreenMap[graph["title"]] - equal, reason = is_equal(graph, oldGraph) - if not equal: - graph["id"] = oldGraph["graph_id"] - graph["screen_id"] = screenId - updateConfigList.append((graph, reason)) - - for graph in deleteConfigList: - delete_graph(graphTitle=graph["title"], graphId=graph["graph_id"]) - for graph in createConfigList: - create_graph(graph) - for graph,reason in updateConfigList: - update_graph(graph, reason) - - print "INFO: %d graphs deleted, %d graphs created, %d graphs updated" \ - % (len(deleteConfigList), len(createConfigList), len(updateConfigList)) - - logout() - diff --git a/scripts/pegasus_check_clusters.py b/scripts/pegasus_check_clusters.py deleted file mode 100755 index 1c8ee60f953e491dfbedd0b0de92df3ffa38c18d..0000000000000000000000000000000000000000 --- a/scripts/pegasus_check_clusters.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Basic usage: - -> vim ~/.bashrc -export PYTHONPATH=$PYTHONPATH:$HOME/.local/lib/python2.7/site-packages/ -export PEGASUS_CONFIG_PATH=$HOME/work/conf_pegasus -export PEGASUS_SHELL_PATH=$HOME/work/pegasus -> pip install --user click -> ./pegasus_check_clusters.py --env c3srv -""" - -import os -import click - -from py_utils import * - - -@click.command() -@click.option( - "--env", default="", help="Env of pegasus cluster, eg. c3srv or c4tst") -@click.option('-v', '--verbose', count=True) -def main(env, verbose): - pegasus_config_path = os.getenv("PEGASUS_CONFIG_PATH") - if pegasus_config_path is None: - echo( - "Please configure environment variable PEGASUS_CONFIG_PATH in your bashrc or zshrc", - "red") - exit(1) - if env != "": - echo("env = " + env) - set_global_verbose(verbose) - clusters = list_pegasus_clusters(pegasus_config_path, env) - for cluster in clusters: - echo("=== " + cluster.name()) - try: - cluster.print_imbalance_nodes() - cluster.print_unhealthy_partitions() - except RuntimeError as e: - echo(str(e), "red") - return - echo("===") - - -if __name__ == "__main__": - main() diff --git a/scripts/pegasus_check_ports.py b/scripts/pegasus_check_ports.py deleted file mode 100755 index e6a7ad5428cfedacc6fd6728b8d3da689acf1154..0000000000000000000000000000000000000000 --- a/scripts/pegasus_check_ports.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Basic usage: - -> vim ~/.bashrc -export PYTHONPATH=$PYTHONPATH:$HOME/.local/lib/python2.7/site-packages/ -export PEGASUS_CONFIG_PATH=$HOME/work/conf_pegasus -export PEGASUS_SHELL_PATH=$HOME/work/pegasus -> pip install --user click -> ./pegasus_check_posts.py --env c3srv -""" - -import os -import click - -from py_utils import * - - -@click.command() -@click.option("--env", help="Env of pegasus cluster, eg. c3srv or c4tst") -def main(env): - pegasus_config_path = os.getenv("PEGASUS_CONFIG_PATH") - if pegasus_config_path is None: - echo( - "Please configure environment variable PEGASUS_CONFIG_PATH in your bashrc or zshrc", - "red") - exit(1) - clusters = list_pegasus_clusters(pegasus_config_path, env) - host_to_ports = {} - for cluster in clusters: - try: - p = cluster.get_meta_port() - h = cluster.get_meta_host() - if not h in host_to_ports: - host_to_ports[h] = set() - if p in host_to_ports[h]: - echo( - "port number conflicted: {0} {1} [{2}]".format( - p, cluster.name(), h), "red") - continue - host_to_ports[h].add(p) - echo("cluster {0}: {1} [{2}]".format(cluster.name(), p, h)) - except RuntimeError as e: - echo(str(e), "red") - return - - echo("") - for h in host_to_ports: - echo("recommended port number for [{0}] is: {1}".format( - h, str(max(host_to_ports[h]) + 1000))) - echo("host [{0}] has in total {1} clusters on it".format( - h, len(host_to_ports[h]))) - echo("") - - -if __name__ == "__main__": - main() diff --git a/scripts/pegasus_falcon_screen.sh b/scripts/pegasus_falcon_screen.sh deleted file mode 100755 index 6313ac0b38be3d6b49b103650c08f95d97f09c69..0000000000000000000000000000000000000000 --- a/scripts/pegasus_falcon_screen.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -PID=$$ - -if [ $# -ne 2 ] -then - echo "This tool is for create or update falcon screen for specified cluster." - echo "USAGE: $0 " - exit 1 -fi - -pwd="$( cd "$( dirname "$0" )" && pwd )" -shell_dir="$( cd $pwd/.. && pwd )" -cd $shell_dir - -operate=$1 -cluster=$2 - -if [ "$operate" != "create" -a "$operate" != "update" ]; then - echo "ERROR: invalid operation type: $operate" - exit 1 -fi - -echo "UID: $UID" -echo "PID: $PID" -echo "cluster: $cluster" -echo "operate: $operate" -echo "Start time: `date`" -all_start_time=$((`date +%s`)) -echo - -cd $shell_dir -echo ls | ./run.sh shell -n $cluster &>/tmp/$UID.$PID.pegasus.ls -grep AVAILABLE /tmp/$UID.$PID.pegasus.ls | awk '{print $3}' >/tmp/$UID.$PID.pegasus.table.list -table_count=`cat /tmp/$UID.$PID.pegasus.table.list | wc -l` -if [ $table_count -eq 0 ]; then - echo "ERROR: table list is empty, please check the cluster $cluster" - exit 1 -fi -cd $pwd - -python falcon_screen.py $cluster falcon_screen.json /tmp/$UID.$PID.pegasus.table.list $operate -if [ $? -ne 0 ]; then - echo "ERROR: falcon screen $operate failed" - exit 1 -fi - -echo -echo "Finish time: `date`" -all_finish_time=$((`date +%s`)) -echo "Falcon screen $operate done, elasped time is $((all_finish_time - all_start_time)) seconds." - -rm -f /tmp/$UID.$PID.pegasus.* &>/dev/null diff --git a/scripts/py_utils/__init__.py b/scripts/py_utils/__init__.py deleted file mode 100644 index 5643f3da92401068042d78dae237a9628f46edfa..0000000000000000000000000000000000000000 --- a/scripts/py_utils/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .lib import set_global_verbose, echo, list_pegasus_clusters, PegasusCluster - -__all__ = [ - 'set_global_verbose', 'echo', 'list_pegasus_clusters', 'PegasusCluster' -] diff --git a/scripts/py_utils/lib.py b/scripts/py_utils/lib.py deleted file mode 100644 index 7c26132f04d0fa5160ce5a3bbd4b756b1cad1f0e..0000000000000000000000000000000000000000 --- a/scripts/py_utils/lib.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import click -import commands -import os -import json - -_global_verbose = False - - -def set_global_verbose(val): - _global_verbose = val - - -def echo(message, color=None): - click.echo(click.style(message, fg=color)) - - -class PegasusCluster(object): - def __init__(self, cfg_file_name=None, cluster_name=None): - if cluster_name is None: - self._cluster_name = os.path.basename(cfg_file_name).replace( - "pegasus-", "").replace(".cfg", "") - else: - self._cluster_name = cluster_name - self._shell_path = os.getenv("PEGASUS_SHELL_PATH") - self._cfg_file_name = cfg_file_name - if self._shell_path is None: - echo( - "Please configure environment variable PEGASUS_SHELL_PATH in your bashrc or zshrc", - "red") - exit(1) - - def print_unhealthy_partitions(self): - list_detail = self._run_shell("ls -d -j").strip() - - list_detail_json = json.loads(list_detail) - read_unhealthy_app_count = int( - list_detail_json["summary"]["read_unhealthy_app_count"]) - write_unhealthy_app_count = int( - list_detail_json["summary"]["write_unhealthy_app_count"]) - if write_unhealthy_app_count > 0: - echo("cluster is write unhealthy, write_unhealthy_app_count = " + - str(write_unhealthy_app_count)) - return - if read_unhealthy_app_count > 0: - echo("cluster is read unhealthy, read_unhealthy_app_count = " + - str(read_unhealthy_app_count)) - return - - def print_imbalance_nodes(self): - nodes_detail = self._run_shell("nodes -d -j").strip() - - primaries_per_node = {} - min_ = 0 - max_ = 0 - for ip_port, node_info in json.loads(nodes_detail)["details"].items(): - primary_count = int(node_info["primary_count"]) - min_ = min(min_, primary_count) - max_ = max(max_, primary_count) - primaries_per_node[ip_port] = primary_count - if float(min_) / float(max_) < 0.8: - print json.dumps(primaries_per_node, indent=4) - - def get_meta_port(self): - with open(self._cfg_file_name) as cfg: - for line in cfg.readlines(): - if line.strip().startswith("base_port"): - return int(line.split("=")[1]) - - def get_meta_host(self): - with open(self._cfg_file_name) as cfg: - for line in cfg.readlines(): - if line.strip().startswith("host.0"): - return line.split("=")[1].strip() - - def create_table(self, table, parts): - create_result = self._run_shell( - "create {} -p {}".format(table, parts)).strip() - if "ERR_INVALID_PARAMETERS" in create_result: - raise ValueError("failed to create table \"{}\"".format(table)) - - def get_app_envs(self, table): - envs_result = self._run_shell( - "use {} \n get_app_envs".format(table)).strip()[len("OK\n"):] - if "ERR_OBJECT_NOT_FOUND" in envs_result: - raise ValueError("table {} does not exist".format(table)) - if envs_result == "": - return None - envs_result = self._run_shell( - "use {} \n get_app_envs -j".format(table)).strip()[len("OK\n"):] - return json.loads(envs_result)['app_envs'] - - def set_app_envs(self, table, env_name, env_value): - envs_result = self._run_shell( - "use {} \n set_app_envs {} {}".format( - table, env_name, env_value)).strip()[ - len("OK\n"):] - if "ERR_OBJECT_NOT_FOUND" in envs_result: - raise ValueError("table {} does not exist".format(table)) - - def has_table(self, table): - app_result = self._run_shell("app {} ".format(table)).strip() - return "ERR_OBJECT_NOT_FOUND" not in app_result - - def _run_shell(self, args): - """ - :param args: arguments passed to ./run.sh shell (type `string`) - :return: shell output - """ - global _global_verbose - - cmd = "cd {1}; echo -e \"{0}\" | ./run.sh shell -n {2}".format( - args, self._shell_path, self._cluster_name) - if _global_verbose: - echo("executing command: \"{0}\"".format(cmd)) - - status, output = commands.getstatusoutput(cmd) - if status != 0: - raise RuntimeError("failed to execute \"{0}\": {1}".format( - cmd, output)) - - result = "" - result_begin = False - for line in output.splitlines(): - if line.startswith("The cluster meta list is:"): - result_begin = True - continue - if line.startswith("dsn exit with code"): - break - if result_begin: - result += line + "\n" - return result - - def name(self): - return self._cluster_name - - -def list_pegasus_clusters(config_path, env): - clusters = [] - for fname in os.listdir(config_path): - if not os.path.isfile(config_path + "/" + fname): - continue - if not fname.startswith("pegasus-" + env): - continue - if not fname.endswith(".cfg"): - continue - if fname.endswith("proxy.cfg"): - continue - clusters.append(PegasusCluster(config_path + "/" + fname)) - return clusters diff --git a/scripts/scp-no-interactive b/scripts/scp-no-interactive deleted file mode 100755 index 09e971341391ab87fc1a3d7685b024ca3beb4e82..0000000000000000000000000000000000000000 --- a/scripts/scp-no-interactive +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/expect - -# USAGE: scp-no-interactive - -set timeout 10 -set host [lindex $argv 0] -set username [lindex $argv 1] -set password [lindex $argv 2] -set src_file [lindex $argv 3] -set dest_file [lindex $argv 4] -spawn scp $src_file $username@$host:$dest_file - expect { - "(yes/no)?" - { - send "yes\n" - expect "*assword:" { send "$password\n"} - } - "*assword:" -{ - send "$password\n" -} -} -expect "100%" -expect eof diff --git a/scripts/ssh-no-interactive b/scripts/ssh-no-interactive deleted file mode 100755 index 8ab5bf88e01e2d0e4efe1daf271d9b628ed2317a..0000000000000000000000000000000000000000 --- a/scripts/ssh-no-interactive +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/expect - -# USAGE: ssh-no-interactive - -set timeout 10 -set host [lindex $argv 0] -set username [lindex $argv 1] -set password [lindex $argv 2] -set command [lindex $argv 3] -spawn ssh $username@$host "$command" - expect { - "(yes/no)?" - { - send "yes\n" - expect "*assword:" { send "$password\n"} - } - "*assword:" -{ - send "$password\n" -} -} -expect eof diff --git a/scripts/update_qt_config.sh b/scripts/update_qt_config.sh deleted file mode 100755 index bd61c473e5a416cfd9f476b1f9fb10e536bc8683..0000000000000000000000000000000000000000 --- a/scripts/update_qt_config.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# This is used for updating the meta-data of Qt Creator IDE. - -PREFIX=pegasus -if [ $# -eq 1 ] -then - PREFIX=$1 -fi - -pwd="$( cd "$( dirname "$0" )" && pwd )" -shell_dir="$( cd $pwd/.. && pwd )" -cd $shell_dir - -# config -CONFIG_OUT="${PREFIX}.config" -echo "Generate $CONFIG_OUT" -rm $CONFIG_OUT &>/dev/null -echo "#define __cplusplus 201103L" >>$CONFIG_OUT -echo "#define _DEBUG" >>$CONFIG_OUT -echo "#define DSN_USE_THRIFT_SERIALIZATION" >>$CONFIG_OUT -echo "#define DSN_ENABLE_THRIFT_RPC" >>$CONFIG_OUT -echo "#define DSN_BUILD_TYPE" >>$CONFIG_OUT -echo "#define DSN_BUILD_HOSTNAME" >>$CONFIG_OUT -echo "#define ROCKSDB_PLATFORM_POSIX" >>$CONFIG_OUT -echo "#define OS_LINUX" >>$CONFIG_OUT -echo "#define ROCKSDB_FALLOCATE_PRESENT" >>$CONFIG_OUT -echo "#define GFLAGS google" >>$CONFIG_OUT -echo "#define ZLIB" >>$CONFIG_OUT -echo "#define BZIP2" >>$CONFIG_OUT -echo "#define ROCKSDB_MALLOC_USABLE_SIZE" >>$CONFIG_OUT -#echo "#define __FreeBSD__" >>$CONFIG_OUT -#echo "#define _WIN32" >>$CONFIG_OUT - -# includes -INCLUDES_OUT="${PREFIX}.includes" -echo "Generate $INCLUDES_OUT" -rm $INCLUDES_OUT &>/dev/null -echo "/usr/include" >>$INCLUDES_OUT -echo "/usr/include/c++/4.8" >>$INCLUDES_OUT -echo "/usr/include/x86_64-linux-gnu" >>$INCLUDES_OUT -echo "/usr/include/x86_64-linux-gnu/c++/4.8" >>$INCLUDES_OUT -echo "rdsn/include" >>$INCLUDES_OUT -echo "rdsn/thirdparty/output/include" >>$INCLUDES_OUT -echo "rdsn/include/dsn/dist/failure_detector" >>$INCLUDES_OUT -echo "rdsn/src/dist/replication/client_lib" >>$INCLUDES_OUT -echo "rdsn/src/dist/replication/lib" >>$INCLUDES_OUT -echo "rdsn/src/dist/replication/meta_server" >>$INCLUDES_OUT -echo "rdsn/src/dist/replication/zookeeper" >>$INCLUDES_OUT -echo "rdsn/thirdparty/output/include" >>$INCLUDES_OUT -echo "rdsn/src/dist/block_service/fds" >>$INCLUDES_OUT -echo "rdsn/src/dist/block_service/local" >>$INCLUDES_OUT -echo "rdsn/src" >> $INCLUDES_OUT -echo "rocksdb" >>$INCLUDES_OUT -echo "rocksdb/include" >>$INCLUDES_OUT -echo "src" >>$INCLUDES_OUT -echo "src/include" >>$INCLUDES_OUT -echo "src/redis_protocol/proxy_lib" >>$INCLUDES_OUT - -# files -FILES_OUT="${PREFIX}.files" -echo "Generate $FILES_OUT" -rm $FILES_OUT >&/dev/null -echo "build.sh" >>$FILES_OUT -echo "rdsn/CMakeLists.txt" >>$FILES_OUT -echo "rdsn/bin/dsn.cmake" >>$FILES_OUT -FILES_DIR=" -src rocksdb rdsn scripts -" -for i in $FILES_DIR -do - find $i -name '*.h' -o -name '*.cpp' -o -name '*.c' -o -name '*.cc' \ - -o -name '*.thrift' -o -name '*.ini' -o -name '*.act' \ - -o -name 'CMakeLists.txt' -o -name '*.sh' \ - | grep -v '\\|rdsn\/thirdparty\|\.zk_install' >>$FILES_OUT -done