diff --git a/.github/ISSUE_TEMPLATE/general_question.md b/.github/ISSUE_TEMPLATE/general_question.md index 72cd078d96ece176996bb76e07f842b27cb19825..fde639586f0e8d1d537575c46c1c22cf4deb054e 100644 --- a/.github/ISSUE_TEMPLATE/general_question.md +++ b/.github/ISSUE_TEMPLATE/general_question.md @@ -8,6 +8,6 @@ about: General question that isn't answered in docs Before asking a question, make sure you have: -- Searched open and closed [GitHub issues](https://github.com/XiaoMi/pegasus/issues) +- Searched open and closed [GitHub issues](https://github.com/apache/incubator-pegasus/issues) - Read the documentation: - [Pegasus Doc](https://pegasus.apache.org) diff --git a/docker/build_docker.sh b/docker/build_docker.sh deleted file mode 100755 index c418ff648318a5110042390484ecb5407f80ecb4..0000000000000000000000000000000000000000 --- a/docker/build_docker.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -set -e - -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -ROOT=$( - cd "$(dirname "${SCRIPT_DIR}")" || exit 1 - pwd -) -SERVER_PKG_NAME=$(cat "${ROOT}"/PACKAGE) - -### -# configurable variables -IMAGE_NAME=pegasus:latest -### - -echo "Building image ${IMAGE_NAME}" -cd "${ROOT}" || exit 1 - -if [[ -z ${SERVER_PKG_NAME} ]]; then - echo "SERVER_PKG_NAME is empty" - exit 1 -fi - -if [[ ! -f "${ROOT}"/${SERVER_PKG_NAME}.tar.gz ]]; then - echo "Failed to find package ${SERVER_PKG_NAME}.tar.gz in ${ROOT}" - exit 1 -else - echo "Found package ${ROOT}/${SERVER_PKG_NAME}.tar.gz" -fi - -cd "${ROOT}" || exit 1 -mkdir -p ${IMAGE_NAME} -cp -f docker/dev/linux/entrypoint.sh ${IMAGE_NAME}/entrypoint.sh -cp -f docker/dev/linux/Dockerfile ${IMAGE_NAME}/Dockerfile -sed -i 's/@SERVER_PKG_NAME@/'"${SERVER_PKG_NAME}"'/' ${IMAGE_NAME}/Dockerfile -cp ${SERVER_PKG_NAME}.tar.gz ${IMAGE_NAME} - -cd ${IMAGE_NAME} || exit 1 -docker build --build-arg SERVER_PKG_NAME="${SERVER_PKG_NAME}" -t ${IMAGE_NAME} . - -cd "${ROOT}" || exit 1 -docker images - -# clean up -rm -r ${IMAGE_NAME} diff --git a/docker/clear_onebox.sh b/docker/clear_onebox.sh deleted file mode 100755 index d4462c2794da08ff4098fa4d3cebba9d27147eec..0000000000000000000000000000000000000000 --- a/docker/clear_onebox.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 - -source cluster_args.sh - -if ! [[ -d "${DOCKER_DIR}" ]]; then - echo "Cleared ${DOCKER_DIR} already" - exit 0 -fi -cd "${DOCKER_DIR}" || exit 1 -pwd - -docker-compose kill -docker-compose rm -f -v - -cd "${ROOT}" || exit 1 - -./run.sh clear_zk -sudo rm -rf "${DOCKER_DIR}" diff --git a/docker/cluster_args.sh b/docker/cluster_args.sh deleted file mode 100755 index a1e7787112b2f84cb8c2bc12e6e41b3685242b99..0000000000000000000000000000000000000000 --- a/docker/cluster_args.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# Configure the following variables to customize the docker cluster. # - -export PARTITION_COUNT=8 -export APP_NAME=temp - -# The ip prefix for each nodes. -# Meta-x's ip address is 172.21.0.1{x}:34601. -# Replica-x's ip address is 172.21.0.2{x}:34801 -export NODE_IP_PREFIX=172.21.0 - -export CLUSTER_NAME=onebox2 - -export IMAGE_NAME=pegasus:latest - -# allow_non_idempotent_write = true -# for jepsen test this option must be enabled. -export IDEMPOTENT=true - -# Config End # -############## - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -ROOT=$(dirname "${SCRIPT_DIR}") -LOCAL_IP=$("${ROOT}"/scripts/get_local_ip) - -export SCRIPT_DIR -export ROOT -export LOCAL_IP - -export DOCKER_DIR=${ROOT}/${CLUSTER_NAME}-docker # Where docker onebox resides. -export META_COUNT=2 # Number of meta instances. -export REPLICA_COUNT=5 # Number of replica instances. diff --git a/docker/dev/centos7/Dockerfile b/docker/dev/centos7/Dockerfile deleted file mode 100644 index 137322ea5acd7cb4f204a94a5eddd1bf31ca84a6..0000000000000000000000000000000000000000 --- a/docker/dev/centos7/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# This is a docker with all stuff built after compilation. It is a -# large image that's only used for testing. -# -# ```sh -# docker build -t pegasus_onebox . -# docker run -it pegasus_onebox bash -# ``` - - -FROM centos:7 - -MAINTAINER Wu Tao - -RUN yum -y install gcc gcc-c++ automake autoconf libtool make cmake git file wget unzip python-devel which && \ - yum -y install openssl-devel boost-devel libaio-devel snappy-devel bzip2-devel zlib zlib-devel patch - -RUN git clone --recursive https://github.com/XiaoMi/pegasus.git /pegasus - -RUN cd /pegasus && \ - ./run.sh build -c - -RUN yum -y install jre nmap-ncat.x86_64 - -CMD cd /pegasus - diff --git a/docker/dev/jepsen/Dockerfile b/docker/dev/jepsen/Dockerfile deleted file mode 100644 index 85c6051439f1970db5ce24cbd5ff64a3af2e9cde..0000000000000000000000000000000000000000 --- a/docker/dev/jepsen/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM jacobmbr/ubuntu-jepsen:v0.1.0 - -# Usage: -# -# ./run.sh build -c -# ./run.sh pack_server -# mv pegasus-server-{YOUR_VERSION}.tar.gz docker/dev/jepsen/ -# cd docker/dev/jepsen/ -# docker build --build-arg SERVER_PKG_NAME=pegasus-server-{YOUR_VERSION} -t pegasus:latest . - -# Install Jepsen dependencies -RUN sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/' /etc/apt/sources.list; \ - rm /etc/apt/apt.conf.d/docker-clean && apt-get update -y -RUN apt-get install -y openssh-server \ - curl faketime iproute2 iptables iputils-ping libzip4 \ - logrotate man man-db net-tools ntpdate psmisc python rsyslog \ - sudo unzip vim wget apt-transport-https \ - && apt-get remove -y --purge --auto-remove systemd - -ARG SERVER_PKG_NAME - -COPY ./$SERVER_PKG_NAME.tar.gz / -RUN tar xvf /$SERVER_PKG_NAME.tar.gz; \ - mv $SERVER_PKG_NAME pegasus; \ - rm $SERVER_PKG_NAME.tar.gz - -COPY ./entrypoint.sh / -RUN chmod +x /entrypoint.sh - -ENV LD_LIBRARY_PATH=/pegasus/bin - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/dev/jepsen/entrypoint.sh b/docker/dev/jepsen/entrypoint.sh deleted file mode 100644 index 297b1a53e93c3e5bca08670f02db39f606b61987..0000000000000000000000000000000000000000 --- a/docker/dev/jepsen/entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -/usr/sbin/sshd -D & - -/pegasus/bin/pegasus_server /pegasus/bin/config.ini -app_list "$1" diff --git a/docker/dev/linux/Dockerfile b/docker/dev/linux/Dockerfile deleted file mode 100644 index aa6979411ceea5a93a67f63c78efc2f5e83c5fb4..0000000000000000000000000000000000000000 --- a/docker/dev/linux/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM ubuntu:18.04 - -# This is a docker image for any usage once you have a pre-built -# binary of Pegasus. -# -# Usage: -# -# ./run.sh build -c -# ./run.sh pack_server -# mv pegasus-server-{YOUR_VERSION}.tar.gz docker/dev/linux/ -# cd docker/dev/linux/ -# docker build --build-arg SERVER_PKG_NAME=pegasus-server-{YOUR_VERSION} -t pegasus:latest . -# -# Or simply run docker/build_docker.sh to build image named pegasus:latest. -# - -ARG SERVER_PKG_NAME - -# Install libunwind -RUN sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/' /etc/apt/sources.list; \ - rm /etc/apt/apt.conf.d/docker-clean && apt-get update -y; \ - apt-get install -y libunwind-dev libgssapi-krb5-2; \ - rm -rf /var/lib/apt/lists/* - -COPY ./$SERVER_PKG_NAME.tar.gz / -RUN tar xvf /$SERVER_PKG_NAME.tar.gz; \ - mv $SERVER_PKG_NAME pegasus; \ - rm $SERVER_PKG_NAME.tar.gz - -COPY ./entrypoint.sh / -RUN chmod +x /entrypoint.sh - -ENV LD_LIBRARY_PATH=/pegasus/bin - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/dev/linux/entrypoint.sh b/docker/dev/linux/entrypoint.sh deleted file mode 100644 index f301aceec100c7b068bfb9942c259320d17a8a16..0000000000000000000000000000000000000000 --- a/docker/dev/linux/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -/pegasus/bin/pegasus_server /pegasus/bin/config.ini -app_list "$1" diff --git a/docker/dev/ubuntu16.04/Dockerfile b/docker/dev/ubuntu16.04/Dockerfile deleted file mode 100644 index cc332c466a3e11efb4527b0f8b56cc3faf4532bc..0000000000000000000000000000000000000000 --- a/docker/dev/ubuntu16.04/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:16.04 - -MAINTAINER WU TAO - -RUN sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/' /etc/apt/sources.list; \ - apt-get update -y; \ - apt-get -y install build-essential cmake libboost-system-dev libboost-filesystem-dev libboost-regex-dev \ - libaio-dev libsnappy-dev libbz2-dev libtool \ - zlib1g zlib1g.dev patch git curl zip automake libssl-dev; - -RUN git clone --recursive https://github.com/XiaoMi/pegasus.git /pegasus; \ - cd /pegasus; \ - ./run.sh build; \ - ./run.sh pack_tools; \ - cp -r pegasus-tools-* /; cd /; rm -rf /pegasus \ - rm pegasus-tools-*.tar.gz; - -RUN apt-get -y install default-jre nmap netcat diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index d5218aa7106980092329fc27496b09cd6e95e86b..0000000000000000000000000000000000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,111 +0,0 @@ -version: '3.3' - -networks: - frontend: - ipam: - config: - - subnet: @NODE_IP_PREFIX@.0/24 -services: - meta1: - image: @IMAGE_NAME@ - ports: - - "34601" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./meta1/data:/pegasus/data - command: - - meta - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.11 - restart: on-failure - meta2: - image: @IMAGE_NAME@ - ports: - - "34601" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./meta2/data:/pegasus/data - command: - - meta - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.12 - restart: on-failure - replica1: - image: @IMAGE_NAME@ - ports: - - "34801" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./replica1/data:/pegasus/data - - ./replica1/slog:/pegasus/slog - command: - - replica - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.21 - restart: on-failure - replica2: - image: @IMAGE_NAME@ - ports: - - "34801" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./replica2/data:/pegasus/data - - ./replica2/slog:/pegasus/slog - command: - - replica - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.22 - restart: on-failure - replica3: - image: @IMAGE_NAME@ - ports: - - "34801" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./replica3/data:/pegasus/data - - ./replica3/slog:/pegasus/slog - command: - - replica - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.23 - restart: on-failure - replica4: - image: @IMAGE_NAME@ - ports: - - "34801" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./replica4/data:/pegasus/data - - ./replica4/slog:/pegasus/slog - command: - - replica - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.24 - restart: on-failure - replica5: - image: @IMAGE_NAME@ - ports: - - "34801" - volumes: - - ./config.ini:/pegasus/bin/config.ini:ro - - ./replica5/data:/pegasus/data - - ./replica5/slog:/pegasus/slog - command: - - replica - privileged: true - networks: - frontend: - ipv4_address: @NODE_IP_PREFIX@.25 - restart: on-failure diff --git a/docker/prepare.sh b/docker/prepare.sh deleted file mode 100755 index 8cd463a612a0414d1ff31d6b838b3437c1824311..0000000000000000000000000000000000000000 --- a/docker/prepare.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 -source cluster_args.sh - -function print_nodes() -{ - echo "" - for i in $(seq "${META_COUNT}"); do - echo "META${i}: ${NODE_IP_PREFIX}.1$((i)):34601" - done - for i in $(seq "${REPLICA_COUNT}"); do - echo "REPLICA${i}: ${NODE_IP_PREFIX}.2$((i)):34801" - done - - cd "${DOCKER_DIR}" || exit 1 - echo "" - docker-compose ps - echo "" - cd - || exit 1 -} - -if ! [[ -x "$(command -v docker-compose)" ]]; then - echo 'ERROR: docker-compose is not installed.' >&2 - echo 'See this document for installation manual:' >&2 - echo ' https://docs.docker.com/compose/install' >&2 - exit 1 -fi - -if [[ -d "${DOCKER_DIR}" ]]; then - echo "ERROR: ${DOCKER_DIR} already exists, please remove it first" >&2 - print_nodes - exit 1 -fi - -mkdir -p "${DOCKER_DIR}" - -cp -f "${ROOT}"/src/server/config.ini "${DOCKER_DIR}/config.ini" -sed -i 's/@META_PORT@/34601/' "${DOCKER_DIR}/config.ini" -sed -i 's/@REPLICA_PORT@/34801/' "${DOCKER_DIR}/config.ini" -sed -i 's/%{cluster.name}/'"${CLUSTER_NAME}"'/g' "${DOCKER_DIR}/config.ini" -sed -i 's/%{app.dir}/\/pegasus\/data/g' "${DOCKER_DIR}/config.ini" -sed -i 's/%{slog.dir}/\/pegasus\/slog/g' "${DOCKER_DIR}/config.ini" -sed -i 's/%{data.dirs}//g' "${DOCKER_DIR}/config.ini" -sed -i 's@%{home.dir}@'"/pegasus"'@g' "${DOCKER_DIR}/config.ini" -sed -i "s/allow_non_idempotent_write = false/allow_non_idempotent_write = ${IDEMPOTENT}/" "${DOCKER_DIR}/config.ini" -for i in $(seq "${META_COUNT}"); do - meta_port=34601 - meta_ip=${NODE_IP_PREFIX}.1$((i)) - if [ "${i}" -eq 1 ]; then - meta_list="${meta_ip}:$meta_port" - else - meta_list="$meta_list,${meta_ip}:$meta_port" - fi -done -sed -i 's/%{meta.server.list}/'"$meta_list"'/g' "${DOCKER_DIR}/config.ini" -sed -i 's/%{zk.server.list}/'"${LOCAL_IP}"':22181/g' "${DOCKER_DIR}/config.ini" -sed -i 's/app_name = .*$/app_name = '"$APP_NAME"'/' "${DOCKER_DIR}/config.ini" -sed -i 's/partition_count = .*$/partition_count = '"$PARTITION_COUNT"'/' "${DOCKER_DIR}/config.ini" - -cp -f "${SCRIPT_DIR}"/docker-compose.yml "${DOCKER_DIR}" -sed -i 's/@NODE_IP_PREFIX@/'"${NODE_IP_PREFIX}"'/' "${DOCKER_DIR}"/docker-compose.yml -sed -i 's/@IMAGE_NAME@/'"${IMAGE_NAME}"'/' "${DOCKER_DIR}"/docker-compose.yml - -echo "${DOCKER_DIR} is ready" -print_nodes diff --git a/docker/start_onebox.sh b/docker/start_onebox.sh deleted file mode 100755 index 6701bb90cfc67e9afeca4c5e3e65bf3db98f3d76..0000000000000000000000000000000000000000 --- a/docker/start_onebox.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 - -source cluster_args.sh - -cd "${ROOT}" || exit 1 - -./docker/clear_onebox.sh -./docker/prepare.sh -./run.sh start_zk - -cd "${DOCKER_DIR}" || exit 1 -pwd - -docker-compose up -d diff --git a/docs/benchmark/r1.png b/docs/benchmark/r1.png deleted file mode 100644 index 424d4f3ae7440b47f9766dde6885ee18e38b2231..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/r1.png and /dev/null differ diff --git a/docs/benchmark/r2.png b/docs/benchmark/r2.png deleted file mode 100644 index f22e682f49f38e09f30e9640cc3b99af60e39978..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/r2.png and /dev/null differ diff --git a/docs/benchmark/r3.png b/docs/benchmark/r3.png deleted file mode 100644 index 0a28c26af058c248c627506326b895a61aa8e184..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/r3.png and /dev/null differ diff --git a/docs/benchmark/w1.png b/docs/benchmark/w1.png deleted file mode 100644 index 8b7bc970d9789c442165609613f76325e2c745db..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/w1.png and /dev/null differ diff --git a/docs/benchmark/w2.png b/docs/benchmark/w2.png deleted file mode 100644 index e11d8ba3fef7d7a957d0d98a94489e34baca8191..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/w2.png and /dev/null differ diff --git a/docs/benchmark/w3.png b/docs/benchmark/w3.png deleted file mode 100644 index 6c59908c0e4e15ac0d01ecbf5a77e9954f0b0e34..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/w3.png and /dev/null differ diff --git a/docs/benchmark/ycsb_read.png b/docs/benchmark/ycsb_read.png deleted file mode 100644 index aacfd4a314c1e3dbd70ed6b52b4e597aa5c1ed84..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/ycsb_read.png and /dev/null differ diff --git a/docs/benchmark/ycsb_write.png b/docs/benchmark/ycsb_write.png deleted file mode 100644 index b11dd573b5f66691f4f55fbb36c85e8cb832b28f..0000000000000000000000000000000000000000 Binary files a/docs/benchmark/ycsb_write.png and /dev/null differ diff --git a/docs/client-development.md b/docs/client-development.md deleted file mode 100644 index 2c8ebadfe70229ed147fe874393db93132e585e9..0000000000000000000000000000000000000000 --- a/docs/client-development.md +++ /dev/null @@ -1,81 +0,0 @@ -Notices For Client Library Development -========== - -Currently C++ and Java are supported to visit Pegasus cluster. You may want to refer to [C++ documentations](https://github.com/XiaoMi/pegasus/wiki/Cpp%E5%AE%A2%E6%88%B7%E7%AB%AF%E6%96%87%E6%A1%A3) or [Java documentation](https://github.com/XiaoMi/pegasus/wiki/Java%E5%AE%A2%E6%88%B7%E7%AB%AF%E6%96%87%E6%A1%A3) for details. - -Here we give some notices on the client library, which may be helpful to development of other language bindings. - -## message protocol - -### request to server - -Message format sent to some single server of Pegasus Cluster: - -` -(A special header of 48 bytes) + (body) -` - -#### header - -The 48-byte header consists of: - -| bytes | type | comments -|-------| -----|--------- | -| 0~3 | "THFT" | (header_type) -| 4~7 | 32bit int | header version -| 8~11 | 32bit int | header_length -| 12~15 | 32bit int | header_crc32 -| 16~19 | 32bit int | body_length -| 20~23 | 32bit int | body_crc32 -| 24~27 | 32bit int | app_id (rDSN related concept) -| 28~31 | 32bit int | partition_index (rDSN related concept) -| 32~35 | 32bit int | client_timeout -| 36~39 | 32bit int | thread_hash (rDSN related concept) -| 40~47 | 64bit long | partition_hash (rDSN related concept) - -Some notes on the above "header": - * all ints/long in the header are in network order. - * if send request to meta server, the app\_id & partition\_index should set to 0; if request is to replica server, the fields should be set to the target replica. - * pegasus server may use client_timeout to do some optimization. Say, a server may simply discard a request if it has expired when server receives it - * thread\_hash = app\_id * 7919 + partition\_index, it is used for server to decide in which thread to queue the request. - * partition_hash should set to 0, it's only a useful field for RPC client of rDSN framework. - -#### body -the body is a standard thrift struct in binary protocol: - -TMessageBegin + args + TMessageEnd. - -You should write a thrift "TMessage" in TMessageBegin, the structure of TMessage is: -* name: the RPC name (please refer to Java client for detail) -* type: TMessage.CALL -* seqid: the seqid int - -### response from server - -response got from pegasus server: - -` -total_response_length(4 bytes) + error_code_thrift_struct + response_body -` - -Some notes on the above response: - -* **error_code_thrift_struct** is an error_code struct in thrift binary protocol, usually it indicates some error of service status than the response of some specific rpc call. For example, for a meta server, this error may indicate "the meta server is not leader"; for a read/write request to replica server, this error may indicate that the replica server is not a primary or don't serve the partition" -* **response_body** is a standard thrift rpc response of the rpc call, with the format as follows: - * TMessageBegin: rpc_name, TMessage.T_REPLY, seqid_integer - * response_args - * TMessageEnd - -## write/read request process - -You can refer to [TableHandler.java](https://github.com/XiaoMi/pegasus-java-client/blob/thrift-0.11.0-inlined/src/main/java/com/xiaomi/infra/pegasus/rpc/async/TableHandler.java) for the detailed RPC process in write/read request RPCs. - -## how to generate code in thrift - -There are [3 IDL files](https://github.com/XiaoMi/pegasus-java-client/tree/thrift-0.11.0-inlined/idl) for RPC client: -* base.thrift: a placeholder for rDSN specific structures(blob, error\_code, task\_code, RPC_address, gpid), you may use thrift to generate a sketch, and implement the details all by yourself. -* replication.thrift: messages and RPCs used for communicate with meta server. Using generated code is ok. -* rrdb.thrift: messages and RPCs used for communicate with replica server. Using generated code is ok. - -Due to some history reasons, RPC names defined in the IDL can't be recognized by server right now. A proper name should be set manually, please refer to [operators](https://github.com/XiaoMi/pegasus-java-client/tree/thrift-0.11.0-inlined/src/main/java/com/xiaomi/infra/pegasus/operator) for details. Besides, you may also need to refer -to [base](https://github.com/XiaoMi/pegasus-java-client/tree/thrift-0.11.0-inlined/src/main/java/com/xiaomi/infra/pegasus/base) for how to implement serialization for rDSN specific structures. diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 07ea5b23dfbdd2e2efeb82f7f8e6d91d57cd5059..0000000000000000000000000000000000000000 --- a/docs/installation.md +++ /dev/null @@ -1,169 +0,0 @@ -Installing Pegasus -=========== - -Currently Pegasus can only be installed from source. Binary package and docker image is coming soon. - -## Prerequisites and Requirements - -### Hardware - -Pegasus can be deployed in standalone mode which runs all the pegasus jobs as different processes on a single host, or in distributed mode which runs different jobs on different hosts. For distributed mode, more than one hosts are need: - -* One or more hosts to run zookeeper. We need zookeeper to make the cluster metadata persistence. For a fault tolerant zookeeper cluster, at least 3 are necessary. -* One or more hosts to run Pegasus meta-server. At least 2 are needed for fault tolerance. You may share hosts between meta-server and zookeeper. -* At least 3 hosts to run Pegasus replica-server. - -### Operating System requirements - -* Linux: CentOS 7, Ubuntu 14.04(Trusty) -* macOS: not supported -* Microsoft Windows: not supported - -## Build Pegasus from source - -Please notice that Pegasus can not be built until the following packages meet the version requirements: - -* Compiler: the gcc version must >= 4.8 && < 8. If you use other compilers, please make sure that C++14 is supported. -* CMake: must >= 3.5.2 -* Boost: must >= 1.58 - -#### Install the dependencies - -For Ubuntu: - -``` -sudo apt-get install build-essential cmake libboost-all-dev libaio-dev libsnappy-dev libbz2-dev libtool libgflags-dev zlib1g zlib1g.dev patch unzip automake libssl-dev liblz4-dev libzstd-dev -``` -To start onebox: -``` -sudo apt-get install default-jre netcat -``` -For CentOS: -``` -yum -y install cmake boost-devel libaio-devel snappy-devel bzip2-devel zlib-devel lz4-devel automake libtool patch unzip openssl-devel python-devel libzstd-devel -``` -Zstd may not available in the default EPEL. To install the EPEL repository on CentOS-7/6, simply type: -``` -yum install epel-release -``` -To start onebox: -``` -yum install java-1.8.0-openjdk nmap-ncat.x86_64 -``` - -Please make sure you install the proper version of GCC, CMake and Boost. - -#### Build - -1. clone Pegasus and its all subprojects: - - ``` - git clone https://github.com/xiaomi/pegasus.git --recursive - ``` -2. build - - ``` - cd pegasus && ./run.sh build - ``` - if Boost is installed at some custom path , please tell the build script the install path of Boost: - - ``` - ./run.sh build -b /your/boost/installation/path - ``` - -Generally, the build process of Pegasus consists of 4 parts: - -1. Download the thirdparty dependencies to "rdsn/thirdparty" and build them, and the output headers/libraries/binaries are installed in "rdsn/thirdparty/output". -2. build the rdsn project, which is a dependency of our KV storage. The output headers/libraries/binaries are installed in "rdsn/builder/output". The build script will create a symbolic link "DSN_ROOT" in the pegasus project root. -3. build rocksdb in "rocksdb" dir, which is modified from [facebook/rocksdb](https://github.com/facebook/rocksdb) -4. build pegasus's KV-layer in "src". - -**Please make sure the thirdparty are successfully downloaded and built before subsequent parts**. - -#### Common Build Problems -* fatal error: .h: No such file or directory - -This means that Third Party Building did not complete correctly and did not generate the header files. - -Rerun `./run.sh build` with `--clear_thirdparty`. - -## Run in standalone mode - -You can play with pegasus with a **onebox** cluster: - -``` -./run.sh start_onebox -``` - -When running onebox cluster, all pegasus jobs are running in local host: - -* zookeeper: start a zookeeper process with port 22181 -* meta-server: start several meta-server processes -* replica-server: start several replica-server processes - -You can also check the state of onebox cluster: - -``` -./run.sh list_onebox -``` - -Stop the onebox cluster: - -``` -./run.sh stop_onebox -``` - -Clean the onebox cluster: - -``` -./run.sh clear_onebox -``` - -## Interactive shell - -Pegasus provides a shell tool to interact with the cluster. To start the shell, run: -``` -./run.sh shell -``` - -or specify different cluster meta-server address list: -``` -./run.sh shell --cluster 127.0.0.1:34601,127.0.0.1:34602 -``` - -Using the shell, you can do lots of things including: - -* get cluster information and statistics -* create/drop/list tables (we name it app) -* get/set/del/scan data -* migrate replicas from node to node -* send remote commands to server -* dump mutation log and sstable files - -## Running test - -Run the unit test by: -``` -./run.sh test -``` - -or run the killing test by: -``` -./run.sh start_kill_test -``` - -## Benchmark - -You can start a benchmark client quickly (needs onebox cluster already started): -``` -./run.sh bench -``` - -or start a series of benchmark tests: -``` -./scripts/pegasus_bench_run.sh -``` - -Attention: this bench tool depends on gflags, so it would not work if gflags library -is not installed when building Pegasus. - diff --git a/docs/media-img/ddd-diagnose.png b/docs/media-img/ddd-diagnose.png deleted file mode 100644 index 381b13cf4eb9479a484538448845d4d781074c5a..0000000000000000000000000000000000000000 Binary files a/docs/media-img/ddd-diagnose.png and /dev/null differ diff --git a/docs/media-img/network-partition.png b/docs/media-img/network-partition.png deleted file mode 100644 index 956faf63979cbe38e100411753434f3babf48bb7..0000000000000000000000000000000000000000 Binary files a/docs/media-img/network-partition.png and /dev/null differ diff --git a/docs/media-img/pegasus-data-model-sample.png b/docs/media-img/pegasus-data-model-sample.png deleted file mode 100644 index 0aebf59275189848f91bb74eddbe971ad24baae3..0000000000000000000000000000000000000000 Binary files a/docs/media-img/pegasus-data-model-sample.png and /dev/null differ diff --git a/docs/media-img/pegasus-logo-icon.png b/docs/media-img/pegasus-logo-icon.png deleted file mode 100644 index e9e4bcf73b522e562cb136ecf15c1cedcb17a884..0000000000000000000000000000000000000000 Binary files a/docs/media-img/pegasus-logo-icon.png and /dev/null differ diff --git a/docs/media-img/pegasus-logo-transparent.png b/docs/media-img/pegasus-logo-transparent.png deleted file mode 100644 index ae28af61c8bcf08dece057af01acf96fc2f2e4b3..0000000000000000000000000000000000000000 Binary files a/docs/media-img/pegasus-logo-transparent.png and /dev/null differ diff --git a/docs/media-img/pegasus-logo.png b/docs/media-img/pegasus-logo.png deleted file mode 100644 index 42b7872ec3dc151b69254929327bb34a9af686a0..0000000000000000000000000000000000000000 Binary files a/docs/media-img/pegasus-logo.png and /dev/null differ diff --git a/docs/media-img/perfect-failure-detector.png b/docs/media-img/perfect-failure-detector.png deleted file mode 100644 index 7d440dd40c6f5fb3f8f6e6f8f60b010b6bdef953..0000000000000000000000000000000000000000 Binary files a/docs/media-img/perfect-failure-detector.png and /dev/null differ diff --git a/docs/media-img/s2_cap_1.png b/docs/media-img/s2_cap_1.png deleted file mode 100644 index 5c94942f3a2b87cb11edfe82886ab7fa4bb65726..0000000000000000000000000000000000000000 Binary files a/docs/media-img/s2_cap_1.png and /dev/null differ diff --git a/docs/media-img/s2_cap_2.png b/docs/media-img/s2_cap_2.png deleted file mode 100644 index 28906d5d1be6eadee52634ac023effb4bd6b7696..0000000000000000000000000000000000000000 Binary files a/docs/media-img/s2_cap_2.png and /dev/null differ diff --git a/docs/ppt/ArchSummit_Beijing_2016.pdf b/docs/ppt/ArchSummit_Beijing_2016.pdf deleted file mode 100644 index 401dcec2d183cf84afd872be4e402c9376d3f53f..0000000000000000000000000000000000000000 Binary files a/docs/ppt/ArchSummit_Beijing_2016.pdf and /dev/null differ diff --git a/docs/ppt/ArchSummit_Beijing_2016.pptx b/docs/ppt/ArchSummit_Beijing_2016.pptx deleted file mode 100644 index 5e8d927b42fe199ee28f730c68b193c1ab9000ab..0000000000000000000000000000000000000000 Binary files a/docs/ppt/ArchSummit_Beijing_2016.pptx and /dev/null differ diff --git a/docs/ppt/ArchSummit_Shenzhen_2017.pdf b/docs/ppt/ArchSummit_Shenzhen_2017.pdf deleted file mode 100644 index 1fe35c4a72c8ae1430da6770c1ff6aec50c6e478..0000000000000000000000000000000000000000 Binary files a/docs/ppt/ArchSummit_Shenzhen_2017.pdf and /dev/null differ diff --git a/docs/ppt/ArchSummit_Shenzhen_2017.pptx b/docs/ppt/ArchSummit_Shenzhen_2017.pptx deleted file mode 100644 index 0dab2c90125b69074109fff5188d206f1b328916..0000000000000000000000000000000000000000 Binary files a/docs/ppt/ArchSummit_Shenzhen_2017.pptx and /dev/null differ diff --git a/docs/ppt/Pegasus_Intro_2018_07.pptx b/docs/ppt/Pegasus_Intro_2018_07.pptx deleted file mode 100644 index fe5eaad2093ab18f346d025d43dbc7cb9ff6eb15..0000000000000000000000000000000000000000 Binary files a/docs/ppt/Pegasus_Intro_2018_07.pptx and /dev/null differ diff --git a/docs/ppt/Pegasus_Intro_2018_10.pptx b/docs/ppt/Pegasus_Intro_2018_10.pptx deleted file mode 100644 index c8a9549af8d0634062d7fed1c4ea8e668db62e21..0000000000000000000000000000000000000000 Binary files a/docs/ppt/Pegasus_Intro_2018_10.pptx and /dev/null differ