add cli test frame work bats (#181)

* import bats for cli test

  https://github.com/sstephenson/bats

* add simple test case for cli use bats

* format main.bats add env

* replace main_test.sh with main.bats

  make explain suggestion sorted
  make index suggestion sorted

* add test database world_x

* update bats golden result

* make docker add 180s timeout

  1. add timeout for docker start
  2. add not explain able sql test case into explain_test.go

* explain info Using index VS Using index condition

  remove nonappearence key from explain extra info

* add timeout for test

.gitignore

 bin/
 release/
+test/tmp/
 common/version.go
 doc/blueprint/
 *.iml

.travis.yml

@@ -20,11 +20,15 @@ services:
 
 before_install:
   - docker pull mysql
+  - sudo add-apt-repository ppa:duggan/bats --yes
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq bats
 
 script:
   - make build
   - make docker
   - make cover
+  - make test-cli
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -72,15 +72,24 @@ fmt: go_version_check
 .PHONY: test
 test:
 	@echo "$(CGREEN)Run all test cases ...$(CEND)"
-	go test -race ./...
+	go test -timeout 10m -race ./...
 	@echo "test Success!"
 
 # Rule golang test cases with `-update` flag
+.PHONY: test-update
 test-update:
 	@echo "$(CGREEN)Run all test cases with -update flag ...$(CEND)"
 	go test ./... -update
 	@echo "test-update Success!"
 
+# Using bats test framework run all cli test cases
+# https://github.com/sstephenson/bats
+.PHONY: test-cli
+test-cli: build
+	@echo "$(CGREEN)Run all cli test cases ...$(CEND)"
+	bats ./test
+	@echo "test-cli Success!"
+
 # Code Coverage
 # colorful coverage numerical >=90% GREEN, <80% RED, Other YELLOW
 .PHONY: cover
@@ -180,46 +189,44 @@ release: build
 docker:
 	@echo "$(CGREEN)Build mysql test enviorment ...$(CEND)"
 	@docker stop soar-mysql 2>/dev/null || true
-	@docker wait soar-mysql 2>/dev/null || true
+	@docker wait soar-mysql 2>/dev/null >/dev/null || true
 	@echo "docker run --name soar-mysql $(MYSQL_RELEASE):$(MYSQL_VERSION)"
 	@docker run --name soar-mysql --rm -d \
 	-e MYSQL_ROOT_PASSWORD=1tIsB1g3rt \
 	-e MYSQL_DATABASE=sakila \
 	-p 3306:3306 \
-	-v `pwd`/doc/example/sakila.sql.gz:/docker-entrypoint-initdb.d/sakila.sql.gz \
+	-v `pwd`/test/sql/init.sql.gz:/docker-entrypoint-initdb.d/init.sql.gz \
 	$(MYSQL_RELEASE):$(MYSQL_VERSION)
 
 	@echo "waiting for sakila database initializing "
-	@while ! docker exec soar-mysql mysql --user=root --password=1tIsB1g3rt --host "127.0.0.1" --silent -NBe "do 1" >/dev/null 2>&1 ; do \
-	printf '.' ; \
-	sleep 1 ; \
-	done ; \
-	echo '.'
-	@echo "mysql test enviorment is ready!"
+	@timeout=180; while [ $${timeout} -gt 0 ] ; do \
+		if ! docker exec soar-mysql mysql --user=root --password=1tIsB1g3rt --host "127.0.0.1" --silent -NBe "do 1" >/dev/null 2>&1 ; then \
+			timeout=`expr $$timeout - 1`; \
+			printf '.' ;  sleep 1 ; \
+		else \
+			echo "." ; echo "mysql test enviorment is ready!" ; break ; \
+		fi ; \
+		if [ $$timeout = 0 ] ; then \
+			echo "." ; echo "$(CRED)docker soar-mysql start timeout(180 s)!$(CEND)" ; exit 1 ; \
+		fi ; \
+	done
 
 .PHONY: docker-connect
 docker-connect:
-	docker exec -it soar-mysql mysql --user=root --password=1tIsB1g3rt --host "127.0.0.1"
+	@docker exec -it soar-mysql mysql --user=root --password=1tIsB1g3rt --host "127.0.0.1" sakila
 
 # attach docker container with bash interactive mode
 .PHONY: docker-it
 docker-it:
 	docker exec -it soar-mysql /bin/bash
 
-.PHONY: main_test
-main_test: install
-	@echo "$(CGREEN)running main_test ...$(CEND)"
-	@echo "soar -list-test-sqls | soar"
-	@./doc/example/main_test.sh
-	@echo "main_test Success!"
-
 .PHONY: daily
-daily: | deps fmt vendor docker cover doc lint release install main_test clean logo
+daily: | deps fmt vendor docker cover doc lint release install test-cli clean logo
 	@echo "$(CGREEN)daily build finished ...$(CEND)"
 
 # vendor, docker will cost long time, if all those are ready, daily-quick will much more fast.
 .PHONY: daily-quick
-daily-quick: | deps fmt cover main_test doc lint logo
+daily-quick: | deps fmt cover test-cli doc lint logo
 	@echo "$(CGREEN)daily-quick build finished ...$(CEND)"
 
 .PHONY: logo
@@ -238,7 +245,7 @@ clean:
 			rm -f ${BINARY}.$${GOOS}-$${GOARCH} ;\
 		done ;\
 	done
-	rm -f ${BINARY} coverage.*
+	rm -f ${BINARY} coverage.* test/tmp/*
 	find . -name "*.log" -delete
 	git clean -fi
 	docker stop soar-mysql 2>/dev/null || true

advisor/index.go

@@ -18,6 +18,7 @@ package advisor
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/XiaoMi/soar/ast"
@@ -989,7 +990,13 @@ func (idxAdvs IndexAdvises) Format() map[string]Rule {
 		rules[advKey].Content = strings.Trim(rules[advKey].Content, common.Config.Delimiter)
 	}
 
+	var sortAdvs []string
 	for adv := range rules {
+		sortAdvs = append(sortAdvs, adv)
+	}
+	sort.Strings(sortAdvs)
+
+	for _, adv := range sortAdvs {
 		key := fmt.Sprintf("IDX.%03d", number)
 		ddl := ast.MergeAlterTables(sqls[adv]...)
 		// 由于传入合并的SQL都是一张表的，所以一定只会输出一条ddl语句

common/tricks.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -67,24 +66,22 @@ func captureOutput(f func()) string {
 	r, w, _ := os.Pipe()
 	os.Stdout = w
 
-	// execute function
-	f()
-
-	outC := make(chan string)
 	// copy the output in a separate goroutine so printing can't block indefinitely
+	outC := make(chan string)
 	go func() {
-		var buf bytes.Buffer
-		_, err := io.Copy(&buf, r)
+		buf, err := ioutil.ReadAll(r)
 		if err != nil {
-			Log.Warning(err.Error())
+			panic(err)
 		}
-		outC <- buf.String()
+		outC <- string(buf)
 	}()
 
+	// execute function
+	f()
+
 	// back to normal state
-	err := w.Close()
-	if err != nil {
-		Log.Warning(err.Error())
+	if err := w.Close(); err != nil {
+		panic(err)
 	}
 	os.Stdout = oldStdout // restoring the real stdout
 	out := <-outC

common/tricks_test.go

+/*
+ * Copyright 2018 Xiaomi, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package common
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestCaptureOutput(t *testing.T) {
+	c1 := make(chan string, 1)
+	// test output buf large than 65535
+	length := 1<<16 + 1
+	go func() {
+		str := captureOutput(
+			func() {
+				var str []string
+				for i := 0; i < length; i++ {
+					str = append(str, "a")
+				}
+				fmt.Println(strings.Join(str, ""))
+			},
+		)
+		c1 <- str
+	}()
+
+	select {
+	case res := <-c1:
+		if len(res) <= length {
+			t.Errorf("want %d, got %d", length, len(res))
+		}
+	case <-time.After(1 * time.Second):
+		t.Error("capture timeout, pipe read hangup")
+	}
+}

database/explain.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"regexp"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -652,6 +653,7 @@ func ExplainInfoTranslator(exp *ExplainInfo) string {
 	}
 	if len(selectTypeBuf) > 0 {
 		buf = append(buf, fmt.Sprint("#### SelectType信息解读\n"))
+		sort.Strings(selectTypeBuf)
 		buf = append(buf, strings.Join(selectTypeBuf, "\n"))
 	}
 
@@ -681,6 +683,7 @@ func ExplainInfoTranslator(exp *ExplainInfo) string {
 	}
 	if len(accessTypeBuf) > 0 {
 		buf = append(buf, fmt.Sprint("#### Type信息解读\n"))
+		sort.Strings(accessTypeBuf)
 		buf = append(buf, strings.Join(accessTypeBuf, "\n"))
 	}
 
@@ -693,10 +696,11 @@ func ExplainInfoTranslator(exp *ExplainInfo) string {
 		for _, row := range rows {
 			for k, c := range explainExtra {
 				if strings.Contains(row.Extra, k) {
-					if k == "Impossible WHERE" {
-						if strings.Contains(row.Extra, "Impossible WHERE noticed after reading const tables") {
-							continue
-						}
+					if k == "Impossible WHERE" && strings.Contains(row.Extra, "Impossible WHERE noticed after reading const tables") {
+						continue
+					}
+					if k == "Using index" && strings.Contains(row.Extra, "Using index condition") {
+						continue
 					}
 					warn := false
 					for _, w := range common.Config.ExplainWarnExtra {
@@ -716,6 +720,7 @@ func ExplainInfoTranslator(exp *ExplainInfo) string {
 	}
 	if len(extraTypeBuf) > 0 {
 		buf = append(buf, fmt.Sprint("#### Extra信息解读\n"))
+		sort.Strings(extraTypeBuf)
 		buf = append(buf, strings.Join(extraTypeBuf, "\n"))
 	}
 

database/explain_test.go

@@ -26,6 +26,7 @@ import (
 )
 
 var sqls = []string{
+	`use sakila`, // not explain able sql, will convert to empty!
 	`select * from city where country_id = 44;`,
 	`select * from address where address2 is not null;`,
 	`select * from address where address2 is null;`,

deps.sh

 #!/bin/bash
 
-NEEDED_COMMANDS="docker git go govendor retool"
+NEEDED_COMMANDS="docker git go govendor retool bats"
 
 for cmd in ${NEEDED_COMMANDS} ; do
     if ! command -v "${cmd}" &> /dev/null ; then
@@ -25,3 +25,7 @@ done
 
 # retool
 ## go get github.com/twitchtv/retool
+
+# bats https://github.com/sstephenson/bats
+## Ubuntu: apt-get install bats
+## Mac: brew install bats

doc/example/main_test.sh

-#!/bin/bash
-
-GOPATH=$(go env GOPATH)
-PROJECT_PATH=${GOPATH}/src/github.com/XiaoMi/soar/
-
-if [ "$1x" == "-updatex" ]; then
-  cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.md
-  if [ ! $? -eq 0 ]; then
-    exit 1
-  fi
-else
-  cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.log
-  if [ ! $? -eq 0 ]; then
-    exit 1
-  fi
-  # optimizer_XXX 库名，散粒度，以及索引先后顺序每次可能会不一致
-  DIFF_LINES=$(cat ./doc/example/main_test.log ./doc/example/main_test.md | grep -v "optimizer\|散粒度" | sort | uniq -u | wc -l)
-  if [ "${DIFF_LINES}" -gt 0 ]; then
-    git diff ./doc/example/main_test.log ./doc/example/main_test.md
-  fi
-fi

test/fixture/test_Simple_Query_Optimizer.golden

+# Query: 5767EE37339B2402
+
+★ ★ ★ ★ ☆ 85分
+
+```sql
+
+SELECT  
+  * 
+FROM  
+  film  
+WHERE  
+  LENGTH  > 120
+```
+
+##  Explain信息
+
+| id | select\_type | table | partitions | type | possible_keys | key | key\_len | ref | rows | filtered | scalability | Extra |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| 1  | SIMPLE | *film* | NULL | ALL | NULL | NULL | NULL | NULL | 1000 | 33.33% | ☠️ **O(n)** | Using where |
+
+
+
+### Explain信息解读
+
+#### SelectType信息解读
+
+* **SIMPLE**: 简单SELECT(不使用UNION或子查询等).
+
+#### Type信息解读
+
+* ☠️ **ALL**: 最坏的情况, 从头到尾全表扫描.
+
+#### Extra信息解读
+
+* **Using where**: WHERE条件用于筛选出与下一个表匹配的数据然后返回给客户端. 除非故意做的全表扫描, 否则连接类型是ALL或者是index, 且在Extra列的值中没有Using Where, 则该查询可能是有问题的.
+
+
+##  为sakila库的film表添加索引
+
+* **Item:**  IDX.001
+
+* **Severity:**  L2
+
+
+* **Case:** ALTER TABLE \`sakila\`.\`film\` add index \`idx\_length\` (\`length\`) ;
+
+
+
+##  不建议使用 SELECT * 类型查询
+
+* **Item:**  COL.001
+
+* **Severity:**  L1
+
+* **Content:**  当表结构变更时，使用 \* 通配符选择所有列将导致查询的含义和行为会发生更改，可能导致查询返回更多的数据。
+

test/main.bats

+#!/usr/bin/env bats
+
+load test_helper
+
+@test "Simple Query Optimizer" {
+  ${SOAR_BIN_ENV} -query "select * from film where length > 120" | grep -v "散粒度" > ${BATS_TMP_DIRNAME}/${BATS_TEST_NAME}.golden
+  run golden_diff ${BATS_TEST_NAME}
+  [ $status -eq 0 ]
+}
+
+@test "Syntax Check" {
+  run ${SOAR_BIN} -query "select * frm film" -only-syntax-check
+  [ $status -eq 1 ]
+}
+
+@test "Run all test cases" {
+  ${SOAR_BIN} -list-test-sqls | ${SOAR_BIN_ENV} | grep -v "散粒度" > ${BATS_TMP_DIRNAME}/${BATS_TEST_NAME}.golden
+  run golden_diff ${BATS_TEST_NAME}
+  [ $status -eq 0 ]
+}

test/sql/README.md

+# Test Database
+
+## sakila
+
+* Download MySQL sakila database from http://downloads.mysql.com/docs/sakila-db.tar.gz .
+* InnoDB added FULLTEXT support in 5.6.10, you should add version comment `/*!50610 xxx */` for `film_text` table and it's triggers.
+* Merge schema and data into one file `sakila.sql`
+
+## world\_x
+
+world\_x contain JSON datatype, SOAR use this database for JSON testing.
+
+* Download MySQL world\_x database from http://downloads.mysql.com/docs/world_x-db.tar.gz .
+* MySQL support JSON datatype since 5.7.8, you should add version comment `/*!50708 xxx */` for `city`, `countryinfo`.
+* Merge `sakila.sql`, `world_x.sql` into init.sql.
+
+```bash
+gzip init.sql
+```

test/test_helper.bash

+setup() {
+  export SOAR_DEV_DIRNAME="${BATS_TEST_DIRNAME}/../"
+  export SOAR_BIN="${SOAR_DEV_DIRNAME}/bin/soar" 
+  export SOAR_BIN_ENV="${SOAR_DEV_DIRNAME}/bin/soar -config ${SOAR_DEV_DIRNAME}/etc/soar.yaml" 
+  export BATS_TMP_DIRNAME="${BATS_TEST_DIRNAME}/tmp"
+  export BATS_FIXTURE_DIRNAME="${BATS_TEST_DIRNAME}/fixture"
+  mkdir -p "${BATS_TMP_DIRNAME}"
+}
+
+golden_diff() {
+  FUNC_NAME=$1
+  diff "${BATS_TMP_DIRNAME}/${FUNC_NAME}.golden" "${BATS_FIXTURE_DIRNAME}/${FUNC_NAME}.golden" >/dev/null
+  return $?
+}