generate article content

data/words/v1/article.txt → data/words/article.txt

-“秋天的天气”+（形容词-天气-秋季-好）、（形容词-天气-秋季-好），（形容词-外貌-女性）+（名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），“因为”+（名词-称呼=F）+（动词-及物+名词-意愿）+“带她去”+（名词-地点-名山）+（动词-及物+名词-动作-眼部-中性）+（形容词-颜色-植物-树木）+（名词-植物-树木）+（连词-并列关系）+（形容词-颜色-植物-花卉）+（名词-植物-花卉）。（S：形容词-外貌-女性）+（S：名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），（副词-形容词做副词-情绪-快乐）+（动词-及物+名词-动作-手臂-亲切）+（形容词-性格-积极）+（S:名词-称呼=F）。
+“秋天的天气”+（形容词-天气-秋季-好）、（形容词-天气-秋季-好），（形容词-外貌-女性）+（名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），“因为”+（名词-称呼=F）+（动词-及物_名词-意愿）+“带她去”+（名词-地点-名山）+（动词-及物_名词-动作-眼部-中性）+（形容词-颜色-植物-树木）+（名词-植物-树木）+（连词-并列关系）+（形容词-颜色-植物-花卉）+（名词-植物-花卉）。（S：形容词-外貌-女性）+（S：名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），（副词-形容词做副词-情绪-快乐）+（动词-及物_名词-动作-手臂-亲切）+（形容词-性格-积极）+（S:名词-称呼=F）。
\ No newline at end of file

demo/article.yaml

-author: ZenData
-desc: Generated from article text automatically
-from: words.v1
-title: automation
-type: article
-version: ""
-
-fields:
-    - field: "1"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: 秋天的天气
-      rand: true
-      select: xingrongci-tianqi-qiuji-hao
-      useLastSameValue: false
-      where: "true"
-    - field: "3"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: 、
-      rand: true
-      select: xingrongci-tianqi-qiuji-hao
-      useLastSameValue: false
-      where: "true"
-    - field: "5"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ，
-      rand: true
-      select: xingrongci-waimao-nvxing
-      useLastSameValue: false
-      where: "true"
-    - field: "6"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: mingci-xing+mingci-mingzi
-      useLastSameValue: false
-      where: mingci-xing+mingci-mingzi = f
-    - field: "7"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: fuci-chengdufuci
-      useLastSameValue: false
-      where: "true"
-    - field: "8"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: xingrongcizuoweiyu-qingxu-kuaile
-      useLastSameValue: false
-      where: "true"
-    - field: "11"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ，因为
-      rand: true
-      select: mingci-chenghu
-      useLastSameValue: false
-      where: mingci-chenghu = f
-    - field: "12"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: dongci-jiwu+mingci-yiyuan
-      useLastSameValue: false
-      where: "true"
-    - field: "14"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: 带她去
-      rand: true
-      select: mingci-dedian-mingshan
-      useLastSameValue: false
-      where: "true"
-    - field: "15"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: dongci-jiwu+mingci-dongzuo-yanbu-zhongxing
-      useLastSameValue: false
-      where: "true"
-    - field: "16"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: xingrongci-yanse-zhiwu-shumu
-      useLastSameValue: false
-      where: "true"
-    - field: "17"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: mingci-zhiwu-shumu
-      useLastSameValue: false
-      where: "true"
-    - field: "18"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: lianci-binglieguanxi
-      useLastSameValue: false
-      where: "true"
-    - field: "19"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: xingrongci-yanse-zhiwu-huahui
-      useLastSameValue: false
-      where: "true"
-    - field: "20"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: mingci-zhiwu-huahui
-      useLastSameValue: false
-      where: "true"
-    - field: "22"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: 。
-      rand: true
-      select: xingrongci-waimao-nvxing
-      useLastSameValue: true
-      where: "true"
-    - field: "23"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: mingci-xing+mingci-mingzi
-      useLastSameValue: true
-      where: mingci-xing+mingci-mingzi = f
-    - field: "24"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: fuci-chengdufuci
-      useLastSameValue: false
-      where: "true"
-    - field: "25"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: xingrongcizuoweiyu-qingxu-kuaile
-      useLastSameValue: false
-      where: "true"
-    - field: "27"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ，
-      rand: true
-      select: fuci-xingrongcizuofuci-qingxu-kuaile
-      useLastSameValue: false
-      where: "true"
-    - field: "28"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: dongci-jiwu+mingci-dongzuo-shoubi-qinqie
-      useLastSameValue: false
-      where: "true"
-    - field: "29"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: xingrongci-xingge-jiji
-      useLastSameValue: false
-      where: "true"
-    - field: "30"
-      from: data.words.v1
-      limit: 1
-      postfix: ""
-      prefix: ""
-      rand: true
-      select: mingci-chenghu
-      useLastSameValue: true
-      where: mingci-chenghu = f

src/gen/definition.go

@@ -59,11 +59,15 @@ func LoadConfigDef(defaultFile, configFile string, fieldsToExport *[]string) mod
 	mergerDefine(&defaultDef, &configDef, fieldsToExport)
 	orderFields(&defaultDef, *fieldsToExport)
 
-	for _, field := range defaultDef.Fields {
+	for index, field := range defaultDef.Fields {
 		if vari.Trim {
 			field.Prefix = ""
 			field.Postfix = ""
 		}
+
+		if defaultDef.Type == constant.ConfigTypeArticle {
+			defaultDef.Fields[index].Select = strings.Replace(field.Select, "-", "_", -1)
+		}
 	}
 
 	return defaultDef
@@ -91,6 +95,9 @@ func mergerDefine(defaultDef, configDef *model.DefData, fieldsToExport *[]string
 	if configDef.From != "" && defaultDef.From == "" {
 		defaultDef.From = configDef.From
 	}
+	if configDef.Type != "" && defaultDef.Type == "" {
+		defaultDef.Type = configDef.Type
+	}
 
 	for i, field := range defaultDef.Fields {
 		if !isSetFieldsToExport {
@@ -102,7 +109,7 @@ func mergerDefine(defaultDef, configDef *model.DefData, fieldsToExport *[]string
 	for i, field := range configDef.Fields {
 		vari.TopFiledMap[field.Field] = field
 		if !isSetFieldsToExport {
-			if !stringUtils.FindInArr(field.Field, *fieldsToExport) {
+			if !stringUtils.StrInArr(field.Field, *fieldsToExport) {
 				*fieldsToExport = append(*fieldsToExport, field.Field)
 			}
 		}

src/gen/excel.go

@@ -9,9 +9,11 @@ import (
 	fileUtils "github.com/easysoft/zendata/src/utils/file"
 	i118Utils "github.com/easysoft/zendata/src/utils/i118"
 	logUtils "github.com/easysoft/zendata/src/utils/log"
+	stringUtils "github.com/easysoft/zendata/src/utils/string"
 	"github.com/easysoft/zendata/src/utils/vari"
 	"log"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -139,7 +141,7 @@ func ConvertSingleExcelToSQLiteIfNeeded(dbName string, path string) (firstSheet
 		ddl := fmt.Sprintf(ddlTemplate, tableName, colDefine)
 		insertSql := fmt.Sprintf(insertTemplate, tableName, colList, valList)
 
-		db, err := sql.Open("sqlite3", constant.SqliteData)
+		db, err := sql.Open(constant.SqliteDriver, constant.SqliteData)
 		defer db.Close()
 		_, err = db.Exec(dropSql)
 		if err != nil {
@@ -163,8 +165,55 @@ func ConvertSingleExcelToSQLiteIfNeeded(dbName string, path string) (firstSheet
 	return
 }
 
-func ConvertExcelsToSQLiteIfNeeded(dbName string, dir string) {
+func ConvertExcelsToSQLiteIfNeeded(tableName string, dir string) {
+	if !isExcelChanged(dir) {
+		return
+	}
+
+	files := make([]string, 0)
+	fileUtils.GetFilesByExtInDir(dir, ".xlsx", &files)
+
+	seq := 1
+	ddlFields := make([]string, 0)
+	insertSqls := make([]string, 0)
+
+	colMap := map[string]bool{}
+	for _, file := range files {
+		importExcel(file, tableName, &seq, &ddlFields, &insertSqls, &colMap)
+	}
+
+	db, err := sql.Open(constant.SqliteDriver, constant.SqliteData)
+	defer db.Close()
+
+	if err != nil {
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_connect_sqlite", constant.SqliteData, err.Error()))
+		return
+	}
+
+	dropSql := `DROP TABLE IF EXISTS ` + tableName + `;`
+	_, err = db.Exec(dropSql)
+	if err != nil {
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error()))
+		return
+	}
+
+	ddlTemplate := "CREATE TABLE " + tableName + "(\n" +
+		"\t`seq` CHAR (5) PRIMARY KEY ASC UNIQUE,\n" +
+		"%s" +
+		"\n);"
+	ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n"))
+	_, err = db.Exec(ddlSql)
+	if err != nil {
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error()))
+		return
+	}
 
+	sql := strings.Join(insertSqls, "\n")
+	_, err = db.Exec(sql)
+	if err != nil {
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error()))
+		return
+	}
 
 	return
 }
@@ -179,13 +228,19 @@ func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) (
 		return list, ""
 	}
 
-	selectCol := field.Select
-	from := dbName + "_" + tableName
+	selectCol := strings.Replace(field.Select, "-", "_", -1)
+	from := dbName
+	if tableName != "" {
+		from += "_" + tableName
+	}
+
 	where := field.Where
-	if where == "" {
-		where = "true"
+	if where == "" || (vari.Def.Type == constant.ConfigTypeArticle && strings.ToLower(where) == "true") {
+		where = "1=1"
 	}
 
+	where = strings.Replace(where, "-", "_", -1)
+
 	if field.Rand {
 		where += " ORDER BY RANDOM() "
 	}
@@ -198,7 +253,12 @@ func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) (
 		where = where + fmt.Sprintf(" LIMIT %d", total)
 	}
 
-	sqlStr := fmt.Sprintf("SELECT %s FROM %s WHERE %s", selectCol, from, where)
+	colStr := selectCol
+	if vari.Def.Type == constant.ConfigTypeArticle {
+		colStr = "ci AS " + selectCol
+	}
+
+	sqlStr := fmt.Sprintf("SELECT %s FROM %s WHERE %s", colStr, from, where)
 	rows, err := db.Query(sqlStr)
 	if err != nil {
 		logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sqlStr, err.Error()))
@@ -250,6 +310,17 @@ func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) (
 }
 
 func isExcelChanged(path string) bool {
+	if !fileUtils.FileExist(path) {
+		return false
+	}
+
+	fileChangeTime := time.Time{}.Unix()
+	if !fileUtils.IsDir(path) {
+		fileChangeTime = getFileModTime(path).Unix()
+	} else {
+		fileChangeTime = getDirModTime(path).Unix()
+	}
+
 	db, err := sql.Open(constant.SqliteDriver, constant.SqliteData)
 	defer db.Close()
 	if err != nil {
@@ -265,8 +336,6 @@ func isExcelChanged(path string) bool {
 		return true
 	}
 
-	fileChangeTime := getFileModTime(path).Unix()
-
 	found := false
 	changed := false
 	for rows.Next() {
@@ -309,6 +378,20 @@ func isExcelChanged(path string) bool {
 	return changed
 }
 
+func getDirModTime(path string) (dirChangeTime time.Time) {
+	files := make([]string, 0)
+	fileUtils.GetFilesByExtInDir(path, "", &files)
+
+	for _, file := range files {
+		time := getFileModTime(file)
+		if dirChangeTime.Unix() < time.Unix() {
+			dirChangeTime = time
+		}
+	}
+
+	return
+}
+
 func getFileModTime(path string) time.Time {
 	f, err := os.Open(path)
 	if err != nil {
@@ -329,4 +412,90 @@ func getFileModTime(path string) time.Time {
 	//logUtils.Screen(i118Utils.I118Prt.Sprintf("file_change_time", timeStr))
 
 	return fileChangeTime
+}
+
+func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) {
+	excel, err := excelize.OpenFile(filePath)
+	if err != nil {
+		log.Println("fail to read file " + filePath + ", error: " + err.Error())
+		return
+	}
+
+	fileName := fileUtils.GetFileName(filePath)
+	fileName = strings.TrimSuffix(fileName, "词库")
+	colPrefix := stringUtils.GetPinyin(fileName)
+
+	for rowIndex, sheet := range excel.GetSheetList() {
+		rows, _ := excel.GetRows(sheet)
+		if len(rows) == 0 {
+			continue
+		}
+
+		colDefine := ""
+		colList := make([]string, 0)
+
+		colCount := 0
+		index := 0
+		for colIndex, col := range rows[0] {
+			val := strings.TrimSpace(col)
+			if rowIndex == 0 && val == "" {
+				break
+			}
+			colCount++
+
+			colName := stringUtils.GetPinyin(val)
+
+			if colIndex == 0 && colName != "ci" {
+				colName = "ci"
+			}
+			if colName != "ci" {
+				colName = colPrefix + "_" + colName
+			}
+
+			if (*colMap)[colName] == false {
+				colType := "VARCHAR"
+				colDefine = "    " + "`" + colName + "` " + colType + " DEFAULT ''"
+				*ddlFields = append(*ddlFields, colDefine)
+
+				(*colMap)[colName] = true
+			}
+			colList = append(colList, "`" + colName + "`")
+
+			index++
+		}
+
+		valList := make([]string, 0)
+		for rowIndex, row := range rows {
+			if rowIndex == 0 {
+				continue
+			}
+
+			valListItem := make([]string, 0)
+			valListItem = append(valListItem, strconv.Itoa(*seq))
+			*seq += 1
+
+			for i := 0; i < colCount; i++ {
+				val := ""
+				if i == 0 { // word
+					val = strings.TrimSpace(row[i])
+				} else if i <= len(row) - 1 { // excel value
+					val = strings.ToLower(strings.TrimSpace(row[i]))
+					if val != "y" && val != "b" && val != "f" && val != "m" {
+						val = ""
+					}
+				} else {
+					val = ""
+				}
+				valListItem = append(valListItem,"'" + val + "'")
+			}
+			valList = append(valList, "(" + strings.Join(valListItem, ", ") + ")")
+		}
+
+		insertTemplate := "INSERT INTO `" + tableName + "` (`seq`, %s) VALUES %s;"
+		insertSql := fmt.Sprintf(insertTemplate,
+			strings.Join(colList, ", "),
+			strings.Join(valList, ", "),
+		)
+		*insertSqls = append(*insertSqls, insertSql)
+	}
 }
\ No newline at end of file

src/gen/generator.go

@@ -33,7 +33,7 @@ func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]str
 
 	// 为每个field生成值列表
 	for index, field := range vari.Def.Fields {
-		if !stringUtils.FindInArr(field.Field, *fieldsToExport) {
+		if !stringUtils.StrInArr(field.Field, *fieldsToExport) {
 			continue
 		}
 
@@ -51,7 +51,7 @@ func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]str
 	// 处理数据
 	arrOfArr := make([][]string, 0) // 2 dimension arr for child, [ [a,b,c], [1,2,3] ]
 	for _, child := range vari.Def.Fields {
-		if !stringUtils.FindInArr(child.Field, *fieldsToExport) {
+		if !stringUtils.StrInArr(child.Field, *fieldsToExport) {
 			continue
 		}
 

src/gen/res.go

@@ -15,11 +15,12 @@ import (
 func LoadResDef(fieldsToExport []string) map[string]map[string][]string {
 	res := map[string]map[string][]string{}
 
-	for _, field := range vari.Def.Fields {
-		if !stringUtils.FindInArr(field.Field, fieldsToExport) { continue }
+	for index, field := range vari.Def.Fields {
+		if !stringUtils.StrInArr(field.Field, fieldsToExport) { continue }
 
-		if field.Use != "" && field.From == "" {
+		if (field.Use != "" || field.Select != "") && field.From == "" {
 			field.From = vari.Def.From
+			vari.Def.Fields[index].From = vari.Def.From
 		}
 		loadResField(&field, &res)
 	}
@@ -45,8 +46,14 @@ func loadResField(field *model.DefField, res *map[string]map[string][]string) {
 		}
 	} else if field.From != "" {
 		resFile, resType, sheet := fileUtils.GetResProp(field.From)
-		values, _ := getResValue(resFile, resType, sheet, field)
-		(*res)[field.From] = values
+		valueMap, _ := getResValue(resFile, resType, sheet, field)
+
+		if (*res)[field.From] == nil {
+			(*res)[field.From] = map[string][]string{}
+		}
+		for key, val := range valueMap {
+			(*res)[field.From][key] = val
+		}
 
 	} else if field.Config != "" {
 		resFile, resType, _ := fileUtils.GetResProp(field.Config)

src/model/definition.go

@@ -120,7 +120,6 @@ type DefFieldExport struct {
 	Prefix string  `yaml:"prefix"`
 	Postfix  string  `yaml:"postfix"`
 
-	From 	string  `yaml:"from"`
 	Select	string  `yaml:"select"`
 	Where	string  `yaml:"where"`
 	Rand  bool  `yaml:"rand"`

src/utils/const/const.go

@@ -62,7 +62,6 @@ var (
 
 	SqliteDriver = "sqlite3"
 	SqliteData   = "file:" + TmpDir + "cache/.data.db"
-	SqliteWords   = "file:" + TmpDir + "cache/.words.db"
 
 	SqliteTrackTable  = "excel_update"
 

src/utils/file/file.go

@@ -246,19 +246,60 @@ func convertResExcelPath(from string) (ret, sheet string) {
 				}
 				ret = realPth
 				return
-			} else {
-				dir := path.Dir(realPth)
-				if FileExist(dir) {
-					ret = dir
-					return
-				}
 			}
 		}
 	}
 
+	if ret == "" { // try excel dir
+		realPth := vari.WorkDir + constant.ResDirData + constant.PthSep +
+			strings.Replace(from, ".", constant.PthSep, -1)
+		if IsDir(realPth) {
+			ret = realPth
+			return
+		}
+	}
+
 	if ret == "" {
 		color.New(color.FgCyan).Fprintf(color.Output, i118Utils.I118Prt.Sprintf("fail_to_find_res", from) + "\n")
 	}
 
 	return
+}
+
+func GetFilesByExtInDir(folder, ext string, files *[]string) {
+	folder, _ = filepath.Abs(folder)
+
+	if !IsDir(folder) {
+		if ext == "" || path.Ext(folder) == ext {
+			*files = append(*files, folder)
+		}
+
+		return
+	}
+
+	dir, err := ioutil.ReadDir(folder)
+	if err != nil {
+		return
+	}
+
+	for _, fi := range dir {
+		name := fi.Name()
+		if commonUtils.IngoreFile(name) {
+			continue
+		}
+
+		filePath := AddSepIfNeeded(folder) + name
+		if fi.IsDir() {
+			GetFilesByExtInDir(filePath, ext, files)
+		} else if strings.Index(name, "~") != 0 && (ext == "" || path.Ext(filePath) == ext) {
+			*files = append(*files, filePath)
+		}
+	}
+}
+
+func GetFileName(filePath string) string {
+	fileName := path.Base(filePath)
+	fileName = strings.TrimSuffix(fileName, path.Ext(filePath))
+
+	return fileName
 }
\ No newline at end of file

src/utils/string/string.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"github.com/Chain-Zhang/pinyin"
 	"github.com/easysoft/zendata/src/model"
 	constant "github.com/easysoft/zendata/src/utils/const"
 	"github.com/mattn/go-runewidth"
@@ -62,13 +63,26 @@ func BoolToPass(b bool) string {
 	}
 }
 
-func FindInArr(str string, arr []string) bool {
-	for _, s := range arr {
+func FindInArr(str string, arr []string) (bool,int) {
+	for index, s := range arr {
 		if str == s {
-			return true
+			return true, index
 		}
 	}
 
+	return false, -1
+}
+
+func StrInArr(str string, arr []string) bool {
+	found, _ := FindInArr(str, arr)
+	return found
+}
+func InArray(need interface{}, arr []string) bool {
+	for _,v := range arr{
+		if need == v{
+			return true
+		}
+	}
 	return false
 }
 
@@ -100,15 +114,6 @@ func FormatStr(format string, val interface{}) (string, bool) {
 	return str, true
 }
 
-func InArray(need interface{}, arr []string) bool {
-	for _,v := range arr{
-		if need == v{
-			return true
-		}
-	}
-	return false
-}
-
 func AddPad(str string, field model.DefField) string {
 	if  field.Width > 0 && field.Width > runewidth.StringWidth(str) {
 		gap := field.Width - len(str)
@@ -160,4 +165,10 @@ func ConvertForSql(str string) (ret string) {
 	}
 
 	return
+}
+
+func GetPinyin(word string) string {
+	p, _ := pinyin.New(word).Split("").Mode(pinyin.WithoutTone).Convert()
+
+	return p
 }
\ No newline at end of file

test/article/convert_words_test.go

-package main
-
-import (
-	"database/sql"
-	"fmt"
-	"github.com/360EntSecGroup-Skylar/excelize/v2"
-	constant "github.com/easysoft/zendata/src/utils/const"
-	i118Utils "github.com/easysoft/zendata/src/utils/i118"
-	_ "github.com/mattn/go-sqlite3"
-	"log"
-	"strconv"
-	"strings"
-	"testing"
-)
-
-func TestImportSqlite(t *testing.T) {
-	files := make([]string, 0)
-	getFilesInDir("xdoc/words-9.3", ".xlsx", &files)
-
-	tableName := "words"
-	seq := 1
-	ddlFields := make([]string, 0)
-	insertSqls := make([]string, 0)
-
-	colMap := map[string]bool{}
-	for _, file := range files {
-		importExcel(file, tableName, &seq, &ddlFields, &insertSqls, &colMap)
-	}
-
-	db, err := sql.Open("sqlite3", constant.SqliteWords)
-	defer db.Close()
-
-	dropSql := `DROP TABLE IF EXISTS ` + tableName + `;`
-	_, err = db.Exec(dropSql)
-	if err != nil {
-		log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error()))
-		return
-	}
-
-	ddlTemplate := "CREATE TABLE " + tableName + "(\n" +
-		"\t`seq` CHAR (5) PRIMARY KEY ASC UNIQUE,\n" +
-		"%s" +
-		"\n);"
-	ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n"))
-	_, err = db.Exec(ddlSql)
-	if err != nil {
-		log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error()))
-		return
-	}
-
-	sql := strings.Join(insertSqls, "\n")
-	_, err = db.Exec(sql)
-	if err != nil {
-		log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error()))
-		return
-	}
-}
-
-func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) {
-	excel, err := excelize.OpenFile(filePath)
-	if err != nil {
-		log.Println("fail to read file " + filePath + ", error: " + err.Error())
-		return
-	}
-
-	fileName := getFileName(filePath)
-	fileName = strings.TrimSuffix(fileName, "词库")
-	colPrefix := getPinyin(fileName)
-
-	for rowIndex, sheet := range excel.GetSheetList() {
-		rows, _ := excel.GetRows(sheet)
-		if len(rows) == 0 {
-			continue
-		}
-
-		colDefine := ""
-		colList := make([]string, 0)
-
-		colCount := 0
-		index := 0
-		for colIndex, col := range rows[0] {
-			val := strings.TrimSpace(col)
-			if rowIndex == 0 && val == "" {
-				break
-			}
-			colCount++
-
-			colName := getPinyin(val)
-			if colIndex == 0 && colName != "ci" {
-				colName = "ci"
-			}
-			if colName != "ci" {
-				colName = colPrefix + "-" + colName
-			}
-
-			if (*colMap)[colName] == false {
-				colType := "VARCHAR"
-				colDefine = "    " + "`" + colName + "` " + colType
-				*ddlFields = append(*ddlFields, colDefine)
-
-				(*colMap)[colName] = true
-			}
-			colList = append(colList, "`" + colName + "`")
-
-			index++
-		}
-
-		valList := make([]string, 0)
-		for rowIndex, row := range rows {
-			if rowIndex == 0 {
-				continue
-			}
-
-			valListItem := make([]string, 0)
-			valListItem = append(valListItem, strconv.Itoa(*seq))
-			*seq += 1
-
-			for i := 0; i < colCount; i++ {
-				val := ""
-				if i == 0 { // word
-					val = strings.TrimSpace(row[i])
-				} else if i <= len(row) - 1 { // excel value
-					val = strings.ToLower(strings.TrimSpace(row[i]))
-					if val != "y" && val != "b" && val != "f" && val != "m" {
-						val = ""
-					}
-				} else {
-					val = ""
-				}
-				valListItem = append(valListItem,"'" + val + "'")
-			}
-			valList = append(valList, "(" + strings.Join(valListItem, ", ") + ")")
-		}
-
-		insertTemplate := "INSERT INTO `" + tableName + "` (`seq`, %s) VALUES %s;"
-		insertSql := fmt.Sprintf(insertTemplate,
-			strings.Join(colList, ", "),
-			strings.Join(valList, ", "),
-			)
-		*insertSqls = append(*insertSqls, insertSql)
-	}
-}
\ No newline at end of file

test/article/generate_yaml_test.go

@@ -20,9 +20,12 @@ const (
 	expLeft = "（"
 	expRight = "）"
 
-	src = "data/words/v1"
+	src = "data/words"
 	dist = "demo"
 )
+var (
+	compares = []string{"=", "!=", ">", "<"}
+)
 
 func TestGenerate(ts *testing.T) {
 	files := make([]string, 0)
@@ -48,8 +51,9 @@ func convertToYaml(article string) (content string) {
 		val := section["val"]
 
 		if tye == "exp" {
-			field := createField(index, prefix, val)
-			conf.XFields = append(conf.XFields, field)
+			fields := createFields(index, prefix, val)
+			conf.XFields = append(conf.XFields, fields...)
+
 			prefix = ""
 		} else {
 			prefix += val
@@ -79,8 +83,8 @@ func createDef(typ, table string) (conf model.DefExport) {
 	return
 }
 
-func createField(index int, prefix, exp string) (field model.DefFieldExport) {
-	field.From = strings.Replace(src, "/", ".", -1)
+func createFields(index int, prefix, exp string) (fields []model.DefFieldExport) {
+	field := model.DefFieldExport{}
 	field.Field = strconv.Itoa(index)
 	field.Prefix = prefix
 	field.Rand = true
@@ -99,10 +103,34 @@ func createField(index int, prefix, exp string) (field model.DefFieldExport) {
 	if strings.Index(exp, "=") == len(exp) - 2 {
 		exp = string(expArr[:len(expArr) - 2])
 		field.Select = getPinyin(exp)
-		field.Where = fmt.Sprintf("%s = %s", field.Select, string(expArr[len(expArr) - 1]))
+		field.Where = fmt.Sprintf("%s = '%s'", field.Select, string(expArr[len(expArr) - 1]))
 	} else {
 		field.Select = getPinyin(exp)
 		field.Where = "true"
+		//field.Where = getPinyin(exp) + " = 'y'"
+	}
+
+	if strings.Index(field.Select, "+") < 0 {
+		fields = append(fields, field)
+	} else if strings.Index(field.Select, "+") > 0 { // include more than one field, split to two
+		arr := strings.Split(field.Where, "=")
+		right := ""
+		if len(arr) > 1 {
+			right = arr[1]
+		}
+
+		items := strings.Split(field.Select, "+")
+		for _, item := range items {
+			var objClone interface{} = field
+			fieldClone := objClone.(model.DefFieldExport)
+			fieldClone.Select = item
+
+			if len(arr) > 1 { // has conditions
+				fieldClone.Where = item + " = " + right
+			}
+
+			fields = append(fields, fieldClone)
+		}
 	}
 
 	return