diff --git a/data/words/v1/article.txt b/data/words/article.txt similarity index 69% rename from data/words/v1/article.txt rename to data/words/article.txt index 71f4042b521895fffe31e244f824f6b736dc5850..0ab3bb4baf3522725298cede7415106eae819e25 100644 --- a/data/words/v1/article.txt +++ b/data/words/article.txt @@ -1 +1 @@ -“秋天的天气”+（形容词-天气-秋季-好）、（形容词-天气-秋季-好），（形容词-外貌-女性）+（名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），“因为”+（名词-称呼=F）+（动词-及物+名词-意愿）+“带她去”+（名词-地点-名山）+（动词-及物+名词-动作-眼部-中性）+（形容词-颜色-植物-树木）+（名词-植物-树木）+（连词-并列关系）+（形容词-颜色-植物-花卉）+（名词-植物-花卉）。（S：形容词-外貌-女性）+（S：名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），（副词-形容词做副词-情绪-快乐）+（动词-及物+名词-动作-手臂-亲切）+（形容词-性格-积极）+（S:名词-称呼=F）。 \ No newline at end of file +“秋天的天气”+（形容词-天气-秋季-好）、（形容词-天气-秋季-好），（形容词-外貌-女性）+（名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），“因为”+（名词-称呼=F）+（动词-及物_名词-意愿）+“带她去”+（名词-地点-名山）+（动词-及物_名词-动作-眼部-中性）+（形容词-颜色-植物-树木）+（名词-植物-树木）+（连词-并列关系）+（形容词-颜色-植物-花卉）+（名词-植物-花卉）。（S：形容词-外貌-女性）+（S：名词-姓+名词-名字=F）+（副词-程度副词）+（形容词做谓语-情绪-快乐），（副词-形容词做副词-情绪-快乐）+（动词-及物_名词-动作-手臂-亲切）+（形容词-性格-积极）+（S:名词-称呼=F）。 \ No newline at end of file diff --git "a/data/words/v1/\345\220\215\350\257\215\350\257\215\345\272\223.xlsx" "b/data/words/v1/\345\220\215\350\257\215\350\257\215\345\272\223.xlsx" index cda06ee2e442a4c7ae6c341180a39a700859d96f..d6a1f35ae1f7e9236d4937d9f8f80d889b234e9a 100644 Binary files "a/data/words/v1/\345\220\215\350\257\215\350\257\215\345\272\223.xlsx" and "b/data/words/v1/\345\220\215\350\257\215\350\257\215\345\272\223.xlsx" differ diff --git a/demo/article.yaml b/demo/article.yaml deleted file mode 100644 index 275302d905bafb58789e737be76406d7274176c7..0000000000000000000000000000000000000000 --- a/demo/article.yaml +++ /dev/null @@ -1,215 +0,0 @@ -author: ZenData -desc: Generated from article text automatically -from: words.v1 -title: automation -type: article -version: "" - -fields: - - field: "1" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: 秋天的天气 - rand: true - select: xingrongci-tianqi-qiuji-hao - useLastSameValue: false - where: "true" - - field: "3" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: 、 - rand: true - select: xingrongci-tianqi-qiuji-hao - useLastSameValue: false - where: "true" - - field: "5" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: ， - rand: true - select: xingrongci-waimao-nvxing - useLastSameValue: false - where: "true" - - field: "6" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: mingci-xing+mingci-mingzi - useLastSameValue: false - where: mingci-xing+mingci-mingzi = f - - field: "7" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: fuci-chengdufuci - useLastSameValue: false - where: "true" - - field: "8" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: xingrongcizuoweiyu-qingxu-kuaile - useLastSameValue: false - where: "true" - - field: "11" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: ，因为 - rand: true - select: mingci-chenghu - useLastSameValue: false - where: mingci-chenghu = f - - field: "12" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: dongci-jiwu+mingci-yiyuan - useLastSameValue: false - where: "true" - - field: "14" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: 带她去 - rand: true - select: mingci-dedian-mingshan - useLastSameValue: false - where: "true" - - field: "15" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: dongci-jiwu+mingci-dongzuo-yanbu-zhongxing - useLastSameValue: false - where: "true" - - field: "16" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: xingrongci-yanse-zhiwu-shumu - useLastSameValue: false - where: "true" - - field: "17" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: mingci-zhiwu-shumu - useLastSameValue: false - where: "true" - - field: "18" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: lianci-binglieguanxi - useLastSameValue: false - where: "true" - - field: "19" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: xingrongci-yanse-zhiwu-huahui - useLastSameValue: false - where: "true" - - field: "20" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: mingci-zhiwu-huahui - useLastSameValue: false - where: "true" - - field: "22" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: 。 - rand: true - select: xingrongci-waimao-nvxing - useLastSameValue: true - where: "true" - - field: "23" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: mingci-xing+mingci-mingzi - useLastSameValue: true - where: mingci-xing+mingci-mingzi = f - - field: "24" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: fuci-chengdufuci - useLastSameValue: false - where: "true" - - field: "25" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: xingrongcizuoweiyu-qingxu-kuaile - useLastSameValue: false - where: "true" - - field: "27" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: ， - rand: true - select: fuci-xingrongcizuofuci-qingxu-kuaile - useLastSameValue: false - where: "true" - - field: "28" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: dongci-jiwu+mingci-dongzuo-shoubi-qinqie - useLastSameValue: false - where: "true" - - field: "29" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: xingrongci-xingge-jiji - useLastSameValue: false - where: "true" - - field: "30" - from: data.words.v1 - limit: 1 - postfix: "" - prefix: "" - rand: true - select: mingci-chenghu - useLastSameValue: true - where: mingci-chenghu = f diff --git a/src/gen/definition.go b/src/gen/definition.go index e3c3352a8320485d14a0a9837452053b06a20714..14b4bd4c0408e46539388e5e31ba867598f708bf 100644 --- a/src/gen/definition.go +++ b/src/gen/definition.go @@ -59,11 +59,15 @@ func LoadConfigDef(defaultFile, configFile string, fieldsToExport *[]string) mod mergerDefine(&defaultDef, &configDef, fieldsToExport) orderFields(&defaultDef, *fieldsToExport) - for _, field := range defaultDef.Fields { + for index, field := range defaultDef.Fields { if vari.Trim { field.Prefix = "" field.Postfix = "" } + + if defaultDef.Type == constant.ConfigTypeArticle { + defaultDef.Fields[index].Select = strings.Replace(field.Select, "-", "_", -1) + } } return defaultDef @@ -91,6 +95,9 @@ func mergerDefine(defaultDef, configDef *model.DefData, fieldsToExport *[]string if configDef.From != "" && defaultDef.From == "" { defaultDef.From = configDef.From } + if configDef.Type != "" && defaultDef.Type == "" { + defaultDef.Type = configDef.Type + } for i, field := range defaultDef.Fields { if !isSetFieldsToExport { @@ -102,7 +109,7 @@ func mergerDefine(defaultDef, configDef *model.DefData, fieldsToExport *[]string for i, field := range configDef.Fields { vari.TopFiledMap[field.Field] = field if !isSetFieldsToExport { - if !stringUtils.FindInArr(field.Field, *fieldsToExport) { + if !stringUtils.StrInArr(field.Field, *fieldsToExport) { *fieldsToExport = append(*fieldsToExport, field.Field) } } diff --git a/src/gen/excel.go b/src/gen/excel.go index 074f886e8abe40aece3d2b2db083da00f731606d..cdfb458b49f082b00f18786ef98779f236711b3a 100644 --- a/src/gen/excel.go +++ b/src/gen/excel.go @@ -9,9 +9,11 @@ import ( fileUtils "github.com/easysoft/zendata/src/utils/file" i118Utils "github.com/easysoft/zendata/src/utils/i118" logUtils "github.com/easysoft/zendata/src/utils/log" + stringUtils "github.com/easysoft/zendata/src/utils/string" "github.com/easysoft/zendata/src/utils/vari" "log" "os" + "strconv" "strings" "time" ) @@ -139,7 +141,7 @@ func ConvertSingleExcelToSQLiteIfNeeded(dbName string, path string) (firstSheet ddl := fmt.Sprintf(ddlTemplate, tableName, colDefine) insertSql := fmt.Sprintf(insertTemplate, tableName, colList, valList) - db, err := sql.Open("sqlite3", constant.SqliteData) + db, err := sql.Open(constant.SqliteDriver, constant.SqliteData) defer db.Close() _, err = db.Exec(dropSql) if err != nil { @@ -163,8 +165,55 @@ func ConvertSingleExcelToSQLiteIfNeeded(dbName string, path string) (firstSheet return } -func ConvertExcelsToSQLiteIfNeeded(dbName string, dir string) { +func ConvertExcelsToSQLiteIfNeeded(tableName string, dir string) { + if !isExcelChanged(dir) { + return + } + + files := make([]string, 0) + fileUtils.GetFilesByExtInDir(dir, ".xlsx", &files) + + seq := 1 + ddlFields := make([]string, 0) + insertSqls := make([]string, 0) + + colMap := map[string]bool{} + for _, file := range files { + importExcel(file, tableName, &seq, &ddlFields, &insertSqls, &colMap) + } + + db, err := sql.Open(constant.SqliteDriver, constant.SqliteData) + defer db.Close() + + if err != nil { + log.Println(i118Utils.I118Prt.Sprintf("fail_to_connect_sqlite", constant.SqliteData, err.Error())) + return + } + + dropSql := `DROP TABLE IF EXISTS ` + tableName + `;` + _, err = db.Exec(dropSql) + if err != nil { + log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error())) + return + } + + ddlTemplate := "CREATE TABLE " + tableName + "(\n" + + "\t`seq` CHAR (5) PRIMARY KEY ASC UNIQUE,\n" + + "%s" + + "\n);" + ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n")) + _, err = db.Exec(ddlSql) + if err != nil { + log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error())) + return + } + sql := strings.Join(insertSqls, "\n") + _, err = db.Exec(sql) + if err != nil { + log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error())) + return + } return } @@ -179,13 +228,19 @@ func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) ( return list, "" } - selectCol := field.Select - from := dbName + "_" + tableName + selectCol := strings.Replace(field.Select, "-", "_", -1) + from := dbName + if tableName != "" { + from += "_" + tableName + } + where := field.Where - if where == "" { - where = "true" + if where == "" || (vari.Def.Type == constant.ConfigTypeArticle && strings.ToLower(where) == "true") { + where = "1=1" } + where = strings.Replace(where, "-", "_", -1) + if field.Rand { where += " ORDER BY RANDOM() " } @@ -198,7 +253,12 @@ func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) ( where = where + fmt.Sprintf(" LIMIT %d", total) } - sqlStr := fmt.Sprintf("SELECT %s FROM %s WHERE %s", selectCol, from, where) + colStr := selectCol + if vari.Def.Type == constant.ConfigTypeArticle { + colStr = "ci AS " + selectCol + } + + sqlStr := fmt.Sprintf("SELECT %s FROM %s WHERE %s", colStr, from, where) rows, err := db.Query(sqlStr) if err != nil { logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sqlStr, err.Error())) @@ -250,6 +310,17 @@ func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) ( } func isExcelChanged(path string) bool { + if !fileUtils.FileExist(path) { + return false + } + + fileChangeTime := time.Time{}.Unix() + if !fileUtils.IsDir(path) { + fileChangeTime = getFileModTime(path).Unix() + } else { + fileChangeTime = getDirModTime(path).Unix() + } + db, err := sql.Open(constant.SqliteDriver, constant.SqliteData) defer db.Close() if err != nil { @@ -265,8 +336,6 @@ func isExcelChanged(path string) bool { return true } - fileChangeTime := getFileModTime(path).Unix() - found := false changed := false for rows.Next() { @@ -309,6 +378,20 @@ func isExcelChanged(path string) bool { return changed } +func getDirModTime(path string) (dirChangeTime time.Time) { + files := make([]string, 0) + fileUtils.GetFilesByExtInDir(path, "", &files) + + for _, file := range files { + time := getFileModTime(file) + if dirChangeTime.Unix() < time.Unix() { + dirChangeTime = time + } + } + + return +} + func getFileModTime(path string) time.Time { f, err := os.Open(path) if err != nil { @@ -329,4 +412,90 @@ func getFileModTime(path string) time.Time { //logUtils.Screen(i118Utils.I118Prt.Sprintf("file_change_time", timeStr)) return fileChangeTime +} + +func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) { + excel, err := excelize.OpenFile(filePath) + if err != nil { + log.Println("fail to read file " + filePath + ", error: " + err.Error()) + return + } + + fileName := fileUtils.GetFileName(filePath) + fileName = strings.TrimSuffix(fileName, "词库") + colPrefix := stringUtils.GetPinyin(fileName) + + for rowIndex, sheet := range excel.GetSheetList() { + rows, _ := excel.GetRows(sheet) + if len(rows) == 0 { + continue + } + + colDefine := "" + colList := make([]string, 0) + + colCount := 0 + index := 0 + for colIndex, col := range rows[0] { + val := strings.TrimSpace(col) + if rowIndex == 0 && val == "" { + break + } + colCount++ + + colName := stringUtils.GetPinyin(val) + + if colIndex == 0 && colName != "ci" { + colName = "ci" + } + if colName != "ci" { + colName = colPrefix + "_" + colName + } + + if (*colMap)[colName] == false { + colType := "VARCHAR" + colDefine = " " + "`" + colName + "` " + colType + " DEFAULT ''" + *ddlFields = append(*ddlFields, colDefine) + + (*colMap)[colName] = true + } + colList = append(colList, "`" + colName + "`") + + index++ + } + + valList := make([]string, 0) + for rowIndex, row := range rows { + if rowIndex == 0 { + continue + } + + valListItem := make([]string, 0) + valListItem = append(valListItem, strconv.Itoa(*seq)) + *seq += 1 + + for i := 0; i < colCount; i++ { + val := "" + if i == 0 { // word + val = strings.TrimSpace(row[i]) + } else if i <= len(row) - 1 { // excel value + val = strings.ToLower(strings.TrimSpace(row[i])) + if val != "y" && val != "b" && val != "f" && val != "m" { + val = "" + } + } else { + val = "" + } + valListItem = append(valListItem,"'" + val + "'") + } + valList = append(valList, "(" + strings.Join(valListItem, ", ") + ")") + } + + insertTemplate := "INSERT INTO `" + tableName + "` (`seq`, %s) VALUES %s;" + insertSql := fmt.Sprintf(insertTemplate, + strings.Join(colList, ", "), + strings.Join(valList, ", "), + ) + *insertSqls = append(*insertSqls, insertSql) + } } \ No newline at end of file diff --git a/src/gen/generator.go b/src/gen/generator.go index a989a27b8e387d024eccafb44bcb810758affb29..7c4ace4e785a1a575c075ec35d9a5d30be9b121a 100644 --- a/src/gen/generator.go +++ b/src/gen/generator.go @@ -33,7 +33,7 @@ func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]str // 为每个field生成值列表 for index, field := range vari.Def.Fields { - if !stringUtils.FindInArr(field.Field, *fieldsToExport) { + if !stringUtils.StrInArr(field.Field, *fieldsToExport) { continue } @@ -51,7 +51,7 @@ func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]str // 处理数据 arrOfArr := make([][]string, 0) // 2 dimension arr for child, [ [a,b,c], [1,2,3] ] for _, child := range vari.Def.Fields { - if !stringUtils.FindInArr(child.Field, *fieldsToExport) { + if !stringUtils.StrInArr(child.Field, *fieldsToExport) { continue } diff --git a/src/gen/res.go b/src/gen/res.go index b67b0dd9b759c08cd176bde0e9c5938f054aed44..4666cb911af8a2160b81ac2d226f3d34d46e8204 100644 --- a/src/gen/res.go +++ b/src/gen/res.go @@ -15,11 +15,12 @@ import ( func LoadResDef(fieldsToExport []string) map[string]map[string][]string { res := map[string]map[string][]string{} - for _, field := range vari.Def.Fields { - if !stringUtils.FindInArr(field.Field, fieldsToExport) { continue } + for index, field := range vari.Def.Fields { + if !stringUtils.StrInArr(field.Field, fieldsToExport) { continue } - if field.Use != "" && field.From == "" { + if (field.Use != "" || field.Select != "") && field.From == "" { field.From = vari.Def.From + vari.Def.Fields[index].From = vari.Def.From } loadResField(&field, &res) } @@ -45,8 +46,14 @@ func loadResField(field *model.DefField, res *map[string]map[string][]string) { } } else if field.From != "" { resFile, resType, sheet := fileUtils.GetResProp(field.From) - values, _ := getResValue(resFile, resType, sheet, field) - (*res)[field.From] = values + valueMap, _ := getResValue(resFile, resType, sheet, field) + + if (*res)[field.From] == nil { + (*res)[field.From] = map[string][]string{} + } + for key, val := range valueMap { + (*res)[field.From][key] = val + } } else if field.Config != "" { resFile, resType, _ := fileUtils.GetResProp(field.Config) diff --git a/src/model/definition.go b/src/model/definition.go index 0008d47c36533fde97b099a7548de4d597c764d4..fbc9f2bdcd4a2f3feb30458908ccd137e038a72f 100644 --- a/src/model/definition.go +++ b/src/model/definition.go @@ -120,7 +120,6 @@ type DefFieldExport struct { Prefix string `yaml:"prefix"` Postfix string `yaml:"postfix"` - From string `yaml:"from"` Select string `yaml:"select"` Where string `yaml:"where"` Rand bool `yaml:"rand"` diff --git a/src/utils/const/const.go b/src/utils/const/const.go index ccf5c24aaa17cd89f078feff5b602ff61459218f..b2adfae7d67a2d2942a46bf485f122446aa0ccff 100644 --- a/src/utils/const/const.go +++ b/src/utils/const/const.go @@ -62,7 +62,6 @@ var ( SqliteDriver = "sqlite3" SqliteData = "file:" + TmpDir + "cache/.data.db" - SqliteWords = "file:" + TmpDir + "cache/.words.db" SqliteTrackTable = "excel_update" diff --git a/src/utils/file/file.go b/src/utils/file/file.go index 3bb79b1430f6bfccc928bcb15be3cdf06d27cd91..a288815dbf73b2896d20cbbe3addd1c799cdbb78 100644 --- a/src/utils/file/file.go +++ b/src/utils/file/file.go @@ -246,19 +246,60 @@ func convertResExcelPath(from string) (ret, sheet string) { } ret = realPth return - } else { - dir := path.Dir(realPth) - if FileExist(dir) { - ret = dir - return - } } } } + if ret == "" { // try excel dir + realPth := vari.WorkDir + constant.ResDirData + constant.PthSep + + strings.Replace(from, ".", constant.PthSep, -1) + if IsDir(realPth) { + ret = realPth + return + } + } + if ret == "" { color.New(color.FgCyan).Fprintf(color.Output, i118Utils.I118Prt.Sprintf("fail_to_find_res", from) + "\n") } return +} + +func GetFilesByExtInDir(folder, ext string, files *[]string) { + folder, _ = filepath.Abs(folder) + + if !IsDir(folder) { + if ext == "" || path.Ext(folder) == ext { + *files = append(*files, folder) + } + + return + } + + dir, err := ioutil.ReadDir(folder) + if err != nil { + return + } + + for _, fi := range dir { + name := fi.Name() + if commonUtils.IngoreFile(name) { + continue + } + + filePath := AddSepIfNeeded(folder) + name + if fi.IsDir() { + GetFilesByExtInDir(filePath, ext, files) + } else if strings.Index(name, "~") != 0 && (ext == "" || path.Ext(filePath) == ext) { + *files = append(*files, filePath) + } + } +} + +func GetFileName(filePath string) string { + fileName := path.Base(filePath) + fileName = strings.TrimSuffix(fileName, path.Ext(filePath)) + + return fileName } \ No newline at end of file diff --git a/src/utils/string/string.go b/src/utils/string/string.go index 6cee5dd501540465fed25e8cf470aaeb9973ff34..9378ffb1f73220aa10e2f034c8135466e9fc7e22 100644 --- a/src/utils/string/string.go +++ b/src/utils/string/string.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "github.com/Chain-Zhang/pinyin" "github.com/easysoft/zendata/src/model" constant "github.com/easysoft/zendata/src/utils/const" "github.com/mattn/go-runewidth" @@ -62,13 +63,26 @@ func BoolToPass(b bool) string { } } -func FindInArr(str string, arr []string) bool { - for _, s := range arr { +func FindInArr(str string, arr []string) (bool,int) { + for index, s := range arr { if str == s { - return true + return true, index } } + return false, -1 +} + +func StrInArr(str string, arr []string) bool { + found, _ := FindInArr(str, arr) + return found +} +func InArray(need interface{}, arr []string) bool { + for _,v := range arr{ + if need == v{ + return true + } + } return false } @@ -100,15 +114,6 @@ func FormatStr(format string, val interface{}) (string, bool) { return str, true } -func InArray(need interface{}, arr []string) bool { - for _,v := range arr{ - if need == v{ - return true - } - } - return false -} - func AddPad(str string, field model.DefField) string { if field.Width > 0 && field.Width > runewidth.StringWidth(str) { gap := field.Width - len(str) @@ -160,4 +165,10 @@ func ConvertForSql(str string) (ret string) { } return +} + +func GetPinyin(word string) string { + p, _ := pinyin.New(word).Split("").Mode(pinyin.WithoutTone).Convert() + + return p } \ No newline at end of file diff --git a/test/article/convert_words_test.go b/test/article/convert_words_test.go deleted file mode 100644 index 7f82f9dd6bab929601c655a46595bbc0c756b04a..0000000000000000000000000000000000000000 --- a/test/article/convert_words_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "github.com/360EntSecGroup-Skylar/excelize/v2" - constant "github.com/easysoft/zendata/src/utils/const" - i118Utils "github.com/easysoft/zendata/src/utils/i118" - _ "github.com/mattn/go-sqlite3" - "log" - "strconv" - "strings" - "testing" -) - -func TestImportSqlite(t *testing.T) { - files := make([]string, 0) - getFilesInDir("xdoc/words-9.3", ".xlsx", &files) - - tableName := "words" - seq := 1 - ddlFields := make([]string, 0) - insertSqls := make([]string, 0) - - colMap := map[string]bool{} - for _, file := range files { - importExcel(file, tableName, &seq, &ddlFields, &insertSqls, &colMap) - } - - db, err := sql.Open("sqlite3", constant.SqliteWords) - defer db.Close() - - dropSql := `DROP TABLE IF EXISTS ` + tableName + `;` - _, err = db.Exec(dropSql) - if err != nil { - log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error())) - return - } - - ddlTemplate := "CREATE TABLE " + tableName + "(\n" + - "\t`seq` CHAR (5) PRIMARY KEY ASC UNIQUE,\n" + - "%s" + - "\n);" - ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n")) - _, err = db.Exec(ddlSql) - if err != nil { - log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error())) - return - } - - sql := strings.Join(insertSqls, "\n") - _, err = db.Exec(sql) - if err != nil { - log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error())) - return - } -} - -func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) { - excel, err := excelize.OpenFile(filePath) - if err != nil { - log.Println("fail to read file " + filePath + ", error: " + err.Error()) - return - } - - fileName := getFileName(filePath) - fileName = strings.TrimSuffix(fileName, "词库") - colPrefix := getPinyin(fileName) - - for rowIndex, sheet := range excel.GetSheetList() { - rows, _ := excel.GetRows(sheet) - if len(rows) == 0 { - continue - } - - colDefine := "" - colList := make([]string, 0) - - colCount := 0 - index := 0 - for colIndex, col := range rows[0] { - val := strings.TrimSpace(col) - if rowIndex == 0 && val == "" { - break - } - colCount++ - - colName := getPinyin(val) - if colIndex == 0 && colName != "ci" { - colName = "ci" - } - if colName != "ci" { - colName = colPrefix + "-" + colName - } - - if (*colMap)[colName] == false { - colType := "VARCHAR" - colDefine = " " + "`" + colName + "` " + colType - *ddlFields = append(*ddlFields, colDefine) - - (*colMap)[colName] = true - } - colList = append(colList, "`" + colName + "`") - - index++ - } - - valList := make([]string, 0) - for rowIndex, row := range rows { - if rowIndex == 0 { - continue - } - - valListItem := make([]string, 0) - valListItem = append(valListItem, strconv.Itoa(*seq)) - *seq += 1 - - for i := 0; i < colCount; i++ { - val := "" - if i == 0 { // word - val = strings.TrimSpace(row[i]) - } else if i <= len(row) - 1 { // excel value - val = strings.ToLower(strings.TrimSpace(row[i])) - if val != "y" && val != "b" && val != "f" && val != "m" { - val = "" - } - } else { - val = "" - } - valListItem = append(valListItem,"'" + val + "'") - } - valList = append(valList, "(" + strings.Join(valListItem, ", ") + ")") - } - - insertTemplate := "INSERT INTO `" + tableName + "` (`seq`, %s) VALUES %s;" - insertSql := fmt.Sprintf(insertTemplate, - strings.Join(colList, ", "), - strings.Join(valList, ", "), - ) - *insertSqls = append(*insertSqls, insertSql) - } -} \ No newline at end of file diff --git a/test/article/generate_yaml_test.go b/test/article/generate_yaml_test.go index 72a8deb3aa710b791aff0f6c1ccad925dee47c8d..7ee5244b29fa469424e23477363a1fe833659874 100644 --- a/test/article/generate_yaml_test.go +++ b/test/article/generate_yaml_test.go @@ -20,9 +20,12 @@ const ( expLeft = "（" expRight = "）" - src = "data/words/v1" + src = "data/words" dist = "demo" ) +var ( + compares = []string{"=", "!=", ">", "<"} +) func TestGenerate(ts *testing.T) { files := make([]string, 0) @@ -48,8 +51,9 @@ func convertToYaml(article string) (content string) { val := section["val"] if tye == "exp" { - field := createField(index, prefix, val) - conf.XFields = append(conf.XFields, field) + fields := createFields(index, prefix, val) + conf.XFields = append(conf.XFields, fields...) + prefix = "" } else { prefix += val @@ -79,8 +83,8 @@ func createDef(typ, table string) (conf model.DefExport) { return } -func createField(index int, prefix, exp string) (field model.DefFieldExport) { - field.From = strings.Replace(src, "/", ".", -1) +func createFields(index int, prefix, exp string) (fields []model.DefFieldExport) { + field := model.DefFieldExport{} field.Field = strconv.Itoa(index) field.Prefix = prefix field.Rand = true @@ -99,10 +103,34 @@ func createField(index int, prefix, exp string) (field model.DefFieldExport) { if strings.Index(exp, "=") == len(exp) - 2 { exp = string(expArr[:len(expArr) - 2]) field.Select = getPinyin(exp) - field.Where = fmt.Sprintf("%s = %s", field.Select, string(expArr[len(expArr) - 1])) + field.Where = fmt.Sprintf("%s = '%s'", field.Select, string(expArr[len(expArr) - 1])) } else { field.Select = getPinyin(exp) field.Where = "true" + //field.Where = getPinyin(exp) + " = 'y'" + } + + if strings.Index(field.Select, "+") < 0 { + fields = append(fields, field) + } else if strings.Index(field.Select, "+") > 0 { // include more than one field, split to two + arr := strings.Split(field.Where, "=") + right := "" + if len(arr) > 1 { + right = arr[1] + } + + items := strings.Split(field.Select, "+") + for _, item := range items { + var objClone interface{} = field + fieldClone := objClone.(model.DefFieldExport) + fieldClone.Select = item + + if len(arr) > 1 { // has conditions + fieldClone.Where = item + " = " + right + } + + fields = append(fields, fieldClone) + } } return diff --git a/tmp/cache/.data.db b/tmp/cache/.data.db index 198439c2aca799644762e5cf2ef482410f0279e0..ade5af9cfef448ece1d7882b30e8e71c57805e28 100644 Binary files a/tmp/cache/.data.db and b/tmp/cache/.data.db differ diff --git a/tmp/cache/.words.db b/tmp/cache/.words.db deleted file mode 100644 index d70da63c41a227b859921da13fe3b66b2b408737..0000000000000000000000000000000000000000 Binary files a/tmp/cache/.words.db and /dev/null differ