close task#7786

go.mod

@@ -24,5 +24,6 @@ require (
 	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
 	golang.org/x/sys v0.0.0-20200819171115-d785dc25833f // indirect
 	gopkg.in/ini.v1 v1.60.0
+	gopkg.in/yaml.v2 v2.2.8
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )

src/model/definition.go

@@ -107,4 +107,20 @@ func (def *DefSimple) Init(tableName, author, desc, version string) {
 }
 func (fld *FieldSimple) Init(field string) {
 	fld.Field = field
+}
+
+
+type DefExport struct {
+	ClsBase   `yaml:",inline"`
+	XFields  []DefFieldExport `yaml:"xfields,flow"` // control orders
+}
+type DefFieldExport struct {
+	Field string  `yaml:"field"`
+	Prefix string  `yaml:"prefix"`
+	Postfix  string  `yaml:"postfix"`
+
+	Select	string  `yaml:"select"`
+	Where	string  `yaml:"where"`
+	Rand  bool  `yaml:"rand"`
+	Limit	int  `yaml:"limit"`
 }
\ No newline at end of file

test/article/common.go

@@ -36,7 +36,7 @@ func getFilesInDir(folder, ext string, files *[]string) {
 		filePath := fileUtils.AddSepIfNeeded(folder) + name
 		if fi.IsDir() {
 			getFilesInDir(filePath, ext, files)
-		} else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ".xlsx" {
+		} else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ext {
 			*files = append(*files, filePath)
 		}
 	}

test/article/convert_test.go

@@ -6,8 +6,8 @@ import (
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
 	constant "github.com/easysoft/zendata/src/utils/const"
 	i118Utils "github.com/easysoft/zendata/src/utils/i118"
-	logUtils "github.com/easysoft/zendata/src/utils/log"
 	_ "github.com/mattn/go-sqlite3"
+	"log"
 	"strconv"
 	"strings"
 	"testing"
@@ -15,7 +15,7 @@ import (
 
 func TestImportSqlite(t *testing.T) {
 	files := make([]string, 0)
-	getFilesInDir("xdoc/words-9.3", "xlsx", &files)
+	getFilesInDir("xdoc/words-9.3", ".xlsx", &files)
 
 	tableName := "words"
 	seq := 1
@@ -33,7 +33,7 @@ func TestImportSqlite(t *testing.T) {
 	dropSql := `DROP TABLE IF EXISTS ` + tableName + `;`
 	_, err = db.Exec(dropSql)
 	if err != nil {
-		logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error()))
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error()))
 		return
 	}
 
@@ -44,14 +44,14 @@ func TestImportSqlite(t *testing.T) {
 	ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n"))
 	_, err = db.Exec(ddlSql)
 	if err != nil {
-		logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error()))
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error()))
 		return
 	}
 
 	sql := strings.Join(insertSqls, "\n")
 	_, err = db.Exec(sql)
 	if err != nil {
-		logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error()))
+		log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error()))
 		return
 	}
 }
@@ -59,7 +59,7 @@ func TestImportSqlite(t *testing.T) {
 func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) {
 	excel, err := excelize.OpenFile(filePath)
 	if err != nil {
-		logUtils.PrintTo("fail to read file " + filePath + ", error: " + err.Error())
+		log.Println("fail to read file " + filePath + ", error: " + err.Error())
 		return
 	}
 
@@ -90,7 +90,7 @@ func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]
 				colName = "ci"
 			}
 			if colName != "ci" {
-				colName = colPrefix + ":" + colName
+				colName = colPrefix + "-" + colName
 			}
 
 			if (*colMap)[colName] == false {

test/article/generate_test.go

 package main
 
 import (
+	"github.com/easysoft/zendata/src/model"
+	constant "github.com/easysoft/zendata/src/utils/const"
 	fileUtils "github.com/easysoft/zendata/src/utils/file"
 	_ "github.com/mattn/go-sqlite3"
+	"gopkg.in/yaml.v3"
+	"strconv"
+	"strings"
 	"testing"
 )
 
-func TestGenerate(t *testing.T) {
+const (
+	strLeft = "“"
+	strRight = "”"
+
+	expLeft = "（"
+	expRight = "）"
+)
+
+func TestGenerate(ts *testing.T) {
 	files := make([]string, 0)
 	getFilesInDir("xdoc/words-9.3", ".txt", &files)
 
@@ -20,7 +33,129 @@ func TestGenerate(t *testing.T) {
 }
 
 func convertToYaml(article string) (content string) {
+	sections := parseSections(article)
+
+	conf := createDef(constant.ConfigTypeArticle, "words.v1")
+
+	prefix := ""
+	for index, section := range sections {
+		tye := section["type"]
+		val := section["val"]
 
+		if tye == "exp" {
+			field := createField(index, prefix, val)
+			conf.XFields = append(conf.XFields, field)
+			prefix = ""
+		} else {
+			prefix += val
+		}
+	}
+
+	bytes, _ := yaml.Marshal(&conf)
+	content = string(bytes)
+
+	// convert yaml format by using a map
+	m := make(map[string]interface{})
+	yaml.Unmarshal([]byte(content), &m)
+	bytes, _ = yaml.Marshal(&m)
+	content = string(bytes)
+	content = strings.Replace(content, "xfields", "\nfields", -1)
+
+	return
+}
+
+func createDef(typ, table string) (conf model.DefExport) {
+	conf.Title = "automation"
+	conf.Author = "zendata"
+	conf.From = table
+	conf.Type = typ
+	conf.Desc = "generated from article text automatically"
+
+	return
+}
+
+func createField(index int, prefix, exp string) (field model.DefFieldExport) {
+	field.Field = strconv.Itoa(index)
+	field.Prefix = prefix
+
+	field.Select = getPinyin(exp)
+	field.Where = "true"
+	field.Rand = true
+	field.Limit = 1
+
+	return
+}
+
+func parseSections(content string) (sections []map[string]string) {
+	strStart := false
+	expStart := false
+
+	content = strings.TrimSpace(content)
+	runeArr := []rune(content)
+
+	section := ""
+	for i := 0; i < len(runeArr); i++ {
+		item := runeArr[i]
+		str := string(item)
+
+		isCouple, duplicateStr := isCouple(i, runeArr)
+		if isCouple {
+			section += duplicateStr
+			i += 1
+		} else if strStart && str == strRight { // str close
+			addSection(section, "str", &sections)
+
+			strStart = false
+			section = ""
+		} else if expStart && str == expRight { // exp close
+			addSection(section, "exp", &sections)
+
+			expStart = false
+			section = ""
+		} else if !strStart && !expStart && str == strLeft { // str start
+			if section != "" && strings.TrimSpace(section) != "+" {
+				addSection(section, "str", &sections)
+			}
+
+			strStart = true
+			section = ""
+		} else if !strStart && !expStart && str == expLeft { // exp start
+			if section != "" && strings.TrimSpace(section) != "+" {
+				addSection(section, "str", &sections)
+			}
+
+			expStart = true
+			section = ""
+		} else {
+			section += str
+		}
+	}
+
+	return
+}
+
+func addSection(str, typ string, arr *[]map[string]string) {
+	mp := map[string]string{}
+	mp["type"] = typ
+	mp["val"] = str
+
+	*arr = append(*arr, mp)
+}
+
+func isCouple(i int, arr []rune) (isCouple bool, duplicateStr string) {
+	if string(arr[i]) == strLeft && (i + 1 < len(arr) && string(arr[i + 1]) == strLeft) {
+		isCouple = true
+		duplicateStr = string(arr[i])
+	} else if string(arr[i]) == strRight && (i + 1 < len(arr) && string(arr[i + 1]) == strRight) {
+		isCouple = true
+		duplicateStr = string(arr[i])
+	} else if string(arr[i]) == expLeft && (i + 1 < len(arr) && string(arr[i + 1]) == expLeft) {
+		isCouple = true
+		duplicateStr = string(arr[i])
+	} else if string(arr[i]) == expRight && (i + 1 < len(arr) && string(arr[i + 1]) == expRight) {
+		isCouple = true
+		duplicateStr = string(arr[i])
+	}
 
 	return
 }
\ No newline at end of file