Merge remote-tracking branch 'origin/master'

# Conflicts:
#	cmd/test/import/comm/excel.go
#	cmd/test/import/import-address.go
#	cmd/test/import/import-text.go
#	internal/test/import/import-words.go

cmd/test/import/comm/excel.go

@@ -3,7 +3,7 @@ package comm
 import (
 	"fmt"
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
-	fileUtils "github.com/easysoft/zendata/pkg/utils/file"
+	fileUtils "github.com/easysoft/zendata/src/utils/file"
 	"strings"
 )
 

cmd/test/import/import-address.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"fmt"
-	"github.com/easysoft/zendata/internal/pkg/test/import/comm"
-	"github.com/easysoft/zendata/internal/pkg/test/import/model"
+	"github.com/easysoft/zendata/src/test/import/comm"
+	"github.com/easysoft/zendata/src/test/import/model"
 )
 
 func main() {

cmd/test/import/import-text.go

@@ -3,8 +3,8 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/easysoft/zendata/internal/pkg/test/import/comm"
-	fileUtils "github.com/easysoft/zendata/pkg/utils/file"
+	"github.com/easysoft/zendata/src/test/import/comm"
+	fileUtils "github.com/easysoft/zendata/src/utils/file"
 	"strings"
 )
 

cmd/test/import/model/model.go

@@ -54,10 +54,12 @@ func (DataCountry) TableName() string {
 type DataCity struct {
 	BaseModel
 
-	Name    string `json:"name"`
-	Code    string `json:"code"`
-	ZipCode string `json:"zipCode"`
-	State   string `json:"state"`
+	Name        string `json:"name"`
+	Code        string `json:"code"`
+	ZipCode     string `json:"zipCode"`
+	State       string `json:"state"`
+	StateShort  string `json:"stateShort"`
+	StateShort2 string `json:"stateShort2"`
 }
 
 func (DataCity) TableName() string {
@@ -160,3 +162,35 @@ type DataWordsConjunction struct {
 type DataWordsMeasure struct {
 	BaseModel
 }
+
+type DataWordTagGroup struct {
+	BaseModel
+	Name string         `gorm:"uniqueIndex" json:"name"`
+	Tags []*DataWordTag `gorm:"many2many:biz_data_word_tag_group_biz_data_word_tag" json:"tags"`
+}
+
+func (DataWordTagGroup) TableName() string {
+	return "biz_data_word_tag_group"
+}
+
+type DataWordTag struct {
+	BaseModel
+	Name  string      `gorm:"uniqueIndex" json:"name"`
+	Words []*DataWord `gorm:"many2many:biz_data_word_biz_data_word_tag" json:"words"`
+
+	Groups []*DataWordTagGroup `gorm:"many2many:biz_data_word_tag_group_biz_data_word_tag" json:"tags"`
+}
+
+func (DataWordTag) TableName() string {
+	return "biz_data_word_tag"
+}
+
+type DataWord struct {
+	BaseModel
+	Word string         `json:"word"`
+	Tags []*DataWordTag `gorm:"many2many:biz_data_word_biz_data_word_tag" json:"tags"`
+}
+
+func (DataWord) TableName() string {
+	return "biz_data_word"
+}

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/Chain-Zhang/pinyin v0.1.3
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/andybalholm/brotli v1.0.1 // indirect
-	github.com/dzwvip/oracle v1.2.1 // indirect
+	github.com/dzwvip/oracle v1.2.1
 	github.com/elazarl/go-bindata-assetfs v1.0.1
 	github.com/emirpasic/gods v1.12.0
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
@@ -46,5 +46,5 @@ require (
 	gorm.io/driver/sqlite v1.3.2
 	gorm.io/driver/sqlserver v1.3.2
 	gorm.io/gorm v1.23.5
-	gorm.io/plugin/dbresolver v1.2.0 // indirect
+	gorm.io/plugin/dbresolver v1.2.0
 )

go.sum

@@ -160,6 +160,7 @@ golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A=
 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/pkg/const/const.go

@@ -95,4 +95,9 @@ var (
 
 	TablePrefix = "zd_"
 	PageSize    = 15
+
+	// database type  [added by leo 2022/05/10]
+	DBTypeMysql     = "mysql"
+	DBTypeSqlServer = "sqlserver"
+	DBTypeOracle    = "oracle"
 )

internal/pkg/gen/printer.go

@@ -16,15 +16,14 @@ import (
 func Print(rows [][]string, format string, table string, colIsNumArr []bool,
 	fields []string) (lines []interface{}) {
 
+	var sqlHeader string
+
 	if format == constant.FormatText {
 		printTextHeader(fields)
 	} else if format == constant.FormatSql {
-		sqlHeader := getInsertSqlHeader(fields, table)
+		sqlHeader = getInsertSqlHeader(fields, table)
 		if vari.DBDsn != "" {
 			lines = append(lines, sqlHeader)
-		} else {
-			sqlHeader = "INSERT INTO " + sqlHeader
-			logUtils.PrintLine(sqlHeader)
 		}
 	} else if format == constant.FormatJson {
 		printJsonHeader()
@@ -58,12 +57,20 @@ func Print(rows [][]string, format string, table string, colIsNumArr []bool,
 			row = append(row, col)
 			rowMap[field.Field] = col
 
-			colVal := stringUtils.ConvertForSql(col)
+			colVal := col
 			if !colIsNumArr[j] {
-				colVal = "'" + colVal + "'"
+				switch vari.Server {
+				case constant.DBTypeSqlServer:
+					colVal = "'" + stringUtils.EscapeValueOfSqlServer(colVal) + "'"
+				case constant.DBTypeOracle:
+					colVal = "'" + stringUtils.EscapeValueOfOracle(colVal) + "'"
+				// case constant.DBTypeMysql:
+				default:
+					colVal = "'" + stringUtils.EscapeValueOfMysql(colVal) + "'"
+				}
 			}
 			valuesForSql = append(valuesForSql, colVal)
-		}
+		} // for cols
 
 		if format == constant.FormatText && vari.Def.Type == constant.ConfigTypeArticle { // article need to write to more than one files
 			lines = append(lines, lineForText)
@@ -72,15 +79,14 @@ func Print(rows [][]string, format string, table string, colIsNumArr []bool,
 			logUtils.PrintLine(lineForText)
 
 		} else if format == constant.FormatSql {
-
 			if vari.DBDsn != "" { // add to return array for exec sql
 				sql := strings.Join(valuesForSql, ", ")
 				lines = append(lines, sql)
 			} else {
-				sql := genSqlLine(strings.Join(valuesForSql, ", "), i, len(rows))
+
+				sql := genSqlLine(sqlHeader, valuesForSql, vari.Server)
 				logUtils.PrintLine(sql)
 			}
-
 		} else if format == constant.FormatJson {
 			logUtils.PrintLine(genJsonLine(i, row, len(rows), fields))
 		} else if format == constant.FormatXml {
@@ -108,15 +114,17 @@ func printTextHeader(fields []string) {
 	logUtils.PrintLine(headerLine)
 }
 
+// return "Table> (<column1, column2,...)"
 func getInsertSqlHeader(fields []string, table string) string {
 	fieldNames := make([]string, 0)
 	for _, f := range fields {
-		if vari.Server == "mysql" {
-			f = "`" + f + "`"
-		} else if vari.Server == "sqlserver" {
-			f = "[" + f + "]"
-		} else if vari.Server == "oracle" {
+		if vari.Server == constant.DBTypeSqlServer {
+			f = "[" + stringUtils.EscapeColumnOfSqlServer(f) + "]"
+		} else if vari.Server == constant.DBTypeOracle {
 			f = `"` + f + `"`
+		} else {
+			f = "`" + stringUtils.EscapeColumnOfMysql(f) + "`"
+			//vari.Server == constant.DBTypeMysql {
 		}
 
 		fieldNames = append(fieldNames, f)
@@ -124,12 +132,13 @@ func getInsertSqlHeader(fields []string, table string) string {
 
 	var ret string
 	switch vari.Server {
-	case "mysql":
-		ret = fmt.Sprintf("`%s` (%s)", table, strings.Join(fieldNames, ", "))
-	case "sqlserver":
+	case constant.DBTypeSqlServer:
 		ret = fmt.Sprintf("[%s] (%s)", table, strings.Join(fieldNames, ", "))
-	case "oracle":
+	case constant.DBTypeOracle:
 		ret = fmt.Sprintf(`"%s" (%s)`, table, strings.Join(fieldNames, ", "))
+	// case constant.DBTypeMysql:
+	default:
+		ret = fmt.Sprintf("`%s` (%s)", table, strings.Join(fieldNames, ", "))
 	}
 
 	return ret
@@ -155,22 +164,19 @@ func RowToJson(cols []string, fieldsToExport []string) string {
 	return respJson
 }
 
-func genSqlLine(valuesForSql string, i int, length int) string {
-
-	temp := ""
-	if i == 0 {
-		temp = fmt.Sprintf("  VALUES (%s)", valuesForSql)
-	} else {
-		temp = fmt.Sprintf("         (%s)", valuesForSql)
-	}
-
-	if i < length-1 {
-		temp = temp + ", "
-	} else {
-		temp = temp + "; "
+// @return ""
+func genSqlLine(sqlheader string, values []string, dbtype string) string {
+	var tmp string
+	switch dbtype {
+	case constant.DBTypeSqlServer:
+		tmp = "INSERT INTO " + sqlheader + " VALUES (" + strings.Join(values, ",") + "); GO"
+	default:
+		// constant.DBTypeMysql
+		// constant.DBTypeOracle:
+		tmp = "INSERT INTO " + sqlheader + " VALUES (" + strings.Join(values, ",") + ");"
 	}
 
-	return temp
+	return tmp
 }
 
 func genJsonLine(i int, row []string, length int, fields []string) string {

internal/test/import/import-words.go

+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/easysoft/zendata/src/test/import/comm"
+	"github.com/easysoft/zendata/src/test/import/model"
+	"gorm.io/gorm"
+)
+
+func main() {
+	filePathArr := []string{
+		"./data/words/v1/互联网黑话.xlsx",
+		"./data/words/v1/介词词库.xlsx",
+		"./data/words/v1/代词词库.xlsx",
+		"./data/words/v1/副词词库.xlsx",
+		"./data/words/v1/动词词库.xlsx",
+		"./data/words/v1/助词词库.xlsx",
+		"./data/words/v1/名词词库.xlsx",
+		"./data/words/v1/形容词做谓语.xlsx",
+		"./data/words/v1/形容词词库.xlsx",
+		"./data/words/v1/数词词库.xlsx",
+		"./data/words/v1/连词词库.xlsx",
+		"./data/words/v1/量词词库.xlsx",
+	}
+
+	db := comm.GetDB()
+	if err := db.AutoMigrate(
+		&model.DataWordTagGroup{}, &model.DataWordTag{}, &model.DataWord{},
+	); err != nil {
+		panic(err)
+	}
+
+	logs := []string{}
+	for i, path := range filePathArr {
+		sheetName, count := ImoprtWordsFromExcel(path, db)
+		str := fmt.Sprintf("%d) Path:[%s] SheetName:[%s], count:[%d]\n", i, path, sheetName, count)
+		fmt.Print(str)
+		logs = append(logs, str)
+	}
+
+	for _, l := range logs {
+		fmt.Print(l)
+	}
+}
+
+func ImoprtWordsFromExcel(filePath string, db *gorm.DB) (sheetName string, count int) {
+	sheetName, rows := comm.GetExcel1stSheet(filePath)
+	fmt.Printf("importing ... : [%s] \n", sheetName)
+	// fmt.Print(rows)
+
+	if len(rows) == 0 {
+		return
+	}
+
+	headers := rows[0]
+	if len(headers) < 1 {
+		return
+	}
+
+	headers = headers[1:]
+
+	// create word-tag-group
+	group := model.DataWordTagGroup{Name: strings.TrimSpace(sheetName)}
+	if err := db.Save(&group).Error; err != nil {
+		fmt.Errorf("creating word-tag-group : \n%v\n", err)
+		return
+	}
+
+	// create word-tags
+	tags := make([]model.DataWordTag, 0, len(headers))
+	for _, v := range headers {
+		tag := model.DataWordTag{Name: strings.TrimSpace(v)}
+
+		ret := db.First(&tag, "name = ?", strings.TrimSpace(v))
+		if ret.RowsAffected == 0 {
+			tag.Groups = []*model.DataWordTagGroup{&group}
+			if err := db.Save(&tag).Error; err != nil {
+				fmt.Errorf("sheetName:[%s], Tag[%s]", sheetName, v)
+				return
+			}
+		} else {
+			if err := db.Model(&tag).Association("Groups").Append(&group); err != nil {
+				fmt.Errorf("sheetName:[%s], Tag[%s]", sheetName, v)
+				return
+			}
+		}
+
+		tags = append(tags, tag)
+	}
+
+	// create word
+
+	words := make([]model.DataWord, 0, len(rows))
+
+	for i, r := range rows[1:] {
+		if len(r) == 0 {
+			fmt.Printf("shettName:[%s}, row index:[%d]", sheetName, i+1)
+			continue
+		}
+
+		word := model.DataWord{Word: strings.TrimSpace(r[0])}
+
+		flag := false
+		for j, v := range r[1:] {
+			if strings.TrimSpace(v) != "" {
+				word.Tags = []*model.DataWordTag{&tags[j]}
+				flag = true
+				break
+			}
+		}
+
+		if !flag {
+			fmt.Errorf("word (%v) is not tag.", word)
+		}
+
+		words = append(words, word)
+	}
+
+	tx := db.CreateInBatches(&words, 1000)
+	if tx.Error != nil {
+		fmt.Errorf("%v", tx.Error)
+	}
+
+	var count64 int64
+	tx.Count(&count64)
+	count = int(count64)
+
+	return
+}

pkg/utils/string/string.go

@@ -8,17 +8,18 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode"
+
 	"github.com/Chain-Zhang/pinyin"
 	constant "github.com/easysoft/zendata/internal/pkg/const"
 	"github.com/easysoft/zendata/internal/pkg/model"
 	"github.com/mattn/go-runewidth"
 	uuid "github.com/satori/go.uuid"
 	"gopkg.in/yaml.v2"
-	"net/url"
-	"regexp"
-	"strconv"
-	"strings"
-	"unicode"
 )
 
 func TrimAll(str string) string {
@@ -205,6 +206,50 @@ func ConvertForSql(str string) (ret string) {
 	return
 }
 
+func Escape(in string, shouldEscape []rune) (out string) {
+	out = ""
+	escapeChar := shouldEscape[0]
+
+	for _, v := range in {
+		for _, se := range shouldEscape {
+			if v == se {
+				out += string(escapeChar)
+				break
+			}
+		}
+
+		out += string(v)
+	}
+
+	return
+}
+
+func EscapeValueOfMysql(in string) string {
+	return Escape(in, []rune{'\\', '\'', '"'})
+}
+
+func EscapeValueOfSqlServer(in string) string {
+
+	return Escape(in, []rune{'\''})
+}
+
+func EscapeValueOfOracle(in string) string {
+
+	return Escape(in, []rune{'\''})
+}
+
+func EscapeColumnOfMysql(in string) string {
+
+	return Escape(in, []rune{'`'})
+}
+
+func EscapeColumnOfSqlServer(in string) string {
+	return Escape(in, []rune{']'})
+}
+
+// oracle limit
+//func EscapeColumnOfOracle(in string) string
+
 func GetPinyin(word string) string {
 	p, _ := pinyin.New(word).Split("").Mode(pinyin.WithoutTone).Convert()
 

res/en/usage.txt

@@ -6,11 +6,12 @@ Parameters:
   -c  --config     The current config file for data format, and it can override the config in the default file.
   -o  --output     The file name of the data generated. You can specify the output format by the extension name.
                    For example json, xml, sql, csv and xlsx.  The text data in the original format is output by default.
+                   Note: For SQL files, you can use --server to specify a specific database type in Mysql\Oracle\SqlServer.
   -n  --lines      The number of lines of data to be generated. The default is 10.
 
   -F  --field      This parameter can be used to specify the fields, separated by commas. The default is all fields.
   -t  --table      If the output format is sql, using it to specify the table name to insert data to.
-      --dsn        Specify MySQL data source connection string, used to insert data to table directly.
+      --dsn        Specify MySQL\SqlServer\Oracle data source connection string, used to insert data to table directly.
       --clear      Remove data in MySQL table before insert.
   -T  --trim       Remove the prefix and postfix of every field when outputting data or decode data to json string.
   -H  --human      Output a readable format, print the field name, and use the tab key to split.

res/zh/usage.txt

@@ -5,11 +5,12 @@ ZenData是一款通用的数据生成工具，您可以使用yaml文件来定义
   -d  --default    默认的数据格式配置文件。
   -c  --config     当前场景的数据格式配置文件，可以覆盖默认文件里面的设置。
   -o  --output     生成的数据的文件名。可通过扩展名指定输出json|xml|sql|csv|xlsx格式的数据。默认输出原始格式的文本数据。
+                   注意：对于 SQL 文件，您可以使用 --server 指定Mysql\Oracle\SqlServer中具体的数据库类型。
   -n  --lines      要生成的记录条数，默认为10条。
 
   -F  --field      可通过该参数指定要输出的字段列表，用逗号分隔。 默认是所有的字段。
   -t  --table      输出格式为sql时，需通过该参数指定要插入数据的表名。
-      --dns        指定MySQL数据源，用于直接插入数据到MySQL数据库表。
+      --dsn        指定MySQL\SqlServer\Oracle 数据源，用于直接插入数据到数据库表。
       --clear      插入数据前，删除已有数据。
   -T  --trim       输出的字段去除前后缀，通常用在生成SQL格式输出或解析测试数据时。
   -H  --human      输出可读格式，打印字段名，并使用tab键进行分割。