load excel buildin resource

data/system/ip.yaml

@@ -10,10 +10,10 @@ instances:
     postfix: \t
     fields:
      - field: part3
-       range: 0-255
+       range: 0-254
        postfix: .
      - field: part4
-       range: 0-255
+       range: 1-254
 
   - instance: privateB
     prefix: 172.

project1/test.yaml

@@ -5,6 +5,17 @@ version: 1.0
 
 fields:
   - field: field1
-    note:
+    note: 测试自定义字段
     from: numb.yaml
-    use: small,large
\ No newline at end of file
+    use: small,large
+
+  - field: field2
+    note: 测试系统字段
+    from: system.ip.yaml
+    use: privatec,privateb
+
+  - field: field3
+    note: 测试Excel数据
+    from: system.address.city
+    select: name
+    where: state like '%山东%'
\ No newline at end of file

src/gen/cls.go

@@ -7,6 +7,7 @@ import (
 	logUtils "github.com/easysoft/zendata/src/utils/log"
 	stringUtils "github.com/easysoft/zendata/src/utils/string"
 	"github.com/easysoft/zendata/src/utils/vari"
+	"github.com/jinzhu/copier"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
@@ -44,16 +45,15 @@ func loadClsField(field *model.DefField, referFieldValueMap *map[string]map[stri
 			loadClsField(&child, referFieldValueMap)
 		}
 	} else if field.From != "" {
-		referFile, referType, tableName := getReferProp(field.From)
-		values, _ := getReferFieldValue(referFile, referType, tableName)
+		referFile, referType := getReferProp(field.From)
+		values, _ := getReferFieldValue(referFile, referType, field)
 		(*referFieldValueMap)[field.From] = values
 	}
 }
 
-func getReferProp(from string) (string, string, string) {
+func getReferProp(from string) (string, string) {
 	referFile := ""
 	referType := ""
-	tableName := ""
 
 	sep := string(os.PathSeparator)
 
@@ -71,7 +71,6 @@ func getReferProp(from string) (string, string, string) {
 		left = strings.ReplaceAll(left, ".", sep)
 
 		referFile = left + ".xlsx"
-		tableName = from[index:]
 		referType = "excel"
 	}
 
@@ -85,19 +84,21 @@ func getReferProp(from string) (string, string, string) {
 				referFile = ""
 			}
 		}
+	} else {
+		referFile = constant.ResDir + referFile
 	}
 
-	return referFile, referType, tableName
+	return referFile, referType
 }
 
-func getReferFieldValue(referFile string, referType string, tableName string) (map[string][]string, string) {
+func getReferFieldValue(referFile string, referType string, field *model.DefField) (map[string][]string, string) {
 	name := ""
 	values := map[string][]string{}
 
 	if referType == "yaml" {
 		values, name = getReferFieldValueForYaml(referFile)
 	} else if referType == "excel" {
-		values, name = getReferFieldValueForExcel(referFile, tableName)
+		values, name = getReferFieldValueForExcel(referFile, field)
 	}
 
 	return values, name
@@ -116,17 +117,10 @@ func getReferFieldValueForYaml(referFile string) (map[string][]string, string) {
 	}
 
 	err = yaml.Unmarshal(yamlContent, &ranges)
-	if err != nil || ranges.Ranges == nil || len(ranges.Ranges) == 0 {
-		logUtils.Screen("fail to parse ClsRanges " + referFile + ", try to parse as ClsInsts")
+	if err != nil || ranges.Ranges == nil || len(ranges.Ranges) == 0 { // instances
+		logUtils.Screen("not ClsRanges " + referFile + ", try to parse as ClsInsts")
 
 		insts := model.ClsInsts{}
-
-		yamlContent, err := ioutil.ReadFile(referFile)
-		if err != nil {
-			logUtils.Screen("fail to read " + referFile)
-			return valueMap, ""
-		}
-
 		err = yaml.Unmarshal(yamlContent, &insts)
 		if err != nil {
 			return valueMap, ""
@@ -136,7 +130,7 @@ func getReferFieldValueForYaml(referFile string) (map[string][]string, string) {
 
 		name = insts.Field
 
-	} else {
+	} else { // ranges
 		valueMap = getReferFieldValueForRanges(ranges)
 		name = ranges.Field
 	}
@@ -149,26 +143,34 @@ func getReferFieldValueForRanges(ranges model.ClsRanges) map[string][]string {
 
 	for name, exp := range ranges.Ranges {
 		// convert ranges refer to standard field
-		tempField := model.DefField{Field: ranges.Field, Range: exp}
+		tempField := model.DefField{Field: ranges.Field, Range: exp, Type: "cls"}
 
-		values[name] = GenerateFieldItemsFromDefinition(&tempField, constant.Total)
+		values[name] = GenerateFieldItemsFromDefinition(&tempField)
 	}
 
 	return values
 }
 
-func getReferFieldValueForInstances(referFile model.ClsInsts) map[string][]string {
+func getReferFieldValueForInstances(insts model.ClsInsts) map[string][]string {
 	values := map[string][]string{}
 
+	for _, inst := range insts.Instances {
+		// convert ranges refer to standard field
+		tempField := model.DefField{Field: insts.Field, Type: "cls"}
 
+		child := model.DefField{Field: inst.Instance}
+		copier.Copy(&child, inst)
 
-	return values
-}
+		tempField.Fields = append(tempField.Fields, child)
 
-func getReferFieldValueForExcel(referFile string, tableName string) (map[string][]string, string) {
-	values := map[string][]string{}
+		values[inst.Instance] = GenerateForField(&tempField, constant.Total)
+	}
 
+	return values
+}
 
+func getReferFieldValueForExcel(referFile string, field *model.DefField) (map[string][]string, string) {
+	values, dbName := GenerateFieldValuesFromExcel(referFile, field)
 
-	return values, ""
+	return values, dbName
 }
\ No newline at end of file

src/gen/excel.go

@@ -9,40 +9,27 @@ import (
 	logUtils "github.com/easysoft/zendata/src/utils/log"
 	"log"
 	"os"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
 )
 
-func GenerateFieldValuesFromExcel(field *model.DefField, fieldValue *model.FieldValue, level int) {
-	// get file and step string
-	rang := strings.TrimSpace(field.Range)
-	sectionArr := strings.Split(rang, ":")
-	file := sectionArr[0]
-	stepStr := "1"
-	if len(sectionArr) == 2 {
-		stepStr = sectionArr[1]
-	}
+func GenerateFieldValuesFromExcel(path string, field *model.DefField) (map[string][]string, string) {
+	values := map[string][]string{}
+
+	idx := strings.LastIndex(field.From, ".")
+	arr := strings.Split(field.From, ".")
+	dbName := arr[len(arr) - 2]
+	tableName := field.From[idx + 1:]
 
 	list := make([]string, 0)
-	path := constant.ResDir + file
-	ConvertExcelToSQLiteIfNeeded(*field, path)
+	ConvertExcelToSQLiteIfNeeded(dbName, path)
 
-	list = ReadDataFromSQLite(*field)
+	list = ReadDataFromSQLite(*field, dbName, tableName)
 
 	// get step and rand
 	rand := false
 	step := 1
-	if strings.ToLower(strings.TrimSpace(stepStr)) != "r" {
-		stepInt, err := strconv.Atoi(stepStr)
-		if err == nil {
-			step = stepInt
-		}
-	} else {
-		rand = true
-	}
-
 	// get index for data retrieve
 	numbs := GenerateIntItems(0, (int64)(len(list)-1), step, rand)
 	// get data by index
@@ -57,16 +44,14 @@ func GenerateFieldValuesFromExcel(field *model.DefField, fieldValue *model.Field
 			continue
 		}
 
-		fieldValue.Values = append(fieldValue.Values, item)
+		values[tableName] = append(values[tableName], item)
 		index = index + 1
 	}
 
-	if len(fieldValue.Values) == 0 {
-		fieldValue.Values = append(fieldValue.Values, "N/A")
-	}
+	return values, dbName
 }
 
-func ConvertExcelToSQLiteIfNeeded(field model.DefField, path string) {
+func ConvertExcelToSQLiteIfNeeded(dbName string, path string) {
 	excel, err := excelize.OpenFile(path)
 	if err != nil {
 		logUtils.Screen("fail to read file: " + err.Error())
@@ -128,7 +113,7 @@ func ConvertExcelToSQLiteIfNeeded(field model.DefField, path string) {
 			valList = valList + ")"
 		}
 
-		tableName := field.Field + "_" + sheet
+		tableName := dbName + "_" + sheet
 		dropSql := fmt.Sprintf(dropTemplate, tableName)
 		ddl := fmt.Sprintf(ddlTemplate, tableName, colDefine)
 		insertSql := fmt.Sprintf(insertTemplate, tableName, colList, valList)
@@ -151,7 +136,7 @@ func ConvertExcelToSQLiteIfNeeded(field model.DefField, path string) {
 	}
 }
 
-func ReadDataFromSQLite(field model.DefField) []string {
+func ReadDataFromSQLite(field model.DefField, dbName string, tableName string) []string {
 	list := make([]string, 0)
 
 	db, err := sql.Open(constant.SqliteDriver, constant.SqliteSource)
@@ -160,9 +145,15 @@ func ReadDataFromSQLite(field model.DefField) []string {
 		logUtils.Screen("fail to open " + constant.SqliteSource + ": " + err.Error())
 		return list
 	}
-	field.From, field.Where = convertSql(field.From, field.Where)
 
-	sqlStr := fmt.Sprintf("SELECT %s FROM %s WHERE %s", field.Select, field.From, field.Where)
+	selectCol := field.Select
+	from := dbName + "_" + tableName
+	where := field.Where
+	if !strings.Contains(where, "LIMIT") {
+		where = where + " LIMIT " + strconv.Itoa(constant.MaxNumb)
+	}
+
+	sqlStr := fmt.Sprintf("SELECT %s FROM %s WHERE %s", selectCol, from, where)
 	rows, err := db.Query(sqlStr)
 	if err != nil {
 		logUtils.Screen("fail to exec query " + err.Error())
@@ -201,57 +192,16 @@ func ReadDataFromSQLite(field model.DefField) []string {
 		valMapArr = append(valMapArr, rowMap)
 	}
 
-	format := field.Format
 	for _, item := range valMapArr {
-		line := replacePlaceholderWithValue(format,item)
-		list = append(list, line)
-	}
-
-	return list
-}
-
-func replacePlaceholderWithValue(format string, valMap map[string]string) string {
-	// ${user_name}_${numb}@${domain}
-	regx := regexp.MustCompile(`\$\{([a-zA-z0-9_]+)\}`)
-	arrOfName := regx.FindAllStringSubmatch(format, -1)
-
-	ret := ""
-	if len(arrOfName) > 0 {
-		strLeft := format
-		for index, a := range arrOfName {
-			found := a[0]
-			name := a[1]
-
-			arr := strings.Split(strLeft, found)
-
-			// add string constant
-			if arr[0] != "" {
-				ret = ret + arr[0]
-			}
-
-			ret = ret + valMap[name]
-
-			arr = arr[1:]
-			strLeft = strings.Join(arr, "")
-
-			if index == len(arrOfName) - 1 && strLeft != "" { // add last item in arr
-				ret = ret + strLeft
-			}
+		idx := 0
+		for _, val := range item {
+			if idx > 0 { break }
+			list = append(list, val)
+			idx++
 		}
 	}
 
-	return ret
-}
-
-func convertSql(from string, where string) (string, string) {
-	from = strings.Replace(from,"system.", "", -1)
-	from = strings.Replace(from,".", "_", -1)
-
-	if !strings.Contains(where, "LIMIT") {
-		where = where + " LIMIT " + strconv.Itoa(constant.MaxNumb)
-	}
-
-	return from, where
+	return list
 }
 
 func isExcelChanged(path string) bool {
@@ -262,7 +212,7 @@ func isExcelChanged(path string) bool {
 		return true
 	}
 
-	sqlStr := "SELECT id, name, changeTime FROM excel_change"
+	sqlStr := "SELECT id, name, changeTime FROM " + constant.SqliteTrackTable
 	rows, err := db.Query(sqlStr)
 	if err != nil {
 		logUtils.Screen("fail to exec query " + sqlStr + ": " + err.Error())

src/gen/generator.go

@@ -94,13 +94,13 @@ func GenerateForField(field *model.DefField,  total int) []string {
 		// TODO: 需要处理range: small,large等逻辑
 
 	} else {
-		values = GenerateFieldItemsFromDefinition(field, total)
+		values = GenerateFieldItemsFromDefinition(field)
 	}
 
 	return values
 }
 
-func GenerateFieldItemsFromDefinition(field *model.DefField, total int) []string {
+func GenerateFieldItemsFromDefinition(field *model.DefField) []string {
 	if field.Loop == 0 {field.Loop = 1}
 
 	values := make([]string, 0)
@@ -109,7 +109,7 @@ func GenerateFieldItemsFromDefinition(field *model.DefField, total int) []string
 	//datatype := strings.TrimSpace(field.Type)
 	//if datatype == "" { datatype = "list" }
 
-	fieldValue := GenerateList(field, total)
+	fieldValue := GenerateList(field)
 
 	index := 0
 	count := 0
@@ -119,7 +119,7 @@ func GenerateFieldItemsFromDefinition(field *model.DefField, total int) []string
 		values = append(values, str)
 
 		count++
-		if count >= total {
+		if count >= constant.Total {
 			break
 		}
 	}

src/gen/list.go

@@ -7,38 +7,39 @@ import (
 	"strings"
 )
 
-func GenerateList(field *model.DefField, total int) model.FieldValue {
+func GenerateList(field *model.DefField) model.FieldValue {
 	fieldValue := model.FieldValue{}
-	GenerateListField(field, &fieldValue, 0)
+	GenerateListField(field, &fieldValue)
 
 	return fieldValue
 }
 
-func GenerateListField(field *model.DefField, fieldValue *model.FieldValue, level int) {
+func GenerateListField(field *model.DefField, fieldValue *model.FieldValue) {
 	fieldValue.Field = field.Field
 	fieldValue.Precision = field.Precision
 
 	if len(field.Fields) > 0 {
 		for _, child := range field.Fields {
 			childValue := model.FieldValue{}
-			GenerateListField(&child, &childValue, level + 1)
+			GenerateListField(&child, &childValue)
 		}
 	} else {
-		GenerateFieldValues(field, fieldValue, level)
+		GenerateFieldValues(field, fieldValue)
 	}
 }
 
-func GenerateFieldValues(field *model.DefField, fieldValue *model.FieldValue, level int) {
+func GenerateFieldValues(field *model.DefField, fieldValue *model.FieldValue) {
 	if strings.Index(field.Range, ".txt") > -1 {
-		GenerateFieldValuesFromText(field, fieldValue, level)
-	} else if strings.Index(field.Range, ".xlsx") > -1 {
-		GenerateFieldValuesFromExcel(field, fieldValue, level)
+		GenerateFieldValuesFromText(field, fieldValue)
 	} else {
-		GenerateFieldValuesFromList(field, fieldValue, level)
+		GenerateFieldValuesFromList(field, fieldValue)
 	}
+	//else if strings.Index(field.Range, ".xlsx") > -1 {
+	//	GenerateFieldValuesFromExcel(field, fieldValue)
+	//}
 }
 
-func GenerateFieldValuesFromList(field *model.DefField, fieldValue *model.FieldValue, level int) {
+func GenerateFieldValuesFromList(field *model.DefField, fieldValue *model.FieldValue) {
 	//rang := strings.TrimSpace(field.Range)
 	rang := field.Range
 

src/gen/text.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 )
 
-func GenerateFieldValuesFromText(field *model.DefField, fieldValue *model.FieldValue, level int) {
+func GenerateFieldValuesFromText(field *model.DefField, fieldValue *model.FieldValue) {
 	// get file and step string
 	rang := strings.TrimSpace(field.Range)
 	sectionArr := strings.Split(rang, ":")

src/model/definition.go

@@ -23,6 +23,8 @@ type ClsInsts struct {
 type ClsInst struct {
 	FieldBase   `yaml:",inline"`
 	Instance string  `yaml:"instance"`
+	Range    string  `yaml:"range"`
+	Fields  []DefField `yaml:"fields,flow"`
 }
 
 // common item
@@ -34,8 +36,9 @@ type DefField struct {
 	FieldBase   `yaml:",inline"`
 	Field     string  `yaml:"field"`
 	Range    string  `yaml:"range"`
-
 	Fields   []DefField `yaml:"fields,flow"`
+
+	Type     string
 }
 
 type FieldBase struct {

src/utils/const/const.go

@@ -36,6 +36,8 @@ var (
 
 	SqliteDriver string = "sqlite3"
 	SqliteSource string = "file:" + ResDir + ".cache/.data.db"
+	SqliteTrackTable string = "excel_update"
+
 	ExcelBorder string = `{"border": [{"type":"left","color":"999999","style":1}, {"type":"top","color":"999999","style":1},
 		                              {"type":"bottom","color":"999999","style":1}, {"type":"right","color":"999999","style":1}]}`
 	ExcelHeader string = `{"fill":{"type":"pattern","pattern":1,"color":["E0EBF5"]}}`