提交 c4a629d5 编写于 作者: aaronchen2k2k's avatar aaronchen2k2k

close tasks#7684

上级 c61b84b4
......@@ -19,28 +19,31 @@ fields:
postfix: "]\t"
- field: field_nested_range
from: custom.test.number.v1.yaml # 自定义通用ranges,保存于data目录下。
from: jenkins.number.v1.yaml # 自定义通用ranges,保存于data目录下。
use: medium
prefix: "["
postfix: "]\t"
- field: field_nested_instant
from: custom.ip.private.yaml # 內置instances引用其他ranges或instances。
from: jenkins.ip.private.yaml # 內置instances引用其他ranges或instances。
use: all
prefix: "["
postfix: "]\t"
- field: field_from_muti
prefix: "["
postfix: "]\t"
froms:
- from: custom.test.number.v1.yaml
- from: jenkins.number.v1.yaml
use: small{2}
postfix: "\t"
- from: custom.test.number.v1.yaml
postfix: "-"
- from: jenkins.number.v1.yaml
use: large{3}
- field: field_use_yaml
range: test.yaml{2},1-9:R{3}
postfix: "\t"
range: test/misc.yaml{2},1-9:R{3}
prefix: "==="
postfix: "====\t"
- field: field_literal
range: "`0000-00`,`AA[2,a-z]`,[1-3]:2{3},`[1-3]:2{3}`"
......
package action
import (
"encoding/json"
"fmt"
"github.com/easysoft/zendata/src/gen"
constant "github.com/easysoft/zendata/src/utils/const"
i118Utils "github.com/easysoft/zendata/src/utils/i118"
logUtils "github.com/easysoft/zendata/src/utils/log"
stringUtils "github.com/easysoft/zendata/src/utils/string"
"github.com/easysoft/zendata/src/utils/vari"
"github.com/mattn/go-runewidth"
"regexp"
"strings"
"time"
)
func Generate(defaultFile string, configFile string, total int, fieldsToExportStr string, out string, format string, table string) {
func Generate(defaultFile string, configFile string, fieldsToExportStr, format, table string) {
startTime := time.Now().Unix()
if defaultFile != "" && configFile == "" {
......@@ -28,216 +23,14 @@ func Generate(defaultFile string, configFile string, total int, fieldsToExportSt
fieldsToExport = strings.Split(fieldsToExportStr, ",")
}
vari.Total = total
rows, colIsNumArr, err := gen.GenerateForDefinition(defaultFile, configFile, &fieldsToExport, total)
rows, colIsNumArr, err := gen.GenerateForDefinition(defaultFile, configFile, &fieldsToExport)
if err != nil {
return
}
Print(rows, format, table, colIsNumArr, fieldsToExport)
gen.Print(rows, format, table, colIsNumArr, fieldsToExport)
entTime := time.Now().Unix()
if vari.RunMode == constant.RunModeServerRequest {
logUtils.PrintTo(i118Utils.I118Prt.Sprintf("server_response", len(rows), entTime - startTime))
}
}
func Print(rows [][]string, format string, table string, colIsNumArr []bool, fields []string) {
if format == constant.FormatText {
printTextHeader(fields)
} else if format == constant.FormatSql {
printSqlHeader(fields, table)
} else if format == constant.FormatJson {
printJsonHeader()
} else if format == constant.FormatXml {
printXmlHeader(fields, table)
}
for i, cols := range rows {
row := make([]string, 0)
rowMap := map[string]string{}
valuesForSql := make([]string, 0)
lineForText := ""
for j, col := range cols {
col = replacePlaceholder(col)
field := vari.TopFiledMap[fields[j]]
if field.Width > runewidth.StringWidth(col) {
col = stringUtils.AddPad(col, field)
}
if j > 0 && vari.Human {
lineForText = strings.TrimRight(lineForText, "\t")
col = strings.TrimLeft(col, "\t")
lineForText = lineForText + "\t" + col
} else {
lineForText = lineForText + col
}
row = append(row, col)
rowMap[field.Field] = col
colVal := stringUtils.ConvertForSql(col)
if !colIsNumArr[j] { colVal = "'" + colVal + "'" }
valuesForSql = append(valuesForSql, colVal)
}
if format == constant.FormatText {
logUtils.PrintLine(lineForText)
} else if format == constant.FormatSql {
logUtils.PrintLine(genSqlLine(strings.Join(valuesForSql, ", "), i, len(rows)))
} else if format == constant.FormatJson {
logUtils.PrintLine(genJsonLine(i, row, len(rows), fields))
} else if format == constant.FormatXml {
logUtils.PrintLine(getXmlLine(i, rowMap, len(rows)))
}
}
}
func printTextHeader(fields []string) {
if !vari.WithHead {
return
}
headerLine := ""
for idx, field := range fields {
headerLine += field
if idx < len(fields)-1 {
headerLine += "\t"
}
}
logUtils.PrintLine(headerLine)
}
func printSqlHeader(fields []string, table string) {
fieldNames := make([]string, 0)
for _, f := range fields { fieldNames = append(fieldNames, "`" + f + "`") }
logUtils.PrintLine(fmt.Sprintf("INSERT INTO %s(%s)", table, strings.Join(fieldNames, ", ")))
}
func printJsonHeader() {
logUtils.PrintLine("[")
}
func printXmlHeader(fields []string, table string) {
logUtils.PrintLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testdata>\n <title>Test Data</title>")
}
func RowToJson(cols []string, fieldsToExport []string) string {
rowMap := map[string]string{}
for j, col := range cols {
rowMap[fieldsToExport[j]] = col
}
jsonObj, _ := json.Marshal(rowMap)
respJson := string(jsonObj)
return respJson
}
func genSqlLine(valuesForSql string, i int, length int) string {
temp := ""
if i == 0 {
temp = fmt.Sprintf(" VALUES (%s)", valuesForSql)
} else {
temp = fmt.Sprintf(" (%s)", valuesForSql)
}
if i < length - 1 {
temp = temp + ", "
} else {
temp = temp + "; "
}
return temp
}
func genJsonLine(i int, row []string, length int, fields []string) string {
temp := " " + RowToJson(row, fields)
if i < length - 1 {
temp = temp + ", "
} else {
temp = temp + "\n]"
}
return temp
}
func getXmlLine(i int, mp map[string]string, length int) string {
str := ""
j := 0
for key, val := range mp {
str += fmt.Sprintf(" <%s>%s</%s>", key, val, key)
if j != len(mp) - 1 {
str = str + "\n"
}
j++
}
text := fmt.Sprintf(" <row>\n%s\n </row>", str)
if i == length - 1 {
text = text + "\n</testdata>"
}
return text
}
func replacePlaceholder(col string) string {
ret := col
re := regexp.MustCompile("(?siU)\\${(.*)}")
arr := re.FindAllStringSubmatch(col, -1)
strForReplaceMap := map[string][]string{}
for _, childArr := range arr {
placeholderStr := childArr[1]
strForReplaceMap[placeholderStr] = getValForPlaceholder(placeholderStr, len(childArr))
for _, str := range strForReplaceMap[placeholderStr] {
temp := gen.Placeholder(placeholderStr)
ret = strings.Replace(ret, temp, str, 1)
}
}
return ret
}
func getValForPlaceholder(placeholderStr string, count int) []string {
mp := vari.RandFieldNameToValuesMap[placeholderStr]
tp := mp["type"].(string)
repeatObj := mp["repeat"]
repeat := "1"
if repeatObj != nil {
repeat = repeatObj.(string)
}
strs := make([]string, 0)
if tp == "int" {
start := mp["start"].(string)
end := mp["end"].(string)
precision := mp["precision"].(string)
strs = gen.GetRandFromRange("int", start, end, "1", repeat, precision, count)
} else if tp == "float" {
start := mp["start"].(string)
end := mp["end"].(string)
precision := mp["precision"].(string)
strs = gen.GetRandFromRange("float", start, end, "1", repeat, precision, count)
} else if tp == "char" {
start := mp["start"].(string)
end := mp["end"].(string)
precision := mp["precision"].(string)
strs = gen.GetRandFromRange("char", start, end, "1", repeat, precision, count)
} else if tp == "list" {
list := mp["list"].([]string)
strs = gen.GetRandFromList(list, repeat, count)
}
return strs
}
\ No newline at end of file
......@@ -16,7 +16,7 @@ import (
)
func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]string,
total int) (rows [][]string, colIsNumArr []bool, err error) {
) (rows [][]string, colIsNumArr []bool, err error) {
vari.DefaultDir = fileUtils.GetAbsDir(defaultFile)
vari.ConfigDir = fileUtils.GetAbsDir(configFile)
......@@ -39,7 +39,7 @@ func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]str
if field.Use != "" && field.From == "" {
field.From = vari.Def.From
}
values := GenerateForField(&field, total, true)
values := GenerateForField(&field, true)
vari.Def.Fields[index].Precision = field.Precision
topFieldNameToValuesMap[field.Field] = values
......@@ -56,12 +56,12 @@ func GenerateForDefinition(defaultFile, configFile string, fieldsToExport *[]str
childValues := topFieldNameToValuesMap[child.Field]
arrOfArr = append(arrOfArr, childValues)
}
rows = putChildrenToArr(arrOfArr, total)
rows = putChildrenToArr(arrOfArr)
return
}
func GenerateForField(field *model.DefField, total int, withFix bool) (values []string) {
func GenerateForField(field *model.DefField, withFix bool) (values []string) {
if len(field.Fields) > 0 { // sub fields
arrOfArr := make([][]string, 0) // 2 dimension arr for child, [ [a,b,c], [1,2,3] ]
for _, child := range field.Fields {
......@@ -69,14 +69,14 @@ func GenerateForField(field *model.DefField, total int, withFix bool) (values []
child.From = field.From
}
childValues := GenerateForField(&child, total, withFix)
childValues := GenerateForField(&child, withFix)
arrOfArr = append(arrOfArr, childValues)
}
count := total
count := vari.Total
count = getRecordCount(arrOfArr)
if count > total {
count = total
if count > vari.Total {
count = vari.Total
}
values = combineChildrenValues(arrOfArr, count)
values = loopFieldValues(field, values, count, true)
......@@ -88,14 +88,14 @@ func GenerateForField(field *model.DefField, total int, withFix bool) (values []
child.From = field.From
}
childValues := GenerateForField(&child, total, withFix)
childValues := GenerateForField(&child, withFix)
arrOfArr = append(arrOfArr, childValues)
}
count := total
count := vari.Total
count = getRecordCount(arrOfArr)
if count > total {
count = total
if count > vari.Total {
count = vari.Total
}
values = combineChildrenValues(arrOfArr, count)
values = loopFieldValues(field, values, count, true)
......@@ -149,13 +149,13 @@ func GenerateForField(field *model.DefField, total int, withFix bool) (values []
values = append(values, groupValues[slct]...)
}
values = loopFieldValues(field, values, total, true)
values = loopFieldValues(field, values, vari.Total, true)
} else if field.Config != "" { // refer to config
groupValues := vari.Res[field.Config]
values = append(values, groupValues["all"]...)
values = loopFieldValues(field, values, total, true)
values = loopFieldValues(field, values, vari.Total, true)
} else { // leaf field
values = GenerateFieldValuesForDef(field)
......@@ -251,7 +251,7 @@ func loopFieldValues(field *model.DefField, oldValues []string, total int, withF
count++
isRandomAndLoopEnd := (*field).IsRand && (*field).LoopIndex == (*field).LoopEnd
isNotRandomAndValOver := !(*field).IsRand && indexOfRow >= len(fieldValue.Values)
if count >= vari.Total || isRandomAndLoopEnd || isNotRandomAndValOver {
if count >= total || isRandomAndLoopEnd || isNotRandomAndValOver {
break
}
......@@ -326,13 +326,13 @@ func computerLoop(field *model.DefField) {
(*field).LoopIndex = (*field).LoopStart
}
func putChildrenToArr(arrOfArr [][]string, total int) (values [][]string) {
func putChildrenToArr(arrOfArr [][]string) (values [][]string) {
indexArr := make([]int, 0)
if vari.Recursive {
indexArr = getModArr(arrOfArr)
}
for i := 0; i < total; i++ {
for i := 0; i < vari.Total; i++ {
strArr := make([]string, 0)
for j := 0; j < len(arrOfArr); j++ {
child := arrOfArr[j]
......@@ -355,7 +355,7 @@ func putChildrenToArr(arrOfArr [][]string, total int) (values [][]string) {
}
func combineChildrenValues(arrOfArr [][]string, total int) (ret []string) {
valueArr := putChildrenToArr(arrOfArr, total)
valueArr := putChildrenToArr(arrOfArr)
for _, arr := range valueArr {
ret = append(ret, strings.Join(arr, ""))
......
......@@ -58,6 +58,8 @@ func CreateFieldValuesFromList(field *model.DefField, fieldValue *model.FieldWit
items = CreateValuesFromLiteral(field, desc, stepStr, repeat)
} else if typ == "interval" {
items = CreateValuesFromInterval(field, desc, stepStr, repeat)
} else if typ == "yaml" {
items = CreateValuesFromYaml(field, desc, repeat)
}
fieldValue.Values = append(fieldValue.Values, items...)
......@@ -218,6 +220,21 @@ func CreateValuesFromInterval(field *model.DefField, desc string, stepStr string
return
}
func CreateValuesFromYaml(field *model.DefField, yamlFile string, repeat int) (items []interface{}) {
// keep root def, since vari.Def will be overwrite by refer yaml file
rootDef := vari.Def
configFile := vari.ConfigDir + yamlFile
fieldsToExport := make([]string, 0) // set to empty to use all fields
rows, colIsNumArr, _ := GenerateForDefinition("", configFile, &fieldsToExport)
items = Print(rows, constant.FormatData, "", colIsNumArr, fieldsToExport)
// rollback root def when finish to deal with refer yaml file
vari.Def = rootDef
return
}
func Placeholder(str string) string {
return "${" + str + "}"
}
......
package gen
import (
"encoding/json"
"fmt"
constant "github.com/easysoft/zendata/src/utils/const"
logUtils "github.com/easysoft/zendata/src/utils/log"
stringUtils "github.com/easysoft/zendata/src/utils/string"
"github.com/easysoft/zendata/src/utils/vari"
"github.com/mattn/go-runewidth"
"regexp"
"strings"
)
func Print(rows [][]string, format string, table string, colIsNumArr []bool,
fields []string) (lines []interface{}) {
if format == constant.FormatText {
printTextHeader(fields)
} else if format == constant.FormatSql {
printSqlHeader(fields, table)
} else if format == constant.FormatJson {
printJsonHeader()
} else if format == constant.FormatXml {
printXmlHeader(fields, table)
}
for i, cols := range rows {
row := make([]string, 0)
rowMap := map[string]string{}
valuesForSql := make([]string, 0)
lineForText := ""
for j, col := range cols {
col = replacePlaceholder(col)
field := vari.TopFiledMap[fields[j]]
if field.Width > runewidth.StringWidth(col) {
col = stringUtils.AddPad(col, field)
}
if j > 0 && vari.Human {
lineForText = strings.TrimRight(lineForText, "\t")
col = strings.TrimLeft(col, "\t")
lineForText = lineForText + "\t" + col
} else {
lineForText = lineForText + col
}
row = append(row, col)
rowMap[field.Field] = col
colVal := stringUtils.ConvertForSql(col)
if !colIsNumArr[j] { colVal = "'" + colVal + "'" }
valuesForSql = append(valuesForSql, colVal)
}
if format == constant.FormatText {
logUtils.PrintLine(lineForText)
} else if format == constant.FormatSql {
logUtils.PrintLine(genSqlLine(strings.Join(valuesForSql, ", "), i, len(rows)))
} else if format == constant.FormatJson {
logUtils.PrintLine(genJsonLine(i, row, len(rows), fields))
} else if format == constant.FormatXml {
logUtils.PrintLine(getXmlLine(i, rowMap, len(rows)))
} else if format == constant.FormatData {
lines = append(lines, lineForText)
}
}
return
}
func printTextHeader(fields []string) {
if !vari.WithHead {
return
}
headerLine := ""
for idx, field := range fields {
headerLine += field
if idx < len(fields)-1 {
headerLine += "\t"
}
}
logUtils.PrintLine(headerLine)
}
func printSqlHeader(fields []string, table string) {
fieldNames := make([]string, 0)
for _, f := range fields { fieldNames = append(fieldNames, "`" + f + "`") }
logUtils.PrintLine(fmt.Sprintf("INSERT INTO %s(%s)", table, strings.Join(fieldNames, ", ")))
}
func printJsonHeader() {
logUtils.PrintLine("[")
}
func printXmlHeader(fields []string, table string) {
logUtils.PrintLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testdata>\n <title>Test Data</title>")
}
func RowToJson(cols []string, fieldsToExport []string) string {
rowMap := map[string]string{}
for j, col := range cols {
rowMap[fieldsToExport[j]] = col
}
jsonObj, _ := json.Marshal(rowMap)
respJson := string(jsonObj)
return respJson
}
func genSqlLine(valuesForSql string, i int, length int) string {
temp := ""
if i == 0 {
temp = fmt.Sprintf(" VALUES (%s)", valuesForSql)
} else {
temp = fmt.Sprintf(" (%s)", valuesForSql)
}
if i < length - 1 {
temp = temp + ", "
} else {
temp = temp + "; "
}
return temp
}
func genJsonLine(i int, row []string, length int, fields []string) string {
temp := " " + RowToJson(row, fields)
if i < length - 1 {
temp = temp + ", "
} else {
temp = temp + "\n]"
}
return temp
}
func getXmlLine(i int, mp map[string]string, length int) string {
str := ""
j := 0
for key, val := range mp {
str += fmt.Sprintf(" <%s>%s</%s>", key, val, key)
if j != len(mp) - 1 {
str = str + "\n"
}
j++
}
text := fmt.Sprintf(" <row>\n%s\n </row>", str)
if i == length - 1 {
text = text + "\n</testdata>"
}
return text
}
func replacePlaceholder(col string) string {
ret := col
re := regexp.MustCompile("(?siU)\\${(.*)}")
arr := re.FindAllStringSubmatch(col, -1)
strForReplaceMap := map[string][]string{}
for _, childArr := range arr {
placeholderStr := childArr[1]
strForReplaceMap[placeholderStr] = getValForPlaceholder(placeholderStr, len(childArr))
for _, str := range strForReplaceMap[placeholderStr] {
temp := Placeholder(placeholderStr)
ret = strings.Replace(ret, temp, str, 1)
}
}
return ret
}
func getValForPlaceholder(placeholderStr string, count int) []string {
mp := vari.RandFieldNameToValuesMap[placeholderStr]
tp := mp["type"].(string)
repeatObj := mp["repeat"]
repeat := "1"
if repeatObj != nil {
repeat = repeatObj.(string)
}
strs := make([]string, 0)
if tp == "int" {
start := mp["start"].(string)
end := mp["end"].(string)
precision := mp["precision"].(string)
strs = GetRandFromRange("int", start, end, "1", repeat, precision, count)
} else if tp == "float" {
start := mp["start"].(string)
end := mp["end"].(string)
precision := mp["precision"].(string)
strs = GetRandFromRange("float", start, end, "1", repeat, precision, count)
} else if tp == "char" {
start := mp["start"].(string)
end := mp["end"].(string)
precision := mp["precision"].(string)
strs = GetRandFromRange("char", start, end, "1", repeat, precision, count)
} else if tp == "list" {
list := mp["list"].([]string)
strs = GetRandFromList(list, repeat, count)
}
return strs
}
\ No newline at end of file
......@@ -129,7 +129,7 @@ func getResForInstances(insts model.ResInsts) map[string][]string {
// gen values
group := child.Instance
groupedValueParent[group] = GenerateForField(&field, vari.Total, false)
groupedValueParent[group] = GenerateForField(&field, false)
}
} else if len(parentRanges.Ranges) > 0 {
groupedValueParent = getResForRanges(parentRanges)
......@@ -143,7 +143,7 @@ func getResForInstances(insts model.ResInsts) map[string][]string {
// gen values
group := inst.Instance
groupedValue[group] = GenerateForField(&field, vari.Total, false)
groupedValue[group] = GenerateForField(&field, false)
}
return groupedValue
......@@ -201,7 +201,7 @@ func getResForRanges(ranges model.ResRanges) map[string][]string {
tempField.Field = ranges.Field
tempField.Range = exp
groupedValue[group] = GenerateForField(&tempField, vari.Total, false)
groupedValue[group] = GenerateForField(&tempField, false)
}
return groupedValue
......@@ -211,7 +211,7 @@ func getResForConfig(configRes model.DefField) map[string][]string {
groupedValue := map[string][]string{}
// config field is a standard field
groupedValue["all"] = GenerateForField(&configRes, vari.Total, false)
groupedValue["all"] = GenerateForField(&configRes, false)
return groupedValue
}
\ No newline at end of file
......@@ -35,6 +35,7 @@ var (
FormatJson = "json"
FormatXml = "xml"
FormatSql = "sql"
FormatData = "data"
Formats = []string{FormatText, FormatJson, FormatXml, FormatSql}
TypeText = "text"
......
......@@ -157,10 +157,10 @@ func GetResProp(from string) (resFile, resType, sheet string) { // from resource
index := strings.LastIndex(from, ".yaml")
if index > -1 { // yaml, ip.v1.yaml
resFile = convertYamlPath(from)
resFile = ConvertResYamlPath(from)
resType = "yaml"
} else { // excel, like address.cn.v1.china
resFile, sheet = convertExcelPath(from)
resFile, sheet = convertResExcelPath(from)
resType = "excel"
}
......@@ -179,7 +179,7 @@ func GetResProp(from string) (resFile, resType, sheet string) { // from resource
return
}
func convertYamlPath(from string) (ret string) {
func ConvertResYamlPath(from string) (ret string) {
arr := strings.Split(from, ".")
for i := 0; i < len(arr); i++ {
dir := ""
......@@ -208,7 +208,7 @@ func convertYamlPath(from string) (ret string) {
return
}
func convertExcelPath(from string) (ret, sheet string) {
func convertResExcelPath(from string) (ret, sheet string) {
path1 := from // address.cn.v1
index := strings.LastIndex(from, ".")
path2 := from[:index] // address.cn.v1.china
......
......@@ -91,6 +91,8 @@ func PrintLine(line string) {
} else {
PrintToScreen(line)
}
return
}
func PrintToFile(line string) {
fmt.Fprint(FileWriter, line)
......
......@@ -28,7 +28,7 @@ import (
var (
defaultFile string
configFile string
count int
//count int
fields string
root string
......@@ -70,8 +70,8 @@ func main() {
flagSet.StringVar(&input, "i", "", "")
flagSet.StringVar(&input, "input", "", "")
flagSet.IntVar(&count, "n", 10, "")
flagSet.IntVar(&count, "lines", 10, "")
flagSet.IntVar(&vari.Total, "n", 10, "")
flagSet.IntVar(&vari.Total, "lines", 10, "")
flagSet.StringVar(&fields, "F", "", "")
flagSet.StringVar(&fields, "field", "", "")
......@@ -179,7 +179,7 @@ func toGen() {
StartServer()
} else if vari.RunMode == constant.RunModeServerRequest {
format = constant.FormatJson
action.Generate(defaultFile, configFile, count, fields, output, format, table)
action.Generate(defaultFile, configFile, fields, format, table)
} else if vari.RunMode == constant.RunModeParse {
action.ParseSql(input, output)
} else if vari.RunMode == constant.RunModeGen {
......@@ -207,7 +207,7 @@ func toGen() {
return
}
action.Generate(defaultFile, configFile, count, fields, output, format, table)
action.Generate(defaultFile, configFile, fields, format, table)
}
}
......@@ -234,7 +234,7 @@ func StartServer() {
func DataHandler(writer http.ResponseWriter, req *http.Request) {
logUtils.HttpWriter = writer
defaultFile, configFile, fields, count,
defaultFile, configFile, fields, vari.Total,
format, table, decode, input, output = service.ParseRequestParams(req)
if decode {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册