提交 a12c1541 编写于 作者: aaronchen2k2k's avatar aaronchen2k2k

add article convert command and doc

上级 0fe8e4ad
author: ZenData
desc: Generated from article text automatically
from: words.v1
title: automation
type: article
version: ""
fields:
- field: "1"
limit: 1
postfix: ""
prefix: 秋天的天气
rand: true
select: xingrongci-tianqi-qiuji-hao
useLastSameValue: false
where: "true"
- field: "3"
limit: 1
postfix: ""
prefix:
rand: true
select: xingrongci-tianqi-qiuji-hao
useLastSameValue: false
where: "true"
- field: "5"
limit: 1
postfix: ""
prefix:
rand: true
select: xingrongci-waimao-nvxing
useLastSameValue: false
where: "true"
- field: "6"
limit: 1
postfix: ""
prefix: ""
rand: true
select: mingci-xing
useLastSameValue: false
where: mingci-xing = 'f'
- field: "6"
limit: 1
postfix: ""
prefix: ""
rand: true
select: mingci-mingzi
useLastSameValue: false
where: mingci-mingzi = 'f'
- field: "7"
limit: 1
postfix: ""
prefix: ""
rand: true
select: fuci-chengdufuci
useLastSameValue: false
where: "true"
- field: "8"
limit: 1
postfix: ""
prefix: ""
rand: true
select: xingrongcizuoweiyu-qingxu-kuaile
useLastSameValue: false
where: "true"
- field: "11"
limit: 1
postfix: ""
prefix: ,因为
rand: true
select: mingci-chenghu
useLastSameValue: false
where: mingci-chenghu = 'f'
- field: "12"
limit: 1
postfix: ""
prefix: ""
rand: true
select: dongci-jiwu_mingci-yiyuan
useLastSameValue: false
where: "true"
- field: "14"
limit: 1
postfix: ""
prefix: 带她去
rand: true
select: mingci-dedian-mingshan
useLastSameValue: false
where: "true"
- field: "15"
limit: 1
postfix: ""
prefix: ""
rand: true
select: dongci-jiwu_mingci-dongzuo-yanbu-zhongxing
useLastSameValue: false
where: "true"
- field: "16"
limit: 1
postfix: ""
prefix: ""
rand: true
select: xingrongci-yanse-zhiwu-shumu
useLastSameValue: false
where: "true"
- field: "17"
limit: 1
postfix: ""
prefix: ""
rand: true
select: mingci-zhiwu-shumu
useLastSameValue: false
where: "true"
- field: "18"
limit: 1
postfix: ""
prefix: ""
rand: true
select: lianci-binglieguanxi
useLastSameValue: false
where: "true"
- field: "19"
limit: 1
postfix: ""
prefix: ""
rand: true
select: xingrongci-yanse-zhiwu-huahui
useLastSameValue: false
where: "true"
- field: "20"
limit: 1
postfix: ""
prefix: ""
rand: true
select: mingci-zhiwu-huahui
useLastSameValue: false
where: "true"
- field: "22"
limit: 1
postfix: ""
prefix:
rand: true
select: xingrongci-waimao-nvxing
useLastSameValue: true
where: "true"
- field: "23"
limit: 1
postfix: ""
prefix: ""
rand: true
select: mingci-xing
useLastSameValue: true
where: mingci-xing = 'f'
- field: "23"
limit: 1
postfix: ""
prefix: ""
rand: true
select: mingci-mingzi
useLastSameValue: true
where: mingci-mingzi = 'f'
- field: "24"
limit: 1
postfix: ""
prefix: ""
rand: true
select: fuci-chengdufuci
useLastSameValue: false
where: "true"
- field: "25"
limit: 1
postfix: ""
prefix: ""
rand: true
select: xingrongcizuoweiyu-qingxu-kuaile
useLastSameValue: false
where: "true"
- field: "27"
limit: 1
postfix: ""
prefix:
rand: true
select: fuci-xingrongcizuofuci-qingxu-kuaile
useLastSameValue: false
where: "true"
- field: "28"
limit: 1
postfix: ""
prefix: ""
rand: true
select: dongci-jiwu_mingci-dongzuo-shoubi-qinqie
useLastSameValue: false
where: "true"
- field: "29"
limit: 1
postfix: ""
prefix: ""
rand: true
select: xingrongci-xingge-jiji
useLastSameValue: false
where: "true"
- field: "30"
limit: 1
postfix: "\n\n"
prefix: ""
rand: true
select: mingci-chenghu
useLastSameValue: true
where: mingci-chenghu = 'f'
......@@ -16,7 +16,7 @@ Parameters:
-r --recursive Recursive mode. The default mode is parallel, in which each field loops independently.
The value of a field in the recursive mode depends on that of the previous field, which enables the random data.
-p --port Run the HTTP on the specified port. The data in JSON format can be obtained via http://ip/ port.
-p --port Run the HTTP on the specified port. The data in JSON format can be obtained via http:\\ip\ port.
Only data generation is supported.
-b --bind Listen IP addresses. All IP addresses are listened by default.
-R --root The root directory when running HTTP. The client can call the config file under the root directory.
......@@ -26,6 +26,7 @@ Parameters:
You need to specify an output directory by using -o.
-D --decode Referring to the specified configuration file, parse the data file specified by -i and output json.
Also you can output the readable format via -H.
-a --article Convert article to yaml config file in the dir provided by -o parameter.
-l --list List all supported data formats.
-v --view View the detailed definition of a data format.
......@@ -34,24 +35,25 @@ Parameters:
Command Line Examples:
$>zd.exe -d demo/default.yaml # Generate 10 lines of data according to the config file specified by -d.
$>zd.exe -c demo/default.yaml # Generate 10 lines of data according to the config file specified by -c.
$>zd.exe -c demo/default.yaml -r # Generate 10 lines of data according to the config file specified by -c recursively.
$>zd.exe -d demo/default.yaml -c demo/test.yaml -n 100 # Using the parameter of -c and -d at the same time.
$>zd.exe -d demo\default.yaml # Generate 10 lines of data according to the config file specified by -d.
$>zd.exe -c demo\default.yaml # Generate 10 lines of data according to the config file specified by -c.
$>zd.exe -c demo\default.yaml -r # Generate 10 lines of data according to the config file specified by -c recursively.
$>zd.exe -d demo\default.yaml -c demo\test.yaml -n 100 # Using the parameter of -c and -d at the same time.
$>zd.exe -d demo/default.yaml -c demo/test.yaml -n 100 -o test.txt # Output data in original format.
$>zd.exe -d demo/default.yaml -c demo/test.yaml -n 100 -o test.json # Output data in JSON.
$>zd.exe -d demo/default.yaml -c demo/test.yaml -n 100 -o test.xml # Output data in XML.
$>zd.exe -d demo/default.yaml -n 100 -o test.sql -t user # Output the sql inserted into the table user.
$>zd.exe -d demo/default.yaml -o test.sql -t user -s mysql --trim # Remove the prefix and postfix of every field.
$>zd.exe -d demo\default.yaml -c demo\test.yaml -n 100 -o test.txt # Output data in original format.
$>zd.exe -d demo\default.yaml -c demo\test.yaml -n 100 -o test.json # Output data in JSON.
$>zd.exe -d demo\default.yaml -c demo\test.yaml -n 100 -o test.xml # Output data in XML.
$>zd.exe -d demo\default.yaml -n 100 -o test.sql -t user # Output the sql inserted into the table user.
$>zd.exe -d demo\default.yaml -o test.sql -t user -s mysql --trim # Remove the prefix and postfix of every field.
$>zd.exe -i demo/zentao.sql -o db # Generate YAML files for each table by parsing zentao.sql.
$>zd.exe -c demo/default.yaml -i test.txt --decode # Parse the file specified by -i according to the config of -d.
$>zd.exe -i demo\zentao.sql -o db # Generate YAML files for each table by parsing zentao.sql.
$>zd.exe -c demo\default.yaml -i test.txt --decode # Parse the file specified by -i according to the config of -d.
$>zd.exe -a demo\article.txt -o demo # Convert article to yaml config file in demo dir.
$>zd.exe -l # List all build-in data types.
$>zd.exe -v address.cn.v1 # View data types in build-in Excel file data/address/cn.v1.xlsx.
$>zd.exe -v address.cn.v1 # View data types in build-in Excel file data\address\cn.v1.xlsx.
$>zd.exe -v address.cn.v1.china # View data items in Excel sheet "china".
$>zd.exe -v ip.v1.yaml # View data in build-in instances defined in yaml/ip/v1.yaml。
$>zd.exe -v ip.v1.yaml # View data in build-in instances defined in yaml\ip\v1.yaml。
Service Example:
......@@ -60,6 +62,6 @@ $zd.exe -p 80 -R d:\zd\config # Listen port 80. Use d:\zd\config as th
Client Call:
$curl http://localhost:8848/?d=demo/default.yaml&c=demo/config.yaml&n=100 # Specify the server config file via GET.
$curl http://localhost:8848/?default=demo/default.yaml&output=test.sql&table=user # Parameter names can be full.
$curl -i -X POST http://localhost:8848?lines=3 -F default=@demo/default.yaml # The config can be uploaded via POST.
$curl http:\\localhost:8848\?d=demo\default.yaml&c=demo\config.yaml&n=100 # Specify the server config file via GET.
$curl http:\\localhost:8848\?default=demo\default.yaml&output=test.sql&table=user # Parameter names can be full.
$curl -i -X POST http:\\localhost:8848?lines=3 -F default=@demo\default.yaml # The config can be uploaded via POST.
......@@ -20,6 +20,7 @@ ZenData是一款通用的数据生成工具,您可以使用yaml文件来定义
-i --input 指定一个schema文件,输出每个表的yaml配置文件。需通过-o参数指定一个输出的目录。
-D --decode 根据指定的配置文件,将通过-i参数指定的数据文件解析成json格式。
-a --article 将指定的文件或目录下扩展名为.txt的文件,转换成文章yaml配置,输出到-o参数指定的目录下。
-l --list 列出所有支持的数据格式。
-v --view 查看某一个数据格式的详细定义。
......@@ -41,6 +42,7 @@ $>zd.exe -d demo\default.yaml -n 100 -o test.sql -t user --trim # 输出针
$>zd.exe -i demo\zentao.sql -o db # 根据sql的定义生成各表的yaml文件,存储到db目录里面。
$>zd.exe -c demo\default.yaml -i test.txt --decode # 将-i指定的文件根据-d参数的配置进行解析。
$>zd.exe -a demo\article.txt -o demo # 转换文章为yaml配置,输出到demo目录下。
$>zd.exe -l # 列出所有內置数据。
$>zd.exe -v address.cn.v1 # 查看內置Excel文件data/address/cn.v1.xlsx中的数据表。
......
package service
import (
"fmt"
"github.com/easysoft/zendata/src/model"
constant "github.com/easysoft/zendata/src/utils/const"
fileUtils "github.com/easysoft/zendata/src/utils/file"
stringUtils "github.com/easysoft/zendata/src/utils/string"
_ "github.com/mattn/go-sqlite3"
"gopkg.in/yaml.v3"
"path"
"path/filepath"
"strconv"
"strings"
)
const (
strLeft = "“"
strRight = "”"
expLeft = "("
expRight = ")"
table = "words.v1"
//src = "data/words"
//dist = "demo"
)
var (
compares = []string{"=", "!=", ">", "<"}
)
func ConvertArticle(src, dist string) {
files := make([]string, 0)
if !fileUtils.IsDir(src) {
pth, _ := filepath.Abs(src)
files = append(files, pth)
if dist == "" {
dist = path.Dir(pth)
}
} else {
fileUtils.GetFilesInDir(src, ".txt", &files)
if dist == "" {
dist = src
}
}
for _, filePath := range files {
article := fileUtils.ReadFile(filePath)
content := convertToYaml(article, filePath)
newPath := fileUtils.AddSepIfNeeded(dist) + fileUtils.ChangeFileExt(path.Base(filePath), ".yaml")
fileUtils.WriteFile(newPath, content)
}
}
func convertToYaml(article, filePath string) (content string) {
sections := parseSections(article)
conf := createDef(constant.ConfigTypeArticle, table, filePath)
prefix := ""
for index, section := range sections {
tye := section["type"]
val := section["val"]
if tye == "exp" {
fields := createFields(index, prefix, val)
conf.XFields = append(conf.XFields, fields...)
prefix = ""
} else {
prefix += val
}
}
bytes, _ := yaml.Marshal(&conf)
content = string(bytes)
// convert yaml format by using a map
m := make(map[string]interface{})
yaml.Unmarshal([]byte(content), &m)
bytes, _ = yaml.Marshal(&m)
content = string(bytes)
content = strings.Replace(content, "xfields", "\nfields", -1)
return
}
func createDef(typ, table, filePath string) (conf model.DefExport) {
conf.Title = "automation"
conf.Author = "ZenData"
conf.From = table
conf.Type = typ
conf.Desc = "Generated from article " + filePath
return
}
func createFields(index int, prefix, exp string) (fields []model.DefFieldExport) {
field := model.DefFieldExport{}
field.Field = strconv.Itoa(index)
field.Prefix = prefix
field.Rand = true
field.Limit = 1
// deal with exp like S:名词-姓+名词-名字=F
exp = strings.ToLower(strings.TrimSpace(exp))
expArr := []rune(exp)
if string(expArr[0]) == "s" && (string(expArr[1]) == ":" || string(expArr[1]) == ":") {
exp = string(expArr[2:])
expArr = expArr[2:]
field.UseLastSameValue = true
}
if strings.Index(exp, "=") == len(exp) - 2 {
exp = string(expArr[:len(expArr) - 2])
field.Select = stringUtils.GetPinyin(exp)
field.Where = fmt.Sprintf("%s = '%s'", field.Select, string(expArr[len(expArr) - 1]))
} else {
field.Select = stringUtils.GetPinyin(exp)
field.Where = "true"
//field.Where = getPinyin(exp) + " = 'y'"
}
if strings.Index(field.Select, "+") < 0 {
fields = append(fields, field)
} else if strings.Index(field.Select, "+") > 0 { // include more than one field, split to two
arr := strings.Split(field.Where, "=")
right := ""
if len(arr) > 1 {
right = arr[1]
}
items := strings.Split(field.Select, "+")
for _, item := range items {
var objClone interface{} = field
fieldClone := objClone.(model.DefFieldExport)
fieldClone.Select = item
if len(arr) > 1 { // has conditions
fieldClone.Where = item + " = " + right
}
fields = append(fields, fieldClone)
}
}
return
}
func parseSections(content string) (sections []map[string]string) {
strStart := false
expStart := false
content = strings.TrimSpace(content)
runeArr := []rune(content)
section := ""
for i := 0; i < len(runeArr); i++ {
item := runeArr[i]
str := string(item)
isCouple, duplicateStr := isCouple(i, runeArr)
if isCouple {
section += duplicateStr
i += 1
} else if strStart && str == strRight { // str close
addSection(section, "str", &sections)
strStart = false
section = ""
} else if expStart && str == expRight { // exp close
addSection(section, "exp", &sections)
expStart = false
section = ""
} else if !strStart && !expStart && str == strLeft { // str start
if section != "" && strings.TrimSpace(section) != "+" {
addSection(section, "str", &sections)
}
strStart = true
section = ""
} else if !strStart && !expStart && str == expLeft { // exp start
if section != "" && strings.TrimSpace(section) != "+" {
addSection(section, "str", &sections)
}
expStart = true
section = ""
} else {
section += str
}
}
return
}
func addSection(str, typ string, arr *[]map[string]string) {
mp := map[string]string{}
mp["type"] = typ
mp["val"] = str
*arr = append(*arr, mp)
}
func isCouple(i int, arr []rune) (isCouple bool, duplicateStr string) {
if string(arr[i]) == strLeft && (i + 1 < len(arr) && string(arr[i + 1]) == strLeft) {
isCouple = true
duplicateStr = string(arr[i])
} else if string(arr[i]) == strRight && (i + 1 < len(arr) && string(arr[i + 1]) == strRight) {
isCouple = true
duplicateStr = string(arr[i])
} else if string(arr[i]) == expLeft && (i + 1 < len(arr) && string(arr[i + 1]) == expLeft) {
isCouple = true
duplicateStr = string(arr[i])
} else if string(arr[i]) == expRight && (i + 1 < len(arr) && string(arr[i + 1]) == expRight) {
isCouple = true
duplicateStr = string(arr[i])
}
return
}
......@@ -302,4 +302,42 @@ func GetFileName(filePath string) string {
fileName = strings.TrimSuffix(fileName, path.Ext(filePath))
return fileName
}
func GetFilesInDir(folder, ext string, files *[]string) {
folder, _ = filepath.Abs(folder)
if !IsDir(folder) {
if path.Ext(folder) == ext {
*files = append(*files, folder)
}
return
}
dir, err := ioutil.ReadDir(folder)
if err != nil {
return
}
for _, fi := range dir {
name := fi.Name()
if commonUtils.IngoreFile(name) {
continue
}
filePath := AddSepIfNeeded(folder) + name
if fi.IsDir() {
GetFilesInDir(filePath, ext, files)
} else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ext {
*files = append(*files, filePath)
}
}
}
func ChangeFileExt(filePath, ext string) string {
ret := strings.TrimSuffix(filePath, path.Ext(filePath))
ret += ext
return ret
}
\ No newline at end of file
......@@ -38,6 +38,8 @@ var (
format = constant.FormatText
decode bool
article string
listRes bool
viewRes string
viewDetail string
......@@ -96,6 +98,9 @@ func main() {
flagSet.BoolVar(&decode, "D", false, "")
flagSet.BoolVar(&decode, "decode", false, "")
flagSet.StringVar(&article, "a", "", "")
flagSet.StringVar(&article, "article", "", "")
flagSet.StringVar(&vari.Ip, "b", "", "")
flagSet.StringVar(&vari.Ip, "bind", "", "")
flagSet.IntVar(&vari.Port, "p", 0, "")
......@@ -149,6 +154,9 @@ func main() {
} else if decode {
gen.Decode(defaultFile, configFile, fields, input, output)
return
} else if article != "" {
service.ConvertArticle(article, output)
return
}
if vari.Ip != "" || vari.Port != 0 {
......
package main
import (
"github.com/Chain-Zhang/pinyin"
commonUtils "github.com/easysoft/zendata/src/utils/common"
fileUtils "github.com/easysoft/zendata/src/utils/file"
_ "github.com/mattn/go-sqlite3"
"io/ioutil"
"path"
"path/filepath"
"strings"
)
func getFilesInDir(folder, ext string, files *[]string) {
folder, _ = filepath.Abs(folder)
if !fileUtils.IsDir(folder) {
if path.Ext(folder) == ext {
*files = append(*files, folder)
}
return
}
dir, err := ioutil.ReadDir(folder)
if err != nil {
return
}
for _, fi := range dir {
name := fi.Name()
if commonUtils.IngoreFile(name) {
continue
}
filePath := fileUtils.AddSepIfNeeded(folder) + name
if fi.IsDir() {
getFilesInDir(filePath, ext, files)
} else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ext {
*files = append(*files, filePath)
}
}
}
func getFileName(filePath string) string {
fileName := path.Base(filePath)
fileName = strings.TrimSuffix(fileName, path.Ext(filePath))
return fileName
}
func changeFileExt(filePath, ext string) string {
ret := strings.TrimSuffix(filePath, path.Ext(filePath))
ret += ext
return ret