From 73e69b2721bed7a74272222a7a15eb77516a828f Mon Sep 17 00:00:00 2001 From: aaron <462826@qq.com> Date: Thu, 10 Sep 2020 18:28:30 +0800 Subject: [PATCH] close task#7786 --- go.mod | 1 + src/model/definition.go | 16 ++++ test/article/common.go | 2 +- test/article/convert_test.go | 14 ++-- test/article/generate_test.go | 137 +++++++++++++++++++++++++++++++++- tmp/cache/.words.db | Bin 1118208 -> 1118208 bytes 6 files changed, 161 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 6544044a..b2e57288 100644 --- a/go.mod +++ b/go.mod @@ -24,5 +24,6 @@ require ( golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect golang.org/x/sys v0.0.0-20200819171115-d785dc25833f // indirect gopkg.in/ini.v1 v1.60.0 + gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/src/model/definition.go b/src/model/definition.go index 76ddc87d..fa92e073 100644 --- a/src/model/definition.go +++ b/src/model/definition.go @@ -107,4 +107,20 @@ func (def *DefSimple) Init(tableName, author, desc, version string) { } func (fld *FieldSimple) Init(field string) { fld.Field = field +} + + +type DefExport struct { + ClsBase `yaml:",inline"` + XFields []DefFieldExport `yaml:"xfields,flow"` // control orders +} +type DefFieldExport struct { + Field string `yaml:"field"` + Prefix string `yaml:"prefix"` + Postfix string `yaml:"postfix"` + + Select string `yaml:"select"` + Where string `yaml:"where"` + Rand bool `yaml:"rand"` + Limit int `yaml:"limit"` } \ No newline at end of file diff --git a/test/article/common.go b/test/article/common.go index cab80edb..fd07a168 100644 --- a/test/article/common.go +++ b/test/article/common.go @@ -36,7 +36,7 @@ func getFilesInDir(folder, ext string, files *[]string) { filePath := fileUtils.AddSepIfNeeded(folder) + name if fi.IsDir() { getFilesInDir(filePath, ext, files) - } else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ".xlsx" { + } else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ext { *files = append(*files, filePath) } } diff --git a/test/article/convert_test.go b/test/article/convert_test.go index 6209901b..7f82f9dd 100644 --- a/test/article/convert_test.go +++ b/test/article/convert_test.go @@ -6,8 +6,8 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" constant "github.com/easysoft/zendata/src/utils/const" i118Utils "github.com/easysoft/zendata/src/utils/i118" - logUtils "github.com/easysoft/zendata/src/utils/log" _ "github.com/mattn/go-sqlite3" + "log" "strconv" "strings" "testing" @@ -15,7 +15,7 @@ import ( func TestImportSqlite(t *testing.T) { files := make([]string, 0) - getFilesInDir("xdoc/words-9.3", "xlsx", &files) + getFilesInDir("xdoc/words-9.3", ".xlsx", &files) tableName := "words" seq := 1 @@ -33,7 +33,7 @@ func TestImportSqlite(t *testing.T) { dropSql := `DROP TABLE IF EXISTS ` + tableName + `;` _, err = db.Exec(dropSql) if err != nil { - logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error())) + log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error())) return } @@ -44,14 +44,14 @@ func TestImportSqlite(t *testing.T) { ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n")) _, err = db.Exec(ddlSql) if err != nil { - logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error())) + log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error())) return } sql := strings.Join(insertSqls, "\n") _, err = db.Exec(sql) if err != nil { - logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error())) + log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error())) return } } @@ -59,7 +59,7 @@ func TestImportSqlite(t *testing.T) { func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) { excel, err := excelize.OpenFile(filePath) if err != nil { - logUtils.PrintTo("fail to read file " + filePath + ", error: " + err.Error()) + log.Println("fail to read file " + filePath + ", error: " + err.Error()) return } @@ -90,7 +90,7 @@ func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[] colName = "ci" } if colName != "ci" { - colName = colPrefix + ":" + colName + colName = colPrefix + "-" + colName } if (*colMap)[colName] == false { diff --git a/test/article/generate_test.go b/test/article/generate_test.go index dc125a9d..bebcca90 100644 --- a/test/article/generate_test.go +++ b/test/article/generate_test.go @@ -1,12 +1,25 @@ package main import ( + "github.com/easysoft/zendata/src/model" + constant "github.com/easysoft/zendata/src/utils/const" fileUtils "github.com/easysoft/zendata/src/utils/file" _ "github.com/mattn/go-sqlite3" + "gopkg.in/yaml.v3" + "strconv" + "strings" "testing" ) -func TestGenerate(t *testing.T) { +const ( + strLeft = "“" + strRight = "”" + + expLeft = "（" + expRight = "）" +) + +func TestGenerate(ts *testing.T) { files := make([]string, 0) getFilesInDir("xdoc/words-9.3", ".txt", &files) @@ -20,7 +33,129 @@ func TestGenerate(t *testing.T) { } func convertToYaml(article string) (content string) { + sections := parseSections(article) + + conf := createDef(constant.ConfigTypeArticle, "words.v1") + + prefix := "" + for index, section := range sections { + tye := section["type"] + val := section["val"] + if tye == "exp" { + field := createField(index, prefix, val) + conf.XFields = append(conf.XFields, field) + prefix = "" + } else { + prefix += val + } + } + + bytes, _ := yaml.Marshal(&conf) + content = string(bytes) + + // convert yaml format by using a map + m := make(map[string]interface{}) + yaml.Unmarshal([]byte(content), &m) + bytes, _ = yaml.Marshal(&m) + content = string(bytes) + content = strings.Replace(content, "xfields", "\nfields", -1) + + return +} + +func createDef(typ, table string) (conf model.DefExport) { + conf.Title = "automation" + conf.Author = "zendata" + conf.From = table + conf.Type = typ + conf.Desc = "generated from article text automatically" + + return +} + +func createField(index int, prefix, exp string) (field model.DefFieldExport) { + field.Field = strconv.Itoa(index) + field.Prefix = prefix + + field.Select = getPinyin(exp) + field.Where = "true" + field.Rand = true + field.Limit = 1 + + return +} + +func parseSections(content string) (sections []map[string]string) { + strStart := false + expStart := false + + content = strings.TrimSpace(content) + runeArr := []rune(content) + + section := "" + for i := 0; i < len(runeArr); i++ { + item := runeArr[i] + str := string(item) + + isCouple, duplicateStr := isCouple(i, runeArr) + if isCouple { + section += duplicateStr + i += 1 + } else if strStart && str == strRight { // str close + addSection(section, "str", §ions) + + strStart = false + section = "" + } else if expStart && str == expRight { // exp close + addSection(section, "exp", §ions) + + expStart = false + section = "" + } else if !strStart && !expStart && str == strLeft { // str start + if section != "" && strings.TrimSpace(section) != "+" { + addSection(section, "str", §ions) + } + + strStart = true + section = "" + } else if !strStart && !expStart && str == expLeft { // exp start + if section != "" && strings.TrimSpace(section) != "+" { + addSection(section, "str", §ions) + } + + expStart = true + section = "" + } else { + section += str + } + } + + return +} + +func addSection(str, typ string, arr *[]map[string]string) { + mp := map[string]string{} + mp["type"] = typ + mp["val"] = str + + *arr = append(*arr, mp) +} + +func isCouple(i int, arr []rune) (isCouple bool, duplicateStr string) { + if string(arr[i]) == strLeft && (i + 1 < len(arr) && string(arr[i + 1]) == strLeft) { + isCouple = true + duplicateStr = string(arr[i]) + } else if string(arr[i]) == strRight && (i + 1 < len(arr) && string(arr[i + 1]) == strRight) { + isCouple = true + duplicateStr = string(arr[i]) + } else if string(arr[i]) == expLeft && (i + 1 < len(arr) && string(arr[i + 1]) == expLeft) { + isCouple = true + duplicateStr = string(arr[i]) + } else if string(arr[i]) == expRight && (i + 1 < len(arr) && string(arr[i + 1]) == expRight) { + isCouple = true + duplicateStr = string(arr[i]) + } return } \ No newline at end of file diff --git a/tmp/cache/.words.db b/tmp/cache/.words.db index 8b36fb239e78b6ee345a5908e90e2c2f1cf15b23..c967077fdca15a94e9b1ca9f28dc253e1dfde0e0 100644 GIT binary patch delta 5490 zcmZ9Q33yc1702f#civ1U%M41{A%pCYkwH;#!--<03L1Cn0uv#?KncSVR>MpnWDl9- zC3y)UWI~|nrj62&+OOKNwYF|`?Cw6bU-znG_qx#j&KqL=%J=>LcR%;s^X@y$OEkVb z8eblpJmx){O&epg***s@Fj|W))!u6jKCk(VBJCpdR!etiGpU=}Ec6L&s`BmMRAD5v zNe*AQG_;O-R2yfmX;c%8aF$*7?F;QGD>n{ijgzK>$)TN_LZfX=?9LL?-C16TZ$oMM zp0bb;&zeZjt67VbZwpKP*;8ryvc2dnmTt3pnzIY(iK>alGuczl*FMUYrP8-hP0rLz z=XN#TzM*1WNCqxFKm6h+!z)@oH9XN+VxM$=$`<>13r5^79z1DZYUw2NUpv%zBkj~_ zF4PyIYxVhPgFa95nQN*Qg`qje8gYli__r>8XId^}sY9%fj+HyaZl)#C)SQ!ylp`~L z$@!60PVqwKR66C9QhUTH>t1lOso7s+Z8tqfJXo0{9(Ze&H2QNcqfu3>7SYIYEuisM zW+W&Lqsdi-K5h-P)G2wWu3TCFW}Uj2BEMeAgqYP=S+B&yRPG!`p08JPX)Ma~I(_>p zwugM%L*}FPYGPidwXw8(S82KNe(pHtu5M5~U-LkNnn6*zLw&~Z7w8V()=>Gz{UM_` ze>~l9k(sr+QJgd@8x=7(A8u5$abirtEZw($8}p1Y z?g@q`cdFPo1NCZ}RG7H7$vWFp1vAWdcB?7pd!K8v{CX!-c$%eUgx#{ivk_G&vo4g& zy00wgy}S7K>sJ)t&O6a>yy#x0-E}6@KCs&{aCoeA#!8+0VwBqu-C}{ZmnUKX0$5;(O5S_Y!>?BF){aicP08TO^5qWMc(6mF^2H8`%IOX{ z#;JCq60r@mOIS{}OIYIV5|+O9OVAVTg=lYk9(vr6$I)YKK#v)Bpxs6Q?K0%Vorava z-H@wlw8>R%Ym@l3w#ij(X_KpZv~4ch-1Zr?sVxg_Y~6-N8d~qCsBiroTGz^5wbiyx zM{8Q;9vq3u#)o3EaaBwfJR zJd_R1@=(f}o#^@|Iq|wCIq`!{a^lh^Iq_9Za^m79Iq`K(a^mY7<-~V4%7Irl%7O1| zjL3DkwNb9a>P9!Zs^KAYN`q`Xx8Y9of`$NkxIsd_y+Ka={RX*Q--hk`Mef3SDqSF5o3c}}#EX9ENFM5~-YfV2j_UO^7FWxC{%rNt=)Ki*(JQORqW4!Lwkr;Q0bO)h zF6o@Z%h78N`_Wm4r=a5wJJG3!cA-lT1<~b)SFC9r-(=_ocEsni#=7Ykw-P*)6Z!DaX8dUx~ z&xi`=XL&~Sq5KcrX9Jq@45}eF5|yE7{zvY!F->PYtYS3bH)II_(h^%d=(`Fe@e!$Dkf9V1Cdo^S!KBseGiwg9MqQ0#X~Y{PBoCL62A} z!RVk~BJ0DJITq@JX8}p;4AP1{1X05xG8Ls zP-OmN@Fsm8;I-89K^{&nxF833_L>8@#2T4EQ$P;Xc;zCNvHxJ?9DEEP!H4hx{1g5G ze}})pU*RwCKD-C-!aIOdwm-oi;Scb8_#ONfegnUTx8YZig16uecwKJyYgAr^S3pkq z3-~$w41Nkffgi)m@Dlt8UWDi2Id~R+2tR;l;A!|Cd>5XA1bhdc1c~Ce;al)He3Sic zBXAam;Tv!U9)%(JIy?do!`I-eAdlnAFbH3S({Ks~;3UMM4^BWY9ETn_2HnsFCUioi z9WkH{TA>AwLNhc$BQ!ug)IlxOz!8u;atNv*3UZiAcnA)_e%J@{p6-F&PyxGOC+v_V zVJh389Jaz1*bE`q1RG%kl)-vf2M4mb?k z#pUl)84nl1op3WOfwgcCd>(@E0Ne(*Kp`xH3t<9W3Tt32Oon-IHB5&YFbyWcW#ESe za0$$okj;df;7j0zJ76(<7Vd>B;eNOR7Qr0w!Ci0-%z|++6_&zsxEyYPYoP>+U_LB_ zi{W-&KQ~MqLnBE=qfEn_@B%zG5~x;B=Z>g28EQmNM4Vg&9-990xv@ z9bz$ce;7VotFu<7sO=Y>fwHs6-?n zh6tn9Ou)+m+vFRnyGqc69-xo)&9vF@>?v{z>=id(~Vsx|83oarHySPJCR(zIM2mtz$s`FB1u?pWLSjTgc;xQ^gh#?l zc%&uap&&BSDUqQdD6)W{$Z~=rErL??|3MK42#Sw_pvWH(Bq$WbV~!*9iC$sGDwX_I zpsG^M%Jw~2Ar-IrZl#hB^<<@*>GZAJR93!mbI2@?s43ZU{U#rjKRuYm2h|MSx3{9S ze7`m3U~F@!bf-D}pyCff=EICryi8&Dt>X?S@{X9L}g zEt|^*aS@L+OQhzPi7^}?8N0n4p8pVdK$ZhuIe|Z0I!de=F^-^ySf&yl*^wab6m7b9n>KCJ-E?5x zR=l=2iuO^v?Wi#geaM(>=3KjMO=Q@ZkeU-*n7fE}$QWzn zL>E+=<08SN6f0+a?z$x_BR3?C)nh8tbFW&K`@d(5tWA>fmLyMV&eDY|*DcA7#FEC- z^G?z%GiNzVy~&e#+&|fao@VPRyQeyN3OylneB`O*Nq@JtCu^ztW||X6TT$ng=D3ul zYZm2dVD{g`TmBiIU;BT<M#|9j<)+ItIoSD;Ev`An-J3GqC#UDV(6^kf5BknS z6Me^W2S(pa+9Q1vbsv4HsH^vCdd}=!h#t^;8alK02y}Yy-sp_pT~L3ofgatn5>nV>&v!V=TIB zM>4u|dk~$|ehE6I{Ur3X_CwJ#+NYwIv@4?nZ8xA7wCSYJYdarZ&~_YpXxpqjeok&v z#Ammqp?kEhLC3Y~sGn>-6}_@maTK;rLmOLVZl{*((S2HUfRkFzK~HU&i=N-IKYDu0 z1oWa73q7S-r*dxdrRWLGa_-pXL(pTIbsck?T@Lm?BC7jX7?rupqPmVnQC-L4sPbS* zbP~EWD(5b5T8l1gT7>2{`OzzyjzX6=?TfBx(y6R$(y6R!)Tsm;btl&t@8yYgv>m$03yvR~C6gdmckND7n zh$1eEDB|J>sp=@HSE`oQ>-@^rO=L z>XfVXb-SaHI^{t_SQ|HmwQ)48jhn+qqb=bXXlr;p+7?bh+iP{y9kmP4&RQj4SFJ8C zR;vW;sZ|2@)*_C+nic5A8YSS3c{NJF{u(9V%{AKimYQtz*6Qof+p3qLw^!?q@2EZo zy|Y@EGEl8cxvScVZmLqm_f#q3!74?(xk?caRVm_os}%8lRf_ojN=5u&r2>AaQUO0) znWuDkq*CcHTV!2ZFrE=Zz%VkQ{SITnHSIZQ}Yh`oM*UMz>jj{>kpX1FknR}~L=H4z1pzoCG z0N*V=41KRuDf51*8~w0E&c#dQ+(#vHZfl7T{kTNe@kxp5!KWoA`dP6O@bhBD`bF`4 z^vhzM@mIwj^y}hG^qZngrrE#~37BIWbX64yw`u-yGtkY_8Dxx>|O4!9W_$xfm-kwtIJrQTS! zR%z+5mik>94`a{rxtQd4?{E#7;mjnN$yR&7w&2lV0t@)z*5Gi4+QBR>ce}J)HoThV zv)2t{d84I~1ddq#EX^NgEweP$t=<8w^qXOM5@KkhCHLJK+gzIy$rjC#j2$!k%@I2i zwxRrJ{kPKS$t^md#Js$R!}YSUa$)T%xU;WAjn{tlAZ zu>_XF<**D?5v~9g{}r$jRzVQt#A;AM&^xsT*1|gAt#GV|YvDTB0ID(CCj|LW01BcA z@`{lXD1|a8hYF~KDyW7UsD(PHhX^!)PN4~+&Dw_jKItA3cL!h!Rzn_ya{i?+wcy&3-7`E@FB$EBiITb!++ot z_!K^a&*2O9cYFz7!PoE&d<);f_wWP!2tUEk@C*D3zrpXY4gP@b@F(nmzj$r=F_It| zOh^F>Qo#vnkPhQBkW9#eY;b`a#tiw2%qKGX%>Z@D^x4JAlSvr~+7&WRSv382tqU7= zd9r7O)0r_~S4*!7Wzep+EHguP4GOTB8fmxVc1iSNqu=t_70s@5ev0Oh6|mc}QKK1j zghq>IhCOJv{j{l_8LEmBPlkKYvPTl>niWr{77kb{?IY>Puw+0E#?nb+8Xjm+-X#5 z&51j;+NssUPGmq5b21Wk%9)UpLY;7$*a$jhBcL(AJ?5ih$QriK(oK8Na;CV4G7_o3 zE38zO45zYsAXTelsahRQMM9Dw<5nPbo2zHUz1eTO?^C~5;*q!|33(RQH)+JO@7L0v zF$=H5_L+t>A*e4+fC>GkpJ&s@v*}CGRIk0vV=r@CDel1x>kk(el6SZ^nx+@W?7Q@* z$ohb3TesVB+E^wcYuL004Gjj9w2Rwt51Ltokl}Iu&gYEa_-fiI88Fz}on-HgA->Ys zF=IM|K(b86Wzsu(y0(i6d%t1Y8*z|G*^UF0oe_J5ho%)Yh}pw8l{~U7=sCt;1|qlj z39+R)z=A;wABRkj?ZXJKMZf9Q_t|F~OUSdBJ;QBx_~n#W62?g@XeVKUV7Z4B4JB&} zlimELeH+?6%=xh!K69VRIBrj}&nD>d5H!!G&-TDS`eND!gPP{GkJE2&9+&?