diff --git a/conf/app.ini b/conf/app.ini index d45c3fdc52ced49b810e9ffff0515af506ee50b1..6fa7962a441544bfd37d8e4ace582980306f04c0 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -1,6 +1,6 @@ [mysql] -#url=root:Meta.123@tcp(mysql.io:3306)/foot?charset=utf8 -url=root:awe.FOG5188@tcp(39.108.212.245:3306)/foot?charset=utf8 +url=root:Meta.123@tcp(mysql.io:3306)/foot?charset=utf8 +#url=root:awe.FOG5188@tcp(39.108.212.245:3306)/foot?charset=utf8 maxIdle=10 maxConn=50 [cookies] diff --git a/foot-spider/launch/FS101PortalApplication.go b/foot-spider/launch/FS101PortalApplication.go index ae6234171a8dc817640e51b2d6967e124fee085c..0d1d19d1d70384346ee7cf17d78a0167a6d99afe 100644 --- a/foot-spider/launch/FS101PortalApplication.go +++ b/foot-spider/launch/FS101PortalApplication.go @@ -79,11 +79,11 @@ func Spider_History() { Spider_match_his(v) Spider_euroLast_his(v) time.Sleep(28 * time.Minute) - Spider_euroHis_his(v) + Spider_asiaLastNew_his(v) time.Sleep(28 * time.Minute) Spider_baseFace_his(v) time.Sleep(28 * time.Minute) - Spider_asiaLastNew_his(v) + Spider_euroHis_his(v) time.Sleep(28 * time.Minute) mysql.ShowSQL(false) } diff --git a/foot-spider/module/win007/proc/AsiaLastNewProcesser.go b/foot-spider/module/win007/proc/AsiaLastNewProcesser.go index c04b0665ee9d1ea6d7a2cc796736fc0f6eafd2e4..64fddc67a6bfbfcf540877c0074e726f9aa49634 100644 --- a/foot-spider/module/win007/proc/AsiaLastNewProcesser.go +++ b/foot-spider/module/win007/proc/AsiaLastNewProcesser.go @@ -50,6 +50,12 @@ func (this *AsiaLastNewProcesser) Startup() { for i, v := range this.MatchLastList { if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider + //先将前面的spider启动 + newSpider.SetDownloader(down.NewMAsiaLastApiDownloader()) + newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) + newSpider.SetSleepTime("rand", 1, 300) + newSpider.SetThreadnum(1).Run() + processer = GetAsiaLastNewProcesser() processer.Setup(this) newSpider = spider.NewSpider(processer, "AsiaLastNewProcesser"+strconv.Itoa(i)) @@ -65,12 +71,6 @@ func (this *AsiaLastNewProcesser) Startup() { url := strings.Replace(win007.WIN007_ASIAODD_NEW_URL_PATTERN, "${matchId}", win007_id, 1) newSpider = newSpider.AddUrl(url, "json") - if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider - newSpider.SetDownloader(down.NewMAsiaLastApiDownloader()) - newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) - newSpider.SetSleepTime("rand", 1, 300) - newSpider.SetThreadnum(1).Run() - } } newSpider.SetDownloader(down.NewMAsiaLastApiDownloader()) diff --git a/foot-spider/module/win007/proc/BaseFaceProcesser.go b/foot-spider/module/win007/proc/BaseFaceProcesser.go index 7078922ce20a29e3e13b7ecfb631c3575ecd1eb8..a70a4f6be06b95acab5e3d2d129b1e62f103744a 100644 --- a/foot-spider/module/win007/proc/BaseFaceProcesser.go +++ b/foot-spider/module/win007/proc/BaseFaceProcesser.go @@ -55,6 +55,12 @@ func (this *BaseFaceProcesser) Startup() { for i, v := range this.MatchLastList { if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider + //先将前面的spider启动 + newSpider.SetDownloader(down.NewMWin007Downloader()) + newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) + newSpider.SetSleepTime("rand", 1, 300) + newSpider.SetThreadnum(1).Run() + processer = GetBaseFaceProcesser() processer.Setup(this) newSpider = spider.NewSpider(processer, "BaseFaceProcesser"+strconv.Itoa(i)) @@ -70,12 +76,6 @@ func (this *BaseFaceProcesser) Startup() { url := strings.Replace(win007.WIN007_BASE_FACE_URL_PATTERN, "${matchId}", win007_id, 1) newSpider = newSpider.AddUrl(url, "html") - if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider - newSpider.SetDownloader(down.NewMWin007Downloader()) - newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) - newSpider.SetSleepTime("rand", 1, 300) - newSpider.SetThreadnum(1).Run() - } } newSpider.SetDownloader(down.NewMWin007Downloader()) diff --git a/foot-spider/module/win007/proc/EuroLastProcesser.go b/foot-spider/module/win007/proc/EuroLastProcesser.go index 3d35255ecdefe88af104896e3a1b37f74c0423bd..4103ab9e6320ab70e9ff3b87dfbcb56f8835d2c5 100644 --- a/foot-spider/module/win007/proc/EuroLastProcesser.go +++ b/foot-spider/module/win007/proc/EuroLastProcesser.go @@ -57,6 +57,12 @@ func (this *EuroLastProcesser) Startup() { for i, v := range this.MatchLastList { if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider + //先将前面的spider启动 + newSpider.SetDownloader(down.NewMWin007Downloader()) + newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) + newSpider.SetSleepTime("rand", 1, 300) + newSpider.SetThreadnum(1).Run() + processer = GetEuroLastProcesser() processer.Setup(this) newSpider = spider.NewSpider(processer, "EuroLastProcesser"+strconv.Itoa(i)) @@ -72,12 +78,6 @@ func (this *EuroLastProcesser) Startup() { url := strings.Replace(win007.WIN007_EUROODD_URL_PATTERN, "${matchId}", win007_id, 1) newSpider = newSpider.AddUrl(url, "html") - if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider - newSpider.SetDownloader(down.NewMWin007Downloader()) - newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) - newSpider.SetSleepTime("rand", 1, 300) - newSpider.SetThreadnum(1).Run() - } } newSpider.SetDownloader(down.NewMWin007Downloader()) diff --git a/foot-spider/module/win007/proc/EuroTrackProcesser.go b/foot-spider/module/win007/proc/EuroTrackProcesser.go index 84ddccdb4a00d1d99dc00015fe909514ea1418ca..6a8b3060b95a883457678b86ef94b3f4b289d271 100644 --- a/foot-spider/module/win007/proc/EuroTrackProcesser.go +++ b/foot-spider/module/win007/proc/EuroTrackProcesser.go @@ -56,6 +56,12 @@ func (this *EuroTrackProcesser) Startup() { for i, v := range this.MatchLastList { if !this.SingleThread &&i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider + //先将前面的spider启动 + newSpider.SetDownloader(down.NewMWin007Downloader()) + newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) + newSpider.SetSleepTime("rand", 1, 300) + newSpider.SetThreadnum(1).Run() + processer = GetEuroTrackProcesser() processer.Setup(this) newSpider = spider.NewSpider(processer, "EuroTrackProcesser"+strconv.Itoa(i)) @@ -73,12 +79,6 @@ func (this *EuroTrackProcesser) Startup() { url := strings.Replace(base_url, "${cId}", v, 1) newSpider = newSpider.AddUrl(url, "html") } - if !this.SingleThread && i%1000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider - newSpider.SetDownloader(down.NewMWin007Downloader()) - newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) - newSpider.SetSleepTime("rand", 1, 300) - newSpider.SetThreadnum(1).Run() - } } newSpider.SetDownloader(down.NewMWin007Downloader()) diff --git a/foot-spider/module/win007/proc/MatchHisProcesser.go b/foot-spider/module/win007/proc/MatchHisProcesser.go index a0aeec2bdc27de07860fc65c25f2f4f684d8070e..899585e1a4ad3c9eeeb9c4741b059a4c211de20f 100644 --- a/foot-spider/module/win007/proc/MatchHisProcesser.go +++ b/foot-spider/module/win007/proc/MatchHisProcesser.go @@ -67,6 +67,12 @@ func (this *MatchHisProcesser) Startup() { for i, v := range seasonList { if i%10 == 0 { //10个联赛一个spider,总数1000多个联赛,最多100spider + //先将前面的spider启动 + newSpider.SetDownloader(down.NewMWin007Downloader()) + newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) + newSpider.SetSleepTime("rand", 1, 300) + newSpider.SetThreadnum(1).Run() + processer = GetMatchHisProcesser() processer.Setup(this) newSpider = spider.NewSpider(processer, "MatchHisProcesser"+strconv.Itoa(i)) @@ -84,12 +90,6 @@ func (this *MatchHisProcesser) Startup() { processer.SUrl_Season[round_url] = v newSpider = newSpider.AddUrl(round_url, "html") } - if i%10 == 0 { //10个联赛一个spider,总数1000多个联赛,最多100spider - newSpider.SetDownloader(down.NewMWin007Downloader()) - newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole()) - newSpider.SetSleepTime("rand", 1, 300) - newSpider.SetThreadnum(1).Run() - } } newSpider.SetDownloader(down.NewMWin007Downloader())