zjj update

README.md

@@ -28,7 +28,7 @@ issue分析员注意事项
     |    影响的版本     | impact version | [IV][/IV] |
     |    规避方案和措施     | Circumvention plan or mitigation measures | [CPMM][/CPMM] |
     |    影响的包    | impact wrap| [IW][/IW]
-- 分析issue填写模板（ps 每一次分析评论必须用/analysis指令 填写项可以一次填完也可不填写完 填写完成加上/end指令 
+- 分析issue填写模板（ps 每一次分析评论必须用/analysis指令 填写项可以一次填完也可不填写完 填写完成加上/done指令 
  /analysis
  影响分析说明:[IAD]此处为填写影响分析说明的内容[/IAD]
  原理分析:[PA]此处为填写原理分析的内容[/PA]
@@ -37,7 +37,7 @@ issue分析员注意事项
  影响的版本:[IV]此处为填写影响版本的内容[/IV]
  规避方案和措施:[IV]此处为填写规避方案和措施的内容[/IV]
  影响的包:[IW]此处为填写影响的包的内容(内容以英文逗号分隔)[/IW]
- /end
+ /done
 
 
 

conf/app.conf

@@ -37,7 +37,7 @@ maxlines=10000
 maxsize=1024000
 
 [crontab]
-ymalflag = 1
+ymalflag = 2
 cveflag = 2
 oricveflag = 2
 getymal = 00 00 23 * * *
@@ -84,4 +84,4 @@ cveref = https://nvd.nist.gov/vuln/detail/
 openeulernum = 3000
 
 [reflink]
-comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/README.md
+comment_cmd = https://gitee.com/zhangjianjun_code/cvevulner/blob/dev/README.md

controllers/hook.go

@@ -21,7 +21,18 @@ var (
 	NoteHookType    = "Note Hook"                 // type of comment
 	PullReqHookType = "merge_request_hooks"       // type of pull request
 	PushTagHookType = "push_hooks/tag_push_hooks" // type of push or tag
-	IssueHookType   = "issue_hooks"               //type of issue
+	IssueHookType   = "Issue Hook"               //type of issue
+)
+
+const (
+	CommentAnalysisCplTpl ="@%v %v"
+	ReviewPrivateLettersTpl = `%s(%s)analysis is over,CVEScore:%v;OpenEulerScore:%v。Please review!`
+	ReviewRejectScore = `@%v you submit issue score audit failed(reject by %v)，Please re-analyze and submit！`
+	CommentReviewTpl = `@%v The CVE score needs to be reviewed (the review instruction /approve&/reject means agreement and rejection)。`
+	IssueRejectState = "rejected"
+    IssueCloseState = "closed"
+    IssueProgressState = "progressing"
+    IssueOpenState = "open"
 )
 
 type HookEventControllers struct {
@@ -99,9 +110,69 @@ func (c *HookEventControllers) handlePushTag() {
 }
 
 func (c *HookEventControllers) handleIssue() {
+	logs.Info(string(c.Ctx.Input.RequestBody))
+    issueHook := models.IssuePayload{}
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &issueHook)
+	if err != nil{
+		logs.Error(err)
+		return
+	}
+	if issueHook.Action == "state_change"{
+		//handle issue state change
+		err = handleIssueStateChange(&issueHook)
+		if err != nil {
+			logs.Error(err)
+		}
+	}
 
 }
 
+func handleIssueStateChange(issueHook *models.IssuePayload) error{
+	issueTmp := models.IssueTemplate{IssueNum: issueHook.Iid}
+	err := models.GetIssueTemplateByColName(&issueTmp, "issue_num")
+	if err != nil {
+		return err
+	}
+	switch issueHook.State {
+	case IssueOpenState:
+		issueTmp.Status = 1
+	case IssueProgressState:
+		issueTmp.Status = 2
+	case IssueCloseState:
+		issueTmp.Status = 3
+		if isNormalCloseIssue(issueTmp.CveId,issueTmp.IssueStatus){
+			issueTmp.IssueStatus = 2
+		}else {
+			issueTmp.IssueStatus = 6
+		}
+	case IssueRejectState:
+		issueTmp.Status = 4
+		if isNormalCloseIssue(issueTmp.CveId,issueTmp.IssueStatus){
+			issueTmp.IssueStatus = 2
+		}else {
+			issueTmp.IssueStatus = 6
+		}
+	}
+	issueTmp.StatusName = issueHook.Issue.StateName
+	err = models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", "status_name")
+	return err
+}
+
+func isNormalCloseIssue(cveId int64,issueState int8) bool {
+	if issueState == 1{
+		return false
+	}
+	score, err := models.QueryIssueScore(cveId)
+	if err != nil {
+		logs.Error(err)
+		return false
+	}
+	if score.Ostatus != 3 {
+		return false
+	}
+	return true
+}
+
 func handleIssueComment(payload models.CommentPayload) {
 	if payload.Issue == nil || payload.Comment == nil {
 		return
@@ -118,15 +189,50 @@ func handleIssueComment(payload models.CommentPayload) {
 		return
 	}
 	if strings.HasPrefix(cBody, cmdRej) {
-		//审核拒绝
+		//审核拒绝 添加评论 @分析人
+        if !isReviewer(cuAccount){
+			return
+		}
+		issueTmp := models.IssueTemplate{IssueNum: issueNum}
+		err := models.GetIssueTemplateByColName(&issueTmp, "issue_num")
+		if err != nil {
+			logs.Error(err)
+			return
+		}
+		err = changeOpenEulerScoreStatus(issueTmp.CveId, 2)
+		if err != nil {
+			logs.Error(err)
+		}
+		accessToken := os.Getenv("issueaccesstoken")
+		owner := beego.AppConfig.String("gitee::owner")
+		path := beego.AppConfig.String("gitee::path")
+		taskhandler.AddCommentToIssue(fmt.Sprintf(ReviewRejectScore,issueTmp.Assignee,cuAccount),issueTmp.IssueNum,owner,path,accessToken)
 	} else if strings.HasPrefix(cBody, cmdApe) {
-		//审核通过
+		//审核通过 修改评分状态
+        if !isReviewer(cuAccount){
+			return
+		}
+		issueTmp := models.IssueTemplate{IssueNum: issueNum}
+		err := models.GetIssueTemplateByColName(&issueTmp, "issue_num")
+		if err != nil {
+			logs.Error(err)
+			return
+		}
+		err = changeOpenEulerScoreStatus(issueTmp.CveId, 3)
+		if err != nil {
+			logs.Error(err)
+		}
 	} else {
 		analysisComment(issueNum, cuAccount, cBody,&payload)
 	}
 
 }
 
+func isReviewer(path string) bool {
+	sr := models.SecurityReviewer{NameSpace: path}
+	return sr.Read("name_space")
+}
+
 func analysisComment(issueNum string, cuAccount string, cBody string,payload *models.CommentPayload) {
 	cmdAys := beego.AppConfig.DefaultString("analysisCmd", "/analysis")
 	cmdEnd := beego.AppConfig.DefaultString("endCmd", "/done")
@@ -184,7 +290,8 @@ func analysisComment(issueNum string, cuAccount string, cBody string,payload *mo
 							logs.Error(err)
 						} else {
 							score.OpenEulerScore = issueTmp.OpenEulerScore
-							err := models.UpdateScore(&score, "openeuler_score")
+							score.Ostatus = 1
+							err := models.UpdateScore(&score, "openeuler_score","o_score_status")
 							if err != nil {
 								logs.Error(err)
 							}
@@ -207,24 +314,65 @@ func analysisComment(issueNum string, cuAccount string, cBody string,payload *mo
 		//Check whether the data is legal
         if msg,ok :=checkIssueAnalysisComplete(&issueTmp);!ok{
         	//send comment to issue
-        	msg = fmt.Sprintf(`@%v  %v`,issueTmp.Assignee,msg)
+        	msg = fmt.Sprintf(CommentAnalysisCplTpl,issueTmp.Assignee,msg)
 			accessToken := os.Getenv("issueaccesstoken")
 			owner := beego.AppConfig.String("gitee::owner")
 			path := beego.AppConfig.String("gitee::path")
 			taskhandler.AddCommentToIssue(msg,issueTmp.IssueNum,owner,path,accessToken)
 		}else {
-			//Are the cvsScore and openEuler score  equal .If not equal, notify the auditor to review .
+			//1. change issue status
+			issueTmp.IssueStatus = 3
+			err := models.UpdateIssueTemplate(&issueTmp, "issue_status")
+			if err != nil{
+				logs.Error(err)
+			}
+			//2. Are the cvsScore and openEuler score  equal .If not equal, notify the auditor to review .
             if issueTmp.OpenEulerScore != issueTmp.NVDScore {
             	//Notify the responsible person for review
-				accessToken := os.Getenv("issueaccesstoken")
-            	content := fmt.Sprintf(`%s已分析完结,CVEScore:%v,OpenEulerScore:%v。请审核[地址](%s)`,
-            		payload.Issue.Title,issueTmp.NVDScore,issueTmp.OpenEulerScore,payload.Issue.HtmlUrl)
-            	taskhandler.SendPrivateLetters(accessToken,content,"zhangjianjun_code")
+				notifyAuditorReview(payload,issueTmp)
+			}else {
+				// change score status
+				err := changeOpenEulerScoreStatus(issueTmp.CveId, 3)
+				if err != nil {
+					logs.Error(err)
+				}
 			}
 		}
 	}
 }
 
+func notifyAuditorReview(payload *models.CommentPayload,issueTmp models.IssueTemplate){
+	//Notify the responsible person for review
+	list, err := models.GetSecurityReviewerList()
+	if err != nil{
+		logs.Error(err)
+		return
+	}
+	accessToken := os.Getenv("issueaccesstoken")
+	content := fmt.Sprintf(ReviewPrivateLettersTpl,
+		payload.Issue.Title,payload.Issue.HtmlUrl,issueTmp.NVDScore,issueTmp.OpenEulerScore)
+	owner := beego.AppConfig.String("gitee::owner")
+	path := beego.AppConfig.String("gitee::path")
+	for _,v := range list{
+		taskhandler.SendPrivateLetters(accessToken,content,v.NameSpace)
+		//add @comment
+		msg := fmt.Sprintf(CommentReviewTpl,v.NameSpace)
+		taskhandler.AddCommentToIssue(msg,issueTmp.IssueNum,owner,path,accessToken)
+	}
+
+
+}
+
+func changeOpenEulerScoreStatus(cveId int64,status int8)  error{
+	score, err := models.QueryIssueScore(cveId)
+	if err != nil {
+		return err
+	}
+	score.Ostatus = status
+	err = models.UpdateScore(&score, "o_score_status")
+	return  err
+}
+
 func checkIssueAnalysisComplete(i *models.IssueTemplate) (msg string,ok bool) {
 	if i == nil {
 		logs.Error("issue template is nil")

models/giteeissue.go

+package models
+
+import "time"
+
+//GiteOriginIssue 码云上已经存在的issue
+type GiteOriginIssue struct {
+   Id int64 `orm:"pk;auto"`
+   IssueId int64 `json:"issue_id" orm:"unique"`
+   Url string ` description:"issue gitee 链接"`
+   Number string `orm:"size(50);unique" description:"issue 编号"`
+   State string `orm:"size(50)" description:"issue 状态"`
+   Title string `orm:"null" description:"issue 标题"`
+   CveNumber string `json:"cve_number" description:"CVE 编号"`
+   Body string `orm:"null;type(text)" description:"issue 主体"`
+   IssueCreate string `json:"issue_create" description:"issue 创建者"`
+   IssueAssignee string `json:"issue_assignee" orm:"null" description:"issue 责任人"`
+   RepoPath string `json:"repo_path" description:"仓库空间地址"`
+   RepoUrl string  `json:"repo_url" description:"仓库码云地址链接"`
+   IssueType string `json:"issue_type" description:"issue 类型"`
+   IssueExistTpl bool `json:"issue_exist_tpl" description:"此 issue 是否有对应的模板"`
+   SecurityHole bool `json:"security_hole" description:"是否为安全漏洞"`
+   IssueCreateAt time.Time `json:"issue_create_at" orm:"null" description:"issue 创建的时间"`
+   IssueUpdateAt time.Time `json:"issue_update_at" orm:"null" description:"issue 更新的时间"`
+   IssueFinishAt time.Time `json:"issue_finish_at" orm:"null" description:"issue 关闭的时间"`
+   GrabTime time.Time `json:"grab_time" orm:"auto_now;type(datetime)" description:"记录当前issue抓取的时间"`
+
+}

models/hookevent.go

@@ -9,16 +9,37 @@ type HookUser struct {
 	Email     string //用户的邮箱
 	UserName  string `json:"username"`   //用户的码云个人空间地址
 	AvatarUrl string `json:"avatar_url"` //用户头像
-	SiteAdmin bool   `json:"site_admin"`//是不是管理员
+	SiteAdmin bool   `json:"site_admin"` //是不是管理员
 
 }
 
 type HookIssue struct {
-	Id     int64
-	Number string
-	Title  string
-	State  string
-	HtmlUrl  string    `json:"html_url"` //评论在码云上的url
+	Id           int64
+	Number       string
+	Title        string
+	State        string
+	HtmlUrl      string `json:"html_url"` //评论在码云上的url
+	Body         string
+	StateName    string `json:"state_name"`
+	User         HookUser
+	Assignee     HookUser
+	Repository   HookRepository
+	CreateAt     time.Time `json:"create_at"`
+	UpdateAt     time.Time `json:"update_at"`
+	FinishedAt   time.Time `json:"finished_at"`
+	IssueType    string    `json:"issue_type"`
+	SecurityHole bool      `json:"security_hole"`
+}
+
+type HookRepository struct {
+	Id       int64
+	FullName string `json:"full_name"`
+	Url      string `json:"url"`
+	Path     string
+	Name     string
+	Owner    HookUser
+	Private  bool
+	public   bool
 }
 
 type HookComment struct {
@@ -42,9 +63,27 @@ type CommentPayload struct {
 	Comment       *HookComment //评论数据
 	Note          string       //被评论目标的评论数据
 	NoteableType  string       `json:"noteable_type"` //被评论的目标类型
-	NoteableId    int64        `json:"noteable_id"` //被评论的目标ID
+	NoteableId    int64        `json:"noteable_id"`   //被评论的目标ID
 	Title         string       //被评论的目标标题
 	PerId         string       `json:"per_iid"`         //被评论的目标标识
 	ShortCommitId string       `json:"short_commit_id"` //被评论的commit提交中的简短sha
-	Issue *HookIssue
+	Issue         *HookIssue
+}
+
+type IssuePayload struct {
+	HookId      int64     `json:"hook_id"`   //  钩子 id。
+	HookUrl     string    `json:"hook_url"`  // 钩子路由。
+	HookName    string    `json:"hook_name"` // 钩子名，固定为 issue_hooks。
+	Password    string    `json:"password"`  // 钩子密码
+	Action      string    //issue 状态
+	Issue       HookIssue //issue 信息
+	Sender      HookUser  //触发 hook 的用户信息。
+	TargetUser  HookUser  `json:"target_user"` //被委托处理 issue 的用户信息
+	User        HookUser  //issue 的创建者
+	Assignee    HookUser  //issue 的负责人
+	Iid         string    //issue 的标识
+	Title       string    //issue 的标题
+	Description string    //issue 的描述
+	State       string    //issue 的状态
+	Url         string    //issue 在码云上的url
 }

models/modeldb.go

@@ -143,7 +143,7 @@ type IssueTemplate struct {
 	Assignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"`
 	Status int8 `orm:"default(0);column(status)" description:"1:待办的；2：进行中；3：已完成；4：已拒绝"`
 	StatusName string `orm:"size(128);column(status_name)" description:"issue状态名称"`
-	IssueStatus int8 `orm:"default(0);column(issue_status)" description:"1:待分析；2：已关闭；3已分析，待修复；4：已修复；5：已发布"`
+	IssueStatus int8 `orm:"default(0);column(issue_status)" description:"1:待分析；2：已正常关闭；3已分析，待修复；4：已修复；5：已发布；6：已异常关闭"`
 	IssueLabel string `orm:"size(256);column(issue_label)" description:"issue标签, cve/Undisclosed, cve/Disclosed"`
 	Owner string `orm:"size(128);column(owner)" description:"仓库地址"`
 	Repo string `orm:"size(128);column(repo)" description:"仓库路径"`
@@ -299,6 +299,10 @@ type OriginUpstreamConfigNode struct {
 	Operator string `orm:"size(256);column(operator);null"description:"operator"`
 }
 
+type SecurityReviewer struct {
+	Id        int64  `orm:"pk;auto"`
+	NameSpace string `orm:"unique" description:"码云空间地址"`
+}
 type OriginUpstreamConfigNodeCpe struct {
 	CpeId int64 `orm:"pk;auto;column(cpe_id)"`
 	NodeId int64 `orm:"index;column(node_id)"description:"OriginUpstreamConfigNode 外键"`
@@ -436,13 +440,13 @@ func CreateDb() bool {
 			new(OpenEulerSA), new(Score), new(SecurityNotice), new(Package),
 			new(IssueTemplate), new(ScoreRecord), new(IssueHooks),
 			new(GitPackageTable), new(GitOpenEuler), new(GitPackageInfo), new(GitSubPack),
-			new(GitSubPackRequire), new(GitSubPackProvides), new(GitSubPackRequiredby),
+			new(SecurityReviewer),new(GitSubPackRequire), new(GitSubPackProvides), new(GitSubPackRequiredby),
 			new(IpWhite), new(OriginUpstream), new(OriginUpstreamDesc),
 			new(OriginUpstreamConfig), new(OriginUpstreamConfigNode),
 		 	new(OriginUpstreamConfigNodeCpe), new(OriginUpstreamImpact), new(OriginUpstreamImpactScore),
 		 	new(OriginUpstreamImpactScoreV3), new(OriginUpstreamImpactScoreV2),
 		 	new(OriginUpstreamPoc), new(OriginUpstreamEvent), new(OriginUpstreamReference), new(OriginUpstreamVulType),
-		 	new(OriginUpstreamFixSuggest), new(OriginUpstreamFixSuggestRef), new(OriginUpstreamFixSuggestRefTag))
+		 	new(OriginUpstreamFixSuggest),new(OriginUpstreamFixSuggestRefTag) ,new(OriginUpstreamFixSuggestRef))
 		logs.Info("table create success!")
 		errosyn := orm.RunSyncdb("default", false, true)
 		if errosyn != nil {

models/reviewer.go

+package models
+
+import (
+	"github.com/astaxie/beego/logs"
+	"github.com/astaxie/beego/orm"
+)
+
+func (s *SecurityReviewer) Read(filed ...string) (ok bool) {
+	o := orm.NewOrm()
+	err := o.Read(s, filed...)
+	if err != nil {
+		logs.Error(err)
+		return false
+	}
+	return true
+}
+
+func GetSecurityReviewerList()(sr []SecurityReviewer,err error)  {
+	var list []SecurityReviewer
+	o := orm.NewOrm()
+	_, err = o.QueryTable("cve_security_reviewer").All(&list)
+	return list,err
+}

taskhandler/grabissue.go

+package taskhandler
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/astaxie/beego/logs"
+	"io/ioutil"
+	"net/http"
+	"sync"
+)
+
+const (
+	GiteOrgInfoUrl    = `https://gitee.com/api/v5/orgs/%v?access_token=%v` //get gitee org info
+	GiteOrgReposUrl   = `https://gitee.com/api/v5/orgs/%v/repos?access_token=%v&type=all&page=%v&per_page=%v`//get all repository
+	GiteRepoIssuesUrl = `https://gitee.com/api/v5/repos/{owner}/{repo}/issues`
+	perPage =50
+)
+
+var wg sync.WaitGroup
+
+type OrgInfo struct {
+	Id           int32  `json:"id,omitempty"`
+	Login        string `json:"login,omitempty"`
+	Url          string `json:"url,omitempty"`
+	AvatarUrl    string `json:"avatar_url,omitempty"`
+	ReposUrl     string `json:"repos_url,omitempty"`
+	EventsUrl    string `json:"events_url,omitempty"`
+	MembersUrl   string `json:"members_url,omitempty"`
+	Description  string `json:"description,omitempty"`
+	Name         string `json:"name,omitempty"`
+	Enterprise   string `json:"enterprise,omitempty"`
+	Members      int64  `json:"members,omitempty"`
+	PublicRepos  int64  `json:"public_repos,omitempty"`
+	PrivateRepos int64  `json:"private_repos,omitempty"`
+}
+
+//GrabIssueByOrg grab issue by org name
+func GrabIssueByOrg(accToken, org string) {
+	orgInfo, err := GetOrgInfo(accToken, org)
+	if err != nil {
+		logs.Error(err)
+		return
+	}
+	reposNum := orgInfo.PublicRepos + orgInfo.PrivateRepos
+	if reposNum <= 0 {
+		logs.Info(fmt.Sprintf("%v cantain %v repository,grab issue finish!",org,reposNum))
+		return
+	}
+	pageSize := reposNum / int64(perPage)
+	if reposNum%int64(perPage) > 0 {
+		pageSize = pageSize + 1
+	}
+	var i int64
+	for i = 1; i <= pageSize; i++ {
+      wg.Add(1)
+      //load org repository list
+	}
+	wg.Wait()
+}
+
+//GrabIssueByRepo grab issue by repository
+func GrabIssueByRepo(accToken, owner, repo, state string, page int) {
+
+}
+
+func GetOrgInfo(accToken, org string) (OrgInfo, error) {
+	oi := OrgInfo{}
+	resp, err := http.Get(fmt.Sprintf(GiteOrgInfoUrl, org, accToken))
+	if err != nil {
+		return oi, err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return oi, err
+	}
+	err = json.Unmarshal(body, &oi)
+	return oi, err
+}
+
+func GetOrgRepos(accToken,org string) {
+
+}