New Dashboard and support variables in alert_rule_note (#953)

* change alert rule

* Db connect update (#939)

* update target's cluster field when clustername modified in server.conf

* code refactor

* db connect update

* delete DriverName
Co-authored-by: NUlric Qin <ulric.qin@gmail.com>
Co-authored-by: Nzhangjiandong <zhang.jiandong@baiso.com>

* update sql struct

* change sql

* add some files for new dashboard

* add new board apis

* fix query data

* add dashboard migrate api

* rule note support template

* add value as data for template

* parse rule note before persist

* use prometheus var names

* fixbug rule note template

* refactor sql

* add logo

* refactor: add some log

* mv package poster to pkg

* add version

* compute user total in usage reporter

* feat: add some service api
Co-authored-by: N710leo <710leo@gmail.com>
Co-authored-by: Ncountingwww <871138993@qq.com>
Co-authored-by: Nzhangjiandong <zhang.jiandong@baiso.com>

Makefile

@@ -2,7 +2,7 @@
 
 NOW = $(shell date -u '+%Y%m%d%I%M%S')
 
-RELEASE_VERSION = 5.7.1
+RELEASE_VERSION = 5.8.0
 
 APP 			= n9e
 SERVER_BIN  	= $(APP)
@@ -15,7 +15,7 @@ SERVER_BIN  	= $(APP)
 all: build
 
 build:
-	go build -ldflags "-w -s -X main.VERSION=$(RELEASE_VERSION)" -o $(SERVER_BIN) ./src
+	go build -ldflags "-w -s -X github.com/didi/nightingale/v5/src/pkg/version.VERSION=$(RELEASE_VERSION)" -o $(SERVER_BIN) ./src
 
 # start:
 # 	@go run -ldflags "-X main.VERSION=$(RELEASE_TAG)" ./cmd/${APP}/main.go web -c ./configs/config.toml -m ./configs/model.conf --menu ./configs/menu.yaml

README.md

+<img src="doc/img/ccf-n9e.png" width="240">
+
 # Nightingale
 
  Nightingale is an enterprise-level cloud-native monitoring system, which can be used as drop-in replacement of Prometheus for alerting and management.   

README_ZH.md

+<img src="doc/img/ccf-n9e.png" width="240">
+
 # Nightingale
 [English](./README.md) | [中文](./README_ZH.md)
 

docker/initsql/a-n9e.sql

@@ -152,6 +152,28 @@ CREATE TABLE `busi_group_member` (
 
 insert into busi_group_member(busi_group_id, user_group_id, perm_flag) values(1, 1, "rw");
 
+-- for dashboard new version
+CREATE TABLE `board` (
+    `id` bigint unsigned not null auto_increment,
+    `group_id` bigint not null default 0 comment 'busi group id',
+    `name` varchar(191) not null,
+    `tags` varchar(255) not null comment 'split by space',
+    `create_at` bigint not null default 0,
+    `create_by` varchar(64) not null default '',
+    `update_at` bigint not null default 0,
+    `update_by` varchar(64) not null default '',
+    PRIMARY KEY (`id`),
+    UNIQUE KEY (`group_id`, `name`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
+
+-- for dashboard new version
+CREATE TABLE `board_payload` (
+    `id` bigint unsigned not null comment 'dashboard id',
+    `payload` mediumtext not null,
+    UNIQUE KEY (`id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
+
+-- deprecated
 CREATE TABLE `dashboard` (
     `id` bigint unsigned not null auto_increment,
     `group_id` bigint not null default 0 comment 'busi group id',
@@ -166,6 +188,7 @@ CREATE TABLE `dashboard` (
     UNIQUE KEY (`group_id`, `name`)
 ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
+-- deprecated
 -- auto create the first subclass 'Default chart group' of dashboard
 CREATE TABLE `chart_group` (
     `id` bigint unsigned not null auto_increment,
@@ -176,6 +199,7 @@ CREATE TABLE `chart_group` (
     KEY (`dashboard_id`)
 ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
+-- deprecated
 CREATE TABLE `chart` (
     `id` bigint unsigned not null auto_increment,
     `group_id` bigint unsigned not null comment 'chart group id',
@@ -200,7 +224,11 @@ CREATE TABLE `alert_rule` (
     `group_id` bigint not null default 0 comment 'busi group id',
     `cluster` varchar(128) not null,
     `name` varchar(255) not null,
-    `note` varchar(255) not null,
+    `note` varchar(1024) not null default '',
+    `prod` varchar(255) not null default '',
+    `algorithm` varchar(255) not null default '',
+    `algo_params` varchar(255),
+    `delay` int not null default 0,
     `severity` tinyint(1) not null comment '0:Emergency 1:Warning 2:Notice',
     `disabled` tinyint(1) not null comment '0:enabled 1:disabled',
     `prom_for_duration` int not null comment 'prometheus for, unit:s',
@@ -333,7 +361,9 @@ CREATE TABLE `alert_cur_event` (
     `hash` varchar(64) not null comment 'rule_id + vector_pk',
     `rule_id` bigint unsigned not null,
     `rule_name` varchar(255) not null,
-    `rule_note` varchar(512) not null default 'alert rule note',
+    `rule_note` varchar(2048) not null default 'alert rule note',
+    `rule_prod` varchar(255) not null default '',
+    `rule_algo` varchar(255) not null default '',
     `severity` tinyint(1) not null comment '0:Emergency 1:Warning 2:Notice',
     `prom_for_duration` int not null comment 'prometheus for, unit:s',
     `prom_ql` varchar(8192) not null comment 'promql',
@@ -365,7 +395,9 @@ CREATE TABLE `alert_his_event` (
     `hash` varchar(64) not null comment 'rule_id + vector_pk',
     `rule_id` bigint unsigned not null,
     `rule_name` varchar(255) not null,
-    `rule_note` varchar(512) not null default 'alert rule note',
+    `rule_note` varchar(2048) not null default 'alert rule note',
+    `rule_prod` varchar(255) not null default '',
+    `rule_algo` varchar(255) not null default '',
     `severity` tinyint(1) not null comment '0:Emergency 1:Warning 2:Notice',
     `prom_for_duration` int not null comment 'prometheus for, unit:s',
     `prom_ql` varchar(8192) not null comment 'promql',

etc/server.conf

@@ -123,7 +123,9 @@ Address = "127.0.0.1:6379"
 # UseTLS = false
 # TLSMinVersion = "1.2"
 
-[Gorm]
+[DB]
+# postgres: host=%s port=%s user=%s dbname=%s password=%s sslmode=%s
+DSN="root:1234@tcp(127.0.0.1:3306)/n9e_v5?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"
 # enable debug mode or not
 Debug = false
 # mysql postgres
@@ -137,31 +139,7 @@ MaxIdleConns = 50
 # table prefix
 TablePrefix = ""
 # enable auto migrate or not
-EnableAutoMigrate = false
-
-[MySQL]
-# mysql address host:port
-Address = "127.0.0.1:3306"
-# mysql username
-User = "root"
-# mysql password
-Password = "1234"
-# database name
-DBName = "n9e_v5"
-# connection params
-Parameters = "charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"
-
-[Postgres]
-# pg address host:port
-Address = "127.0.0.1:5432"
-# pg user
-User = "root"
-# pg password
-Password = "1234"
-# database name
-DBName = "n9e_v5"
-# ssl mode
-SSLMode = "disable"
+# EnableAutoMigrate = false
 
 [Reader]
 # prometheus base url

etc/webapi.conf

@@ -144,9 +144,10 @@ Address = "127.0.0.1:6379"
 # UseTLS = false
 # TLSMinVersion = "1.2"
 
-[Gorm]
+[DB]
+DSN="root:1234@tcp(127.0.0.1:3306)/n9e_v5?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"
 # enable debug mode or not
-Debug = true
+Debug = false
 # mysql postgres
 DBType = "mysql"
 # unit: s
@@ -158,31 +159,7 @@ MaxIdleConns = 50
 # table prefix
 TablePrefix = ""
 # enable auto migrate or not
-EnableAutoMigrate = false
-
-[MySQL]
-# mysql address host:port
-Address = "127.0.0.1:3306"
-# mysql username
-User = "root"
-# mysql password
-Password = "1234"
-# database name
-DBName = "n9e_v5"
-# connection params
-Parameters = "charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"
-
-[Postgres]
-# pg address host:port
-Address = "127.0.0.1:5432"
-# pg user
-User = "root"
-# pg password
-Password = "1234"
-# database name
-DBName = "n9e_v5"
-# ssl mode
-SSLMode = "disable"
+# EnableAutoMigrate = false
 
 [[Clusters]]
 # Prometheus cluster name

src/main.go

@@ -7,17 +7,15 @@ import (
 	"github.com/toolkits/pkg/runner"
 	"github.com/urfave/cli/v2"
 
+	"github.com/didi/nightingale/v5/src/pkg/version"
 	"github.com/didi/nightingale/v5/src/server"
 	"github.com/didi/nightingale/v5/src/webapi"
 )
 
-// VERSION go build -ldflags "-X main.VERSION=x.x.x"
-var VERSION = "not specified"
-
 func main() {
 	app := cli.NewApp()
 	app.Name = "n9e"
-	app.Version = VERSION
+	app.Version = version.VERSION
 	app.Usage = "Nightingale, enterprise prometheus management"
 	app.Commands = []*cli.Command{
 		newWebapiCmd(),
@@ -44,7 +42,7 @@ func newWebapiCmd() *cli.Command {
 			if c.String("conf") != "" {
 				opts = append(opts, webapi.SetConfigFile(c.String("conf")))
 			}
-			opts = append(opts, webapi.SetVersion(VERSION))
+			opts = append(opts, webapi.SetVersion(version.VERSION))
 
 			webapi.Run(opts...)
 			return nil
@@ -70,7 +68,7 @@ func newServerCmd() *cli.Command {
 			if c.String("conf") != "" {
 				opts = append(opts, server.SetConfigFile(c.String("conf")))
 			}
-			opts = append(opts, server.SetVersion(VERSION))
+			opts = append(opts, server.SetVersion(version.VERSION))
 
 			server.Run(opts...)
 			return nil

src/models/alert_cur_event.go

 package models
 
 import (
+	"bytes"
 	"fmt"
+	"html/template"
 	"strconv"
 	"strings"
+
+	"github.com/didi/nightingale/v5/src/pkg/tplx"
 )
 
 type AlertCurEvent struct {
@@ -15,6 +19,8 @@ type AlertCurEvent struct {
 	RuleId             int64             `json:"rule_id"`
 	RuleName           string            `json:"rule_name"`
 	RuleNote           string            `json:"rule_note"`
+	RuleProd           string            `json:"rule_prod"`
+	RuleAlgo           string            `json:"rule_algo"`
 	Severity           int               `json:"severity"`
 	PromForDuration    int               `json:"prom_for_duration"`
 	PromQl             string            `json:"prom_ql"`
@@ -54,6 +60,34 @@ type AggrRule struct {
 	Value string
 }
 
+func (e *AlertCurEvent) ParseRuleNote() error {
+	e.RuleNote = strings.TrimSpace(e.RuleNote)
+
+	if e.RuleNote == "" {
+		return nil
+	}
+
+	var defs = []string{
+		"{{$labels := .TagsMap}}",
+		"{{$value := .TriggerValue}}",
+	}
+
+	text := strings.Join(append(defs, e.RuleNote), "")
+	t, err := template.New(fmt.Sprint(e.RuleId)).Funcs(tplx.TemplateFuncMap).Parse(text)
+	if err != nil {
+		return err
+	}
+
+	var body bytes.Buffer
+	err = t.Execute(&body, e)
+	if err != nil {
+		return err
+	}
+
+	e.RuleNote = body.String()
+	return nil
+}
+
 func (e *AlertCurEvent) GenCardTitle(rules []*AggrRule) string {
 	arr := make([]string, len(rules))
 	for i := 0; i < len(rules); i++ {
@@ -125,6 +159,8 @@ func (e *AlertCurEvent) ToHis() *AlertHisEvent {
 		Hash:             e.Hash,
 		RuleId:           e.RuleId,
 		RuleName:         e.RuleName,
+		RuleProd:         e.RuleProd,
+		RuleAlgo:         e.RuleAlgo,
 		RuleNote:         e.RuleNote,
 		Severity:         e.Severity,
 		PromForDuration:  e.PromForDuration,

src/models/alert_his_event.go

@@ -15,6 +15,8 @@ type AlertHisEvent struct {
 	RuleId             int64       `json:"rule_id"`
 	RuleName           string      `json:"rule_name"`
 	RuleNote           string      `json:"rule_note"`
+	RuleProd           string      `json:"rule_prod"`
+	RuleAlgo           string      `json:"rule_algo"`
 	Severity           int         `json:"severity"`
 	PromForDuration    int         `json:"prom_for_duration"`
 	PromQl             string      `json:"prom_ql"`

src/models/alert_rule.go

 package models
 
 import (
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
@@ -18,6 +19,11 @@ type AlertRule struct {
 	Cluster              string      `json:"cluster"`                      // take effect by cluster
 	Name                 string      `json:"name"`                         // rule name
 	Note                 string      `json:"note"`                         // will sent in notify
+	Prod                 string      `json:"prod"`                         // product empty means n9e
+	Algorithm            string      `json:"algorithm"`                    // algorithm (''|holtwinters), empty means threshold
+	AlgoParams           string      `json:"-" gorm:"algo_params"`         // params algorithm need
+	AlgoParamsJson       interface{} `json:"algo_params" gorm:"-"`         //
+	Delay                int         `json:"delay"`                        // Time (in seconds) to delay evaluation
 	Severity             int         `json:"severity"`                     // 0: Emergency 1: Warning 2: Notice
 	Disabled             int         `json:"disabled"`                     // 0: enabled, 1: disabled
 	PromForDuration      int         `json:"prom_for_duration"`            // prometheus for, unit:s
@@ -145,7 +151,11 @@ func (ar *AlertRule) Update(arf AlertRule) error {
 		}
 	}
 
-	arf.FE2DB()
+	err := arf.FE2DB()
+	if err != nil {
+		return err
+	}
+
 	arf.Id = ar.Id
 	arf.GroupId = ar.GroupId
 	arf.CreateAt = ar.CreateAt
@@ -203,12 +213,19 @@ func (ar *AlertRule) FillNotifyGroups(cache map[int64]*UserGroup) error {
 	return nil
 }
 
-func (ar *AlertRule) FE2DB() {
+func (ar *AlertRule) FE2DB() error {
 	ar.EnableDaysOfWeek = strings.Join(ar.EnableDaysOfWeekJSON, " ")
 	ar.NotifyChannels = strings.Join(ar.NotifyChannelsJSON, " ")
 	ar.NotifyGroups = strings.Join(ar.NotifyGroupsJSON, " ")
 	ar.Callbacks = strings.Join(ar.CallbacksJSON, " ")
 	ar.AppendTags = strings.Join(ar.AppendTagsJSON, " ")
+	algoParamsByte, err := json.Marshal(ar.AlgoParamsJson)
+	if err != nil {
+		return fmt.Errorf("marshal algo_params err:%v", err)
+	}
+
+	ar.AlgoParams = string(algoParamsByte)
+	return nil
 }
 
 func (ar *AlertRule) DB2FE() {
@@ -217,6 +234,7 @@ func (ar *AlertRule) DB2FE() {
 	ar.NotifyGroupsJSON = strings.Fields(ar.NotifyGroups)
 	ar.CallbacksJSON = strings.Fields(ar.Callbacks)
 	ar.AppendTagsJSON = strings.Fields(ar.AppendTags)
+	json.Unmarshal([]byte(ar.AlgoParams), &ar.AlgoParamsJson)
 }
 
 func AlertRuleDels(ids []int64, busiGroupId int64) error {
@@ -254,7 +272,7 @@ func AlertRuleGets(groupId int64) ([]AlertRule, error) {
 }
 
 func AlertRuleGetsByCluster(cluster string) ([]*AlertRule, error) {
-	session := DB().Where("disabled = ?", 0)
+	session := DB().Where("disabled = ? and prod = ?", 0, "")
 
 	if cluster != "" {
 		session = session.Where("cluster = ?", cluster)
@@ -271,6 +289,20 @@ func AlertRuleGetsByCluster(cluster string) ([]*AlertRule, error) {
 	return lst, err
 }
 
+func AlertRulesGetByProds(prods []string) ([]*AlertRule, error) {
+	session := DB().Where("disabled = ? and prod IN (?)", 0, prods)
+
+	var lst []*AlertRule
+	err := session.Find(&lst).Error
+	if err == nil {
+		for i := 0; i < len(lst); i++ {
+			lst[i].DB2FE()
+		}
+	}
+
+	return lst, err
+}
+
 func AlertRuleGet(where string, args ...interface{}) (*AlertRule, error) {
 	var lst []*AlertRule
 	err := DB().Where(where, args...).Find(&lst).Error
@@ -306,7 +338,7 @@ func AlertRuleGetName(id int64) (string, error) {
 }
 
 func AlertRuleStatistics(cluster string) (*Statistics, error) {
-	session := DB().Model(&AlertRule{}).Select("count(*) as total", "max(update_at) as last_updated").Where("disabled = ?", 0)
+	session := DB().Model(&AlertRule{}).Select("count(*) as total", "max(update_at) as last_updated").Where("disabled = ? and prod = ?", 0, "")
 
 	if cluster != "" {
 		session = session.Where("cluster = ?", cluster)

src/models/board.go

+package models
+
+import (
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/toolkits/pkg/str"
+	"gorm.io/gorm"
+)
+
+type Board struct {
+	Id       int64  `json:"id" gorm:"primaryKey"`
+	GroupId  int64  `json:"group_id"`
+	Name     string `json:"name"`
+	Tags     string `json:"tags"`
+	CreateAt int64  `json:"create_at"`
+	CreateBy string `json:"create_by"`
+	UpdateAt int64  `json:"update_at"`
+	UpdateBy string `json:"update_by"`
+	Configs  string `json:"configs" gorm:"-"`
+}
+
+func (b *Board) TableName() string {
+	return "board"
+}
+
+func (b *Board) Verify() error {
+	if b.Name == "" {
+		return errors.New("Name is blank")
+	}
+
+	if str.Dangerous(b.Name) {
+		return errors.New("Name has invalid characters")
+	}
+
+	return nil
+}
+
+func (b *Board) Add() error {
+	if err := b.Verify(); err != nil {
+		return err
+	}
+
+	now := time.Now().Unix()
+	b.CreateAt = now
+	b.UpdateAt = now
+
+	return Insert(b)
+}
+
+func (b *Board) Update(selectField interface{}, selectFields ...interface{}) error {
+	if err := b.Verify(); err != nil {
+		return err
+	}
+
+	return DB().Model(b).Select(selectField, selectFields...).Updates(b).Error
+}
+
+func (b *Board) Del() error {
+	return DB().Transaction(func(tx *gorm.DB) error {
+		if err := tx.Where("id=?", b.Id).Delete(&BoardPayload{}).Error; err != nil {
+			return err
+		}
+
+		if err := tx.Where("id=?", b.Id).Delete(&Board{}).Error; err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+// BoardGet for detail page
+func BoardGet(where string, args ...interface{}) (*Board, error) {
+	var lst []*Board
+	err := DB().Where(where, args...).Find(&lst).Error
+	if err != nil {
+		return nil, err
+	}
+
+	if len(lst) == 0 {
+		return nil, nil
+	}
+
+	payload, err := BoardPayloadGet(lst[0].Id)
+	if err != nil {
+		return nil, err
+	}
+
+	lst[0].Configs = payload
+
+	return lst[0], nil
+}
+
+func BoardCount(where string, args ...interface{}) (num int64, err error) {
+	return Count(DB().Model(&Board{}).Where(where, args...))
+}
+
+func BoardExists(where string, args ...interface{}) (bool, error) {
+	num, err := BoardCount(where, args...)
+	return num > 0, err
+}
+
+// BoardGets for list page
+func BoardGets(groupId int64, query string) ([]Board, error) {
+	session := DB().Where("group_id=?", groupId).Order("name")
+
+	arr := strings.Fields(query)
+	if len(arr) > 0 {
+		for i := 0; i < len(arr); i++ {
+			if strings.HasPrefix(arr[i], "-") {
+				q := "%" + arr[i][1:] + "%"
+				session = session.Where("name not like ? and tags not like ?", q, q)
+			} else {
+				q := "%" + arr[i] + "%"
+				session = session.Where("(name like ? or tags like ?)", q, q)
+			}
+		}
+	}
+
+	var objs []Board
+	err := session.Find(&objs).Error
+	return objs, err
+}

src/models/board_payload.go

+package models
+
+import "errors"
+
+type BoardPayload struct {
+	Id      int64  `json:"id" gorm:"primaryKey"`
+	Payload string `json:"payload"`
+}
+
+func (p *BoardPayload) TableName() string {
+	return "board_payload"
+}
+
+func (p *BoardPayload) Update(selectField interface{}, selectFields ...interface{}) error {
+	return DB().Model(p).Select(selectField, selectFields...).Updates(p).Error
+}
+
+func BoardPayloadGets(ids []int64) ([]*BoardPayload, error) {
+	if len(ids) == 0 {
+		return nil, errors.New("empty ids")
+	}
+
+	var arr []*BoardPayload
+	err := DB().Where("id in ?", ids).Find(&arr).Error
+	return arr, err
+}
+
+func BoardPayloadGet(id int64) (string, error) {
+	payloads, err := BoardPayloadGets([]int64{id})
+	if err != nil {
+		return "", err
+	}
+
+	if len(payloads) == 0 {
+		return "", nil
+	}
+
+	return payloads[0].Payload, nil
+}
+
+func BoardPayloadSave(id int64, payload string) error {
+	var bp BoardPayload
+	err := DB().Where("id = ?", id).Find(&bp).Error
+	if err != nil {
+		return err
+	}
+
+	if bp.Id > 0 {
+		// already exists
+		bp.Payload = payload
+		return bp.Update("payload")
+	}
+
+	return Insert(&BoardPayload{
+		Id:      id,
+		Payload: payload,
+	})
+}

src/models/dashboard.go

@@ -160,3 +160,9 @@ func DashboardGetsByIds(ids []int64) ([]Dashboard, error) {
 	err := DB().Where("id in ?", ids).Order("name").Find(&lst).Error
 	return lst, err
 }
+
+func DashboardGetAll() ([]Dashboard, error) {
+	var lst []Dashboard
+	err := DB().Find(&lst).Error
+	return lst, err
+}

src/pkg/ormx/ormx.go

 package ormx
 
 import (
+	"time"
 	"fmt"
 	"strings"
-	"time"
-
 	"gorm.io/driver/mysql"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/schema"
 )
 
-// Config GORM Config
-type Config struct {
+// DBConfig GORM DBConfig
+type DBConfig struct {
 	Debug        bool
 	DBType       string
 	DSN          string
@@ -23,7 +22,7 @@ type Config struct {
 }
 
 // New Create gorm.DB instance
-func New(c Config) (*gorm.DB, error) {
+func New(c DBConfig) (*gorm.DB, error) {
 	var dialector gorm.Dialector
 
 	switch strings.ToLower(c.DBType) {

src/pkg/tplx/tplx.go

+package tplx
+
+import (
+	"html/template"
+	"time"
+)
+
+var TemplateFuncMap = template.FuncMap{
+	"unescaped":  func(str string) interface{} { return template.HTML(str) },
+	"urlconvert": func(str string) interface{} { return template.URL(str) },
+	"timeformat": func(ts int64, pattern ...string) string {
+		defp := "2006-01-02 15:04:05"
+		if len(pattern) > 0 {
+			defp = pattern[0]
+		}
+		return time.Unix(ts, 0).Format(defp)
+	},
+	"timestamp": func(pattern ...string) string {
+		defp := "2006-01-02 15:04:05"
+		if len(pattern) > 0 {
+			defp = pattern[0]
+		}
+		return time.Now().Format(defp)
+	},
+}

src/pkg/version/version.go

+package version
+
+// VERSION go build -ldflags "-X pkg.version.VERSION=x.x.x"
+var VERSION = "not specified"

src/server/common/conv/conv.go

@@ -14,6 +14,10 @@ type Vector struct {
 }
 
 func ConvertVectors(value model.Value) (lst []Vector) {
+	if value == nil {
+		return
+	}
+
 	switch value.Type() {
 	case model.ValVector:
 		items, ok := value.(model.Vector)

src/server/common/sender/dingtalk.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/didi/nightingale/v5/src/server/common/poster"
+	"github.com/didi/nightingale/v5/src/pkg/poster"
 	"github.com/toolkits/pkg/logger"
 )
 

src/server/common/sender/feishu.go

@@ -3,7 +3,7 @@ package sender
 import (
 	"time"
 
-	"github.com/didi/nightingale/v5/src/server/common/poster"
+	"github.com/didi/nightingale/v5/src/pkg/poster"
 	"github.com/toolkits/pkg/logger"
 )
 

src/server/common/sender/wecom.go

@@ -3,7 +3,7 @@ package sender
 import (
 	"time"
 
-	"github.com/didi/nightingale/v5/src/server/common/poster"
+	"github.com/didi/nightingale/v5/src/pkg/poster"
 	"github.com/toolkits/pkg/logger"
 )
 

src/server/config/config.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/didi/nightingale/v5/src/pkg/httpx"
 	"github.com/didi/nightingale/v5/src/pkg/logx"
+	"github.com/didi/nightingale/v5/src/pkg/ormx"
 	"github.com/didi/nightingale/v5/src/server/reader"
 	"github.com/didi/nightingale/v5/src/server/writer"
 	"github.com/didi/nightingale/v5/src/storage"
@@ -122,6 +123,7 @@ type Config struct {
 	RunMode            string
 	ClusterName        string
 	BusiGroupLabelKey  string
+	AnomalyDataApi     []string
 	EngineDelay        int64
 	DisableUsageReport bool
 	Log                logx.Config
@@ -132,9 +134,7 @@ type Config struct {
 	Alerting           Alerting
 	NoData             NoData
 	Redis              storage.RedisConfig
-	Gorm               storage.Gorm
-	MySQL              storage.MySQL
-	Postgres           storage.Postgres
+	DB                 ormx.DBConfig
 	WriterOpt          writer.GlobalOpt
 	Writers            []writer.Options
 	Reader             reader.Options

src/server/engine/callback.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/didi/nightingale/v5/src/models"
 	"github.com/didi/nightingale/v5/src/pkg/ibex"
-	"github.com/didi/nightingale/v5/src/server/common/poster"
+	"github.com/didi/nightingale/v5/src/pkg/poster"
 	"github.com/didi/nightingale/v5/src/server/config"
 	"github.com/didi/nightingale/v5/src/server/memsto"
 )

src/server/engine/consume.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 	"time"
 
@@ -42,7 +43,12 @@ func consume(events []interface{}, sema *semaphore.Semaphore) {
 }
 
 func consumeOne(event *models.AlertCurEvent) {
-	logEvent(event, "consume")
+	LogEvent(event, "consume")
+
+	if err := event.ParseRuleNote(); err != nil {
+		event.RuleNote = fmt.Sprintf("failed to parse rule note: %v", err)
+	}
+
 	persist(event)
 
 	if event.IsRecovered && event.NotifyRecovered == 0 {

src/server/engine/logger.go

@@ -5,7 +5,7 @@ import (
 	"github.com/toolkits/pkg/logger"
 )
 
-func logEvent(event *models.AlertCurEvent, location string, err ...error) {
+func LogEvent(event *models.AlertCurEvent, location string, err ...error) {
 	status := "triggered"
 	if event.IsRecovered {
 		status = "recovered"

src/server/engine/notify.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/didi/nightingale/v5/src/models"
 	"github.com/didi/nightingale/v5/src/pkg/sys"
+	"github.com/didi/nightingale/v5/src/pkg/tplx"
 	"github.com/didi/nightingale/v5/src/server/common/sender"
 	"github.com/didi/nightingale/v5/src/server/config"
 	"github.com/didi/nightingale/v5/src/server/memsto"
@@ -31,25 +32,6 @@ import (
 
 var tpls = make(map[string]*template.Template)
 
-var fns = template.FuncMap{
-	"unescaped":  func(str string) interface{} { return template.HTML(str) },
-	"urlconvert": func(str string) interface{} { return template.URL(str) },
-	"timeformat": func(ts int64, pattern ...string) string {
-		defp := "2006-01-02 15:04:05"
-		if len(pattern) > 0 {
-			defp = pattern[0]
-		}
-		return time.Unix(ts, 0).Format(defp)
-	},
-	"timestamp": func(pattern ...string) string {
-		defp := "2006-01-02 15:04:05"
-		if len(pattern) > 0 {
-			defp = pattern[0]
-		}
-		return time.Now().Format(defp)
-	},
-}
-
 func initTpls() error {
 	if config.C.Alerting.TemplatesDir == "" {
 		config.C.Alerting.TemplatesDir = path.Join(runner.Cwd, "etc", "template")
@@ -78,7 +60,7 @@ func initTpls() error {
 	for i := 0; i < len(tplFiles); i++ {
 		tplpath := path.Join(config.C.Alerting.TemplatesDir, tplFiles[i])
 
-		tpl, err := template.New(tplFiles[i]).Funcs(fns).ParseFiles(tplpath)
+		tpl, err := template.New(tplFiles[i]).Funcs(tplx.TemplateFuncMap).ParseFiles(tplpath)
 		if err != nil {
 			return errors.WithMessage(err, "failed to parse tpl: "+tplpath)
 		}
@@ -249,7 +231,7 @@ func handleNotice(notice Notice, bs []byte) {
 }
 
 func notify(event *models.AlertCurEvent) {
-	logEvent(event, "notify")
+	LogEvent(event, "notify")
 
 	notice := genNotice(event)
 	stdinBytes, err := json.Marshal(notice)
@@ -355,7 +337,7 @@ func handleSubscribe(event models.AlertCurEvent, sub *models.AlertSubscribe) {
 		return
 	}
 
-	logEvent(&event, "subscribe")
+	LogEvent(&event, "subscribe")
 
 	fillUsers(&event)
 

src/server/engine/worker.go

@@ -3,11 +3,14 @@ package engine
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/prometheus/common/model"
 	"github.com/toolkits/pkg/logger"
+	"github.com/toolkits/pkg/net/httplib"
 	"github.com/toolkits/pkg/str"
 
 	"github.com/didi/nightingale/v5/src/models"
@@ -89,6 +92,11 @@ func (r RuleEval) Start() {
 	}
 }
 
+type AnomalyPoint struct {
+	Data model.Matrix `json:"data"`
+	Err  string       `json:"error"`
+}
+
 func (r RuleEval) Work() {
 	promql := strings.TrimSpace(r.rule.PromQl)
 	if promql == "" {
@@ -96,15 +104,37 @@ func (r RuleEval) Work() {
 		return
 	}
 
-	value, warnings, err := reader.Reader.Client.Query(context.Background(), promql, time.Now())
-	if err != nil {
-		logger.Errorf("rule_eval:%d promql:%s, error:%v", r.RuleID(), promql, err)
-		return
-	}
+	var value model.Value
+	var err error
+	if r.rule.Algorithm == "" {
+		var warnings reader.Warnings
+		value, warnings, err = reader.Reader.Client.Query(context.Background(), promql, time.Now())
+		if err != nil {
+			logger.Errorf("rule_eval:%d promql:%s, error:%v", r.RuleID(), promql, err)
+			return
+		}
 
-	if len(warnings) > 0 {
-		logger.Errorf("rule_eval:%d promql:%s, warnings:%v", r.RuleID(), promql, warnings)
-		return
+		if len(warnings) > 0 {
+			logger.Errorf("rule_eval:%d promql:%s, warnings:%v", r.RuleID(), promql, warnings)
+			return
+		}
+	} else {
+		var res AnomalyPoint
+		count := len(config.C.AnomalyDataApi)
+		for _, i := range rand.Perm(count) {
+			url := fmt.Sprintf("%s?rid=%d", config.C.AnomalyDataApi[i], r.rule.Id)
+			err = httplib.Get(url).SetTimeout(time.Duration(3000) * time.Millisecond).ToJSON(&res)
+			if err != nil {
+				logger.Errorf("curl %s fail: %v", url, err)
+				continue
+			}
+			if res.Err != "" {
+				logger.Errorf("curl %s fail: %s", url, res.Err)
+				continue
+			}
+			value = res.Data
+			logger.Debugf("curl %s get: %+v", url, res.Data)
+		}
 	}
 
 	r.judge(conv.ConvertVectors(value))
@@ -250,6 +280,8 @@ func (r RuleEval) judge(vectors []conv.Vector) {
 		event.RuleId = r.rule.Id
 		event.RuleName = r.rule.Name
 		event.RuleNote = r.rule.Note
+		event.RuleProd = r.rule.Prod
+		event.RuleAlgo = r.rule.Algorithm
 		event.Severity = r.rule.Severity
 		event.PromForDuration = r.rule.PromForDuration
 		event.PromQl = r.rule.PromQl
@@ -364,6 +396,8 @@ func (r RuleEval) recoverRule(alertingKeys map[string]struct{}, now int64) {
 		// 当然，其实rule的各个字段都可能发生变化了，都更新一下吧
 		event.RuleName = r.rule.Name
 		event.RuleNote = r.rule.Note
+		event.RuleProd = r.rule.Prod
+		event.RuleAlgo = r.rule.Algorithm
 		event.Severity = r.rule.Severity
 		event.PromForDuration = r.rule.PromForDuration
 		event.PromQl = r.rule.PromQl
@@ -387,7 +421,7 @@ func (r RuleEval) pushEventToQueue(event *models.AlertCurEvent) {
 	}
 
 	promstat.CounterAlertsTotal.WithLabelValues(config.C.ClusterName).Inc()
-	logEvent(event, "push_queue")
+	LogEvent(event, "push_queue")
 	if !EventQueue.PushFront(event) {
 		logger.Warningf("event_push_queue: queue is full")
 	}

src/server/router/router.go

@@ -91,4 +91,7 @@ func configRoute(r *gin.Engine, version string) {
 	r.GET("/memory/user-group", userGroupGet)
 
 	r.GET("/metrics", gin.WrapH(promhttp.Handler()))
+
+	service := r.Group("/v1/n9e")
+	service.POST("/event", pushEventToQueue)
 }

src/server/router/router_event.go

+package router
+
+import (
+	"fmt"
+
+	"github.com/didi/nightingale/v5/src/models"
+	"github.com/didi/nightingale/v5/src/server/config"
+	"github.com/didi/nightingale/v5/src/server/engine"
+	promstat "github.com/didi/nightingale/v5/src/server/stat"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+	"github.com/toolkits/pkg/logger"
+)
+
+func pushEventToQueue(c *gin.Context) {
+	var event models.AlertCurEvent
+	ginx.BindJSON(c, &event)
+	if event.RuleId == 0 {
+		ginx.Bomb(200, "event is illegal")
+	}
+
+	promstat.CounterAlertsTotal.WithLabelValues(config.C.ClusterName).Inc()
+	engine.LogEvent(&event, "http_push_queue")
+	if !engine.EventQueue.PushFront(event) {
+		msg := fmt.Sprintf("event:%+v push_queue err: queue is full", event)
+		ginx.Bomb(200, msg)
+		logger.Warningf(msg)
+	}
+	ginx.NewRender(c).Message(nil)
+}

src/server/server.go

@@ -105,11 +105,7 @@ func (s Server) initialize() (func(), error) {
 	}
 
 	// init database
-	if err = storage.InitDB(storage.DBConfig{
-		Gorm:     config.C.Gorm,
-		MySQL:    config.C.MySQL,
-		Postgres: config.C.Postgres,
-	}); err != nil {
+	if err = storage.InitDB(config.C.DB); err != nil {
 		return fns.Ret(), err
 	}
 

src/server/usage/usage.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/didi/nightingale/v5/src/models"
+	"github.com/didi/nightingale/v5/src/pkg/version"
 	"github.com/didi/nightingale/v5/src/server/common/conv"
 	"github.com/didi/nightingale/v5/src/server/reader"
 )
@@ -21,8 +23,10 @@ const (
 
 type Usage struct {
 	Samples    float64 `json:"samples"` // per second
+	Users      float64 `json:"users"`   // user total
 	Maintainer string  `json:"maintainer"`
 	Hostname   string  `json:"hostname"`
+	Version    string  `json:"version"`
 }
 
 func getSamples() (float64, error) {
@@ -61,12 +65,19 @@ func report() {
 		return
 	}
 
+	num, err := models.UserTotal("")
+	if err != nil {
+		return
+	}
+
 	maintainer := "blank"
 
 	u := Usage{
 		Samples:    sps,
+		Users:      float64(num),
 		Hostname:   hostname,
 		Maintainer: maintainer,
+		Version:    version.VERSION,
 	}
 
 	post(u)

src/storage/storage.go

@@ -2,14 +2,10 @@ package storage
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
-	"strings"
-
 	"github.com/go-redis/redis/v8"
 	"gorm.io/gorm"
-
 	"github.com/didi/nightingale/v5/src/pkg/ormx"
 	"github.com/didi/nightingale/v5/src/pkg/tls"
 )
@@ -23,85 +19,16 @@ type RedisConfig struct {
 	tls.ClientConfig
 }
 
-type DBConfig struct {
-	Gorm     Gorm
-	MySQL    MySQL
-	Postgres Postgres
-}
-
-type Gorm struct {
-	Debug             bool
-	DBType            string
-	MaxLifetime       int
-	MaxOpenConns      int
-	MaxIdleConns      int
-	TablePrefix       string
-	EnableAutoMigrate bool
-}
-
-type MySQL struct {
-	Address    string
-	User       string
-	Password   string
-	DBName     string
-	Parameters string
-}
-
-func (a MySQL) DSN() string {
-	return fmt.Sprintf("%s:%s@tcp(%s)/%s?%s",
-		a.User, a.Password, a.Address, a.DBName, a.Parameters)
-}
-
-type Postgres struct {
-	Address  string
-	User     string
-	Password string
-	DBName   string
-	SSLMode  string
-}
-
-func (a Postgres) DSN() string {
-	arr := strings.Split(a.Address, ":")
-	if len(arr) != 2 {
-		panic("pg address(" + a.Address + ") invalid")
-	}
-
-	return fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s",
-		arr[0], arr[1], a.User, a.DBName, a.Password, a.SSLMode)
-}
-
 var DB *gorm.DB
 
-func InitDB(cfg DBConfig) error {
-	db, err := newGormDB(cfg)
+func InitDB(cfg ormx.DBConfig) error {
+	db, err := ormx.New(cfg)
 	if err == nil {
 		DB = db
 	}
 	return err
 }
 
-func newGormDB(cfg DBConfig) (*gorm.DB, error) {
-	var dsn string
-	switch cfg.Gorm.DBType {
-	case "mysql":
-		dsn = cfg.MySQL.DSN()
-	case "postgres":
-		dsn = cfg.Postgres.DSN()
-	default:
-		return nil, errors.New("unknown DBType")
-	}
-
-	return ormx.New(ormx.Config{
-		Debug:        cfg.Gorm.Debug,
-		DBType:       cfg.Gorm.DBType,
-		DSN:          dsn,
-		MaxIdleConns: cfg.Gorm.MaxIdleConns,
-		MaxLifetime:  cfg.Gorm.MaxLifetime,
-		MaxOpenConns: cfg.Gorm.MaxOpenConns,
-		TablePrefix:  cfg.Gorm.TablePrefix,
-	})
-}
-
 var Redis *redis.Client
 
 func InitRedis(cfg RedisConfig) (func(), error) {

src/webapi/config/config.go

@@ -13,6 +13,7 @@ import (
 	"github.com/didi/nightingale/v5/src/pkg/ldapx"
 	"github.com/didi/nightingale/v5/src/pkg/logx"
 	"github.com/didi/nightingale/v5/src/pkg/oidcc"
+	"github.com/didi/nightingale/v5/src/pkg/ormx"
 	"github.com/didi/nightingale/v5/src/storage"
 	"github.com/didi/nightingale/v5/src/webapi/prom"
 )
@@ -90,9 +91,7 @@ type Config struct {
 	AnonymousAccess      AnonymousAccess
 	LDAP                 ldapx.LdapSection
 	Redis                storage.RedisConfig
-	Gorm                 storage.Gorm
-	MySQL                storage.MySQL
-	Postgres             storage.Postgres
+	DB                	 ormx.DBConfig
 	Clusters             []prom.Options
 	Ibex                 Ibex
 	OIDC                 oidcc.Config

src/webapi/router/router.go

@@ -161,12 +161,30 @@ func configRoute(r *gin.Engine, version string) {
 		pages.GET("/targets", jwtAuth(), user(), targetGets)
 		pages.DELETE("/targets", jwtAuth(), user(), perm("/targets/del"), targetDel)
 		pages.GET("/targets/tags", jwtAuth(), user(), targetGetTags)
-		pages.POST("/targets/tags", jwtAuth(), user(), perm("/targets/put"), targetBindTags)
-		pages.DELETE("/targets/tags", jwtAuth(), user(), perm("/targets/put"), targetUnbindTags)
+		pages.POST("/targets/tags", jwtAuth(), user(), perm("/targets/put"), targetBindTagsByFE)
+		pages.DELETE("/targets/tags", jwtAuth(), user(), perm("/targets/put"), targetUnbindTagsByFE)
 		pages.PUT("/targets/note", jwtAuth(), user(), perm("/targets/put"), targetUpdateNote)
 		pages.PUT("/targets/bgid", jwtAuth(), user(), perm("/targets/put"), targetUpdateBgid)
 
-		pages.GET("/dashboards/builtin/list", dashboardBuiltinList)
+		pages.GET("/builtin-boards", builtinBoardGets)
+		pages.GET("/builtin-board/:name", builtinBoardGet)
+
+		pages.GET("/busi-group/:id/boards", jwtAuth(), user(), perm("/dashboards"), bgro(), boardGets)
+		pages.POST("/busi-group/:id/boards", jwtAuth(), user(), perm("/dashboards/add"), bgrw(), boardAdd)
+		pages.POST("/busi-group/:id/board/:bid/clone", jwtAuth(), user(), perm("/dashboards/add"), bgrw(), boardClone)
+
+		pages.GET("/board/:bid", jwtAuth(), user(), boardGet)
+		pages.PUT("/board/:bid", jwtAuth(), user(), perm("/dashboards/put"), boardPut)
+		pages.PUT("/board/:bid/configs", jwtAuth(), user(), perm("/dashboards/put"), boardPutConfigs)
+		pages.DELETE("/boards", jwtAuth(), user(), perm("/dashboards/del"), boardDel)
+
+		// migrate v5.8.0
+		pages.GET("/dashboards", jwtAuth(), admin(), migrateDashboards)
+		pages.GET("/dashboard/:id", jwtAuth(), admin(), migrateDashboardGet)
+		pages.PUT("/dashboard/:id/migrate", jwtAuth(), admin(), migrateDashboard)
+
+		// deprecated ↓
+		pages.GET("/dashboards/builtin/list", builtinBoardGets)
 		pages.POST("/busi-group/:id/dashboards/builtin", jwtAuth(), user(), perm("/dashboards/add"), bgrw(), dashboardBuiltinImport)
 		pages.GET("/busi-group/:id/dashboards", jwtAuth(), user(), perm("/dashboards"), bgro(), dashboardGets)
 		pages.POST("/busi-group/:id/dashboards", jwtAuth(), user(), perm("/dashboards/add"), bgrw(), dashboardAdd)
@@ -186,6 +204,7 @@ func configRoute(r *gin.Engine, version string) {
 		pages.POST("/busi-group/:id/charts", jwtAuth(), user(), bgrw(), chartAdd)
 		pages.PUT("/busi-group/:id/charts", jwtAuth(), user(), bgrw(), chartPut)
 		pages.DELETE("/busi-group/:id/charts", jwtAuth(), user(), bgrw(), chartDel)
+		// deprecated ↑
 
 		pages.GET("/share-charts", chartShareGets)
 		pages.POST("/share-charts", jwtAuth(), chartShareAdd)
@@ -193,10 +212,10 @@ func configRoute(r *gin.Engine, version string) {
 		pages.GET("/alert-rules/builtin/list", alertRuleBuiltinList)
 		pages.POST("/busi-group/:id/alert-rules/builtin", jwtAuth(), user(), perm("/alert-rules/add"), bgrw(), alertRuleBuiltinImport)
 		pages.GET("/busi-group/:id/alert-rules", jwtAuth(), user(), perm("/alert-rules"), alertRuleGets)
-		pages.POST("/busi-group/:id/alert-rules", jwtAuth(), user(), perm("/alert-rules/add"), bgrw(), alertRuleAdd)
+		pages.POST("/busi-group/:id/alert-rules", jwtAuth(), user(), perm("/alert-rules/add"), bgrw(), alertRuleAddByFE)
 		pages.DELETE("/busi-group/:id/alert-rules", jwtAuth(), user(), perm("/alert-rules/del"), bgrw(), alertRuleDel)
 		pages.PUT("/busi-group/:id/alert-rules/fields", jwtAuth(), user(), perm("/alert-rules/put"), bgrw(), alertRulePutFields)
-		pages.PUT("/busi-group/:id/alert-rule/:arid", jwtAuth(), user(), perm("/alert-rules/put"), alertRulePut)
+		pages.PUT("/busi-group/:id/alert-rule/:arid", jwtAuth(), user(), perm("/alert-rules/put"), alertRulePutByFE)
 		pages.GET("/alert-rule/:arid", jwtAuth(), user(), perm("/alert-rules"), alertRuleGet)
 
 		pages.GET("/busi-group/:id/alert-mutes", jwtAuth(), user(), perm("/alert-mutes"), bgro(), alertMuteGets)
@@ -252,11 +271,16 @@ func configRoute(r *gin.Engine, version string) {
 		service.POST("/users", userAddPost)
 
 		service.GET("/targets", targetGets)
-		service.DELETE("/targets", targetDel)
 		service.GET("/targets/tags", targetGetTags)
-		service.POST("/targets/tags", targetBindTags)
-		service.DELETE("/targets/tags", targetUnbindTags)
-		service.PUT("/targets/note", targetUpdateNote)
-		service.PUT("/targets/bgid", targetUpdateBgid)
+		service.POST("/targets/tags", targetBindTagsByService)
+		service.DELETE("/targets/tags", targetUnbindTagsByService)
+		service.PUT("/targets/note", targetUpdateNoteByService)
+
+		service.GET("/alert-rules", alertRuleGets)
+		service.POST("/alert-rules", alertRuleAddByService)
+		service.DELETE("/alert-rules", alertRuleDel)
+		service.PUT("/alert-rule", alertRulePutByService)
+		service.GET("/alert-rule/:arid", alertRuleGet)
+		service.GET("/alert-rules-get-by-prod", alertRulesGetByProds)
 	}
 }

src/webapi/router/router_alert_rule.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -24,8 +25,24 @@ func alertRuleGets(c *gin.Context) {
 	ginx.NewRender(c).Data(ars, err)
 }
 
+func alertRulesGetByProds(c *gin.Context) {
+	prods := ginx.QueryStr(c, "prods", "")
+	arr := strings.Split(prods, ",")
+
+	ars, err := models.AlertRulesGetByProds(arr)
+	if err == nil {
+		cache := make(map[int64]*models.UserGroup)
+		for i := 0; i < len(ars); i++ {
+			ars[i].FillNotifyGroups(cache)
+		}
+	}
+	ginx.NewRender(c).Data(ars, err)
+}
+
 // single or import
-func alertRuleAdd(c *gin.Context) {
+func alertRuleAddByFE(c *gin.Context) {
+	username := c.MustGet("username").(string)
+
 	var lst []models.AlertRule
 	ginx.BindJSON(c, &lst)
 
@@ -34,26 +51,48 @@ func alertRuleAdd(c *gin.Context) {
 		ginx.Bomb(http.StatusBadRequest, "input json is empty")
 	}
 
-	username := c.MustGet("username").(string)
 	bgid := ginx.UrlParamInt64(c, "id")
+	reterr := alertRuleAdd(lst, username, bgid, c.GetHeader("X-Language"))
+
+	ginx.NewRender(c).Data(reterr, nil)
+}
+
+func alertRuleAddByService(c *gin.Context) {
+	var lst []models.AlertRule
+	ginx.BindJSON(c, &lst)
 
+	count := len(lst)
+	if count == 0 {
+		ginx.Bomb(http.StatusBadRequest, "input json is empty")
+	}
+	reterr := alertRuleAdd(lst, "", 0, c.GetHeader("X-Language"))
+	ginx.NewRender(c).Data(reterr, nil)
+}
+
+func alertRuleAdd(lst []models.AlertRule, username string, bgid int64, lang string) map[string]string {
+	count := len(lst)
 	// alert rule name -> error string
 	reterr := make(map[string]string)
 	for i := 0; i < count; i++ {
 		lst[i].Id = 0
 		lst[i].GroupId = bgid
-		lst[i].CreateBy = username
-		lst[i].UpdateBy = username
-		lst[i].FE2DB()
+		if username != "" {
+			lst[i].CreateBy = username
+			lst[i].UpdateBy = username
+		}
+
+		if err := lst[i].FE2DB(); err != nil {
+			reterr[lst[i].Name] = i18n.Sprintf(lang, err.Error())
+			continue
+		}
 
 		if err := lst[i].Add(); err != nil {
-			reterr[lst[i].Name] = i18n.Sprintf(c.GetHeader("X-Language"), err.Error())
+			reterr[lst[i].Name] = i18n.Sprintf(lang, err.Error())
 		} else {
 			reterr[lst[i].Name] = ""
 		}
 	}
-
-	ginx.NewRender(c).Data(reterr, nil)
+	return reterr
 }
 
 func alertRuleDel(c *gin.Context) {
@@ -65,7 +104,7 @@ func alertRuleDel(c *gin.Context) {
 	ginx.NewRender(c).Message(models.AlertRuleDels(f.Ids, ginx.UrlParamInt64(c, "id")))
 }
 
-func alertRulePut(c *gin.Context) {
+func alertRulePutByFE(c *gin.Context) {
 	var f models.AlertRule
 	ginx.BindJSON(c, &f)
 
@@ -84,6 +123,21 @@ func alertRulePut(c *gin.Context) {
 	ginx.NewRender(c).Message(ar.Update(f))
 }
 
+func alertRulePutByService(c *gin.Context) {
+	var f models.AlertRule
+	ginx.BindJSON(c, &f)
+
+	arid := ginx.UrlParamInt64(c, "arid")
+	ar, err := models.AlertRuleGetById(arid)
+	ginx.Dangerous(err)
+
+	if ar == nil {
+		ginx.NewRender(c, http.StatusNotFound).Message("No such AlertRule")
+		return
+	}
+	ginx.NewRender(c).Message(ar.Update(f))
+}
+
 type alertRuleFieldForm struct {
 	Ids    []int64                `json:"ids"`
 	Fields map[string]interface{} `json:"fields"`

src/webapi/router/router_board.go

+package router
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/didi/nightingale/v5/src/models"
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+)
+
+type boardForm struct {
+	Name    string `json:"name"`
+	Tags    string `json:"tags"`
+	Configs string `json:"configs"`
+}
+
+func boardAdd(c *gin.Context) {
+	var f boardForm
+	ginx.BindJSON(c, &f)
+
+	me := c.MustGet("user").(*models.User)
+
+	board := &models.Board{
+		GroupId:  ginx.UrlParamInt64(c, "id"),
+		Name:     f.Name,
+		Tags:     f.Tags,
+		Configs:  f.Configs,
+		CreateBy: me.Username,
+		UpdateBy: me.Username,
+	}
+
+	err := board.Add()
+	ginx.Dangerous(err)
+
+	if f.Configs != "" {
+		ginx.Dangerous(models.BoardPayloadSave(board.Id, f.Configs))
+	}
+
+	ginx.NewRender(c).Data(board, nil)
+}
+
+func boardGet(c *gin.Context) {
+	board, err := models.BoardGet("id = ?", ginx.UrlParamInt64(c, "bid"))
+	ginx.Dangerous(err)
+
+	if board == nil {
+		ginx.Bomb(http.StatusNotFound, "No such dashboard")
+	}
+
+	ginx.NewRender(c).Data(board, nil)
+}
+
+// bgrwCheck
+func boardDel(c *gin.Context) {
+	var f idsForm
+	ginx.BindJSON(c, &f)
+	f.Verify()
+
+	for i := 0; i < len(f.Ids); i++ {
+		bid := f.Ids[i]
+
+		board, err := models.BoardGet("id = ?", bid)
+		ginx.Dangerous(err)
+
+		if board == nil {
+			continue
+		}
+
+		// check permission
+		bgrwCheck(c, board.GroupId)
+
+		ginx.Dangerous(board.Del())
+	}
+
+	ginx.NewRender(c).Message(nil)
+}
+
+func Board(id int64) *models.Board {
+	obj, err := models.BoardGet("id=?", id)
+	ginx.Dangerous(err)
+
+	if obj == nil {
+		ginx.Bomb(http.StatusNotFound, "No such dashboard")
+	}
+
+	return obj
+}
+
+// bgrwCheck
+func boardPut(c *gin.Context) {
+	var f boardForm
+	ginx.BindJSON(c, &f)
+
+	me := c.MustGet("user").(*models.User)
+	bo := Board(ginx.UrlParamInt64(c, "bid"))
+
+	// check permission
+	bgrwCheck(c, bo.GroupId)
+
+	bo.Name = f.Name
+	bo.Tags = f.Tags
+	bo.UpdateBy = me.Username
+	bo.UpdateAt = time.Now().Unix()
+
+	err := bo.Update("name", "tags", "update_by", "update_at")
+	ginx.NewRender(c).Data(bo, err)
+}
+
+// bgrwCheck
+func boardPutConfigs(c *gin.Context) {
+	var f boardForm
+	ginx.BindJSON(c, &f)
+
+	me := c.MustGet("user").(*models.User)
+	bo := Board(ginx.UrlParamInt64(c, "bid"))
+
+	// check permission
+	bgrwCheck(c, bo.GroupId)
+
+	bo.UpdateBy = me.Username
+	bo.UpdateAt = time.Now().Unix()
+	ginx.Dangerous(bo.Update("update_by", "update_at"))
+
+	bo.Configs = f.Configs
+	ginx.Dangerous(models.BoardPayloadSave(bo.Id, f.Configs))
+
+	ginx.NewRender(c).Data(bo, nil)
+}
+
+func boardGets(c *gin.Context) {
+	bgid := ginx.UrlParamInt64(c, "id")
+	query := ginx.QueryStr(c, "query", "")
+
+	boards, err := models.BoardGets(bgid, query)
+	ginx.NewRender(c).Data(boards, err)
+}
+
+func boardClone(c *gin.Context) {
+	me := c.MustGet("user").(*models.User)
+	bo := Board(ginx.UrlParamInt64(c, "bid"))
+
+	newBoard := &models.Board{
+		Name:     bo.Name + " Copy",
+		Tags:     bo.Tags,
+		GroupId:  bo.GroupId,
+		CreateBy: me.Username,
+		UpdateBy: me.Username,
+	}
+
+	ginx.Dangerous(newBoard.Add())
+
+	// clone payload
+	payload, err := models.BoardPayloadGet(bo.Id)
+	ginx.Dangerous(err)
+
+	if payload != "" {
+		ginx.Dangerous(models.BoardPayloadSave(newBoard.Id, payload))
+	}
+
+	ginx.NewRender(c).Message(nil)
+}
+
+// ---- migrate ----
+
+func migrateDashboards(c *gin.Context) {
+	lst, err := models.DashboardGetAll()
+	ginx.NewRender(c).Data(lst, err)
+}
+
+func migrateDashboardGet(c *gin.Context) {
+	dash := Dashboard(ginx.UrlParamInt64(c, "id"))
+	ginx.NewRender(c).Data(dash, nil)
+}
+
+func migrateDashboard(c *gin.Context) {
+	dash := Dashboard(ginx.UrlParamInt64(c, "id"))
+
+	var f boardForm
+	ginx.BindJSON(c, &f)
+
+	me := c.MustGet("user").(*models.User)
+
+	board := &models.Board{
+		GroupId:  dash.GroupId,
+		Name:     f.Name,
+		Tags:     f.Tags,
+		Configs:  f.Configs,
+		CreateBy: me.Username,
+		UpdateBy: me.Username,
+	}
+
+	ginx.Dangerous(board.Add())
+
+	if board.Configs != "" {
+		ginx.Dangerous(models.BoardPayloadSave(board.Id, board.Configs))
+	}
+
+	ginx.NewRender(c).Message(dash.Del())
+}

src/webapi/router/router_builtin.go

@@ -75,7 +75,11 @@ func alertRuleBuiltinImport(c *gin.Context) {
 		lst[i].GroupId = bgid
 		lst[i].CreateBy = username
 		lst[i].UpdateBy = username
-		lst[i].FE2DB()
+
+		if err := lst[i].FE2DB(); err != nil {
+			reterr[lst[i].Name] = i18n.Sprintf(c.GetHeader("X-Language"), err.Error())
+			continue
+		}
 
 		if err := lst[i].Add(); err != nil {
 			reterr[lst[i].Name] = i18n.Sprintf(c.GetHeader("X-Language"), err.Error())
@@ -87,7 +91,7 @@ func alertRuleBuiltinImport(c *gin.Context) {
 	ginx.NewRender(c).Data(reterr, nil)
 }
 
-func dashboardBuiltinList(c *gin.Context) {
+func builtinBoardGets(c *gin.Context) {
 	fp := config.C.BuiltinDashboardsDir
 	if fp == "" {
 		fp = path.Join(runner.Cwd, "etc", "dashboards")
@@ -110,6 +114,25 @@ func dashboardBuiltinList(c *gin.Context) {
 	ginx.NewRender(c).Data(names, nil)
 }
 
+// read the json file content
+func builtinBoardGet(c *gin.Context) {
+	name := ginx.UrlParamStr(c, "name")
+	dirpath := config.C.BuiltinDashboardsDir
+	if dirpath == "" {
+		dirpath = path.Join(runner.Cwd, "etc", "dashboards")
+	}
+
+	jsonfile := path.Join(dirpath, name+".json")
+	if !file.IsExist(jsonfile) {
+		ginx.Bomb(http.StatusBadRequest, "%s not found", jsonfile)
+	}
+
+	body, err := file.ReadString(jsonfile)
+	ginx.NewRender(c).Data(body, err)
+}
+
+// deprecated ↓
+
 type dashboardBuiltinImportForm struct {
 	Name string `json:"name" binding:"required"`
 }

src/webapi/router/router_mw.go

@@ -153,6 +153,20 @@ func bgrwChecks(c *gin.Context, bgids []int64) {
 	}
 }
 
+func bgroCheck(c *gin.Context, bgid int64) {
+	me := c.MustGet("user").(*models.User)
+	bg := BusiGroup(bgid)
+
+	can, err := me.CanDoBusiGroup(bg, "ro")
+	ginx.Dangerous(err)
+
+	if !can {
+		ginx.Bomb(http.StatusForbidden, "forbidden")
+	}
+
+	c.Set("busi_group", bg)
+}
+
 func perm(operation string) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		me := c.MustGet("user").(*models.User)

src/webapi/router/router_target.go

 package router
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -48,7 +49,11 @@ type targetTagsForm struct {
 	Tags   []string `json:"tags" binding:"required"`
 }
 
-func targetBindTags(c *gin.Context) {
+func (t targetTagsForm) Verify() {
+
+}
+
+func targetBindTagsByFE(c *gin.Context) {
 	var f targetTagsForm
 	ginx.BindJSON(c, &f)
 
@@ -58,33 +63,49 @@ func targetBindTags(c *gin.Context) {
 
 	checkTargetPerm(c, f.Idents)
 
-	// verify
+	ginx.NewRender(c).Message(targetBindTags(f))
+}
+
+func targetBindTagsByService(c *gin.Context) {
+	var f targetTagsForm
+	ginx.BindJSON(c, &f)
+
+	if len(f.Idents) == 0 {
+		ginx.Bomb(http.StatusBadRequest, "idents empty")
+	}
+
+	ginx.NewRender(c).Message(targetBindTags(f))
+}
+
+func targetBindTags(f targetTagsForm) error {
 	for i := 0; i < len(f.Tags); i++ {
 		arr := strings.Split(f.Tags[i], "=")
 		if len(arr) != 2 {
-			ginx.Bomb(200, "invalid tag(%s)", f.Tags[i])
+			return fmt.Errorf("invalid tag(%s)", f.Tags[i])
 		}
 
 		if strings.TrimSpace(arr[0]) == "" || strings.TrimSpace(arr[1]) == "" {
-			ginx.Bomb(200, "invalid tag(%s)", f.Tags[i])
+			return fmt.Errorf("invalid tag(%s)", f.Tags[i])
 		}
 
 		if strings.IndexByte(arr[0], '.') != -1 {
-			ginx.Bomb(200, "invalid tagkey(%s): cannot contains .", arr[0])
+			return fmt.Errorf("invalid tagkey(%s): cannot contains . ", arr[0])
 		}
 
 		if strings.IndexByte(arr[0], '-') != -1 {
-			ginx.Bomb(200, "invalid tagkey(%s): cannot contains -", arr[0])
+			return fmt.Errorf("invalid tagkey(%s): cannot contains -", arr[0])
 		}
 
 		if !model.LabelNameRE.MatchString(arr[0]) {
-			ginx.Bomb(200, "invalid tagkey(%s)", arr[0])
+			return fmt.Errorf("invalid tagkey(%s)", arr[0])
 		}
 	}
 
 	for i := 0; i < len(f.Idents); i++ {
 		target, err := models.TargetGetByIdent(f.Idents[i])
-		ginx.Dangerous(err)
+		if err != nil {
+			return err
+		}
 
 		if target == nil {
 			continue
@@ -95,18 +116,19 @@ func targetBindTags(c *gin.Context) {
 			tagkey := strings.Split(f.Tags[j], "=")[0]
 			tagkeyPrefix := tagkey + "="
 			if strings.HasPrefix(target.Tags, tagkeyPrefix) {
-				ginx.NewRender(c).Message("duplicate tagkey(%s)", tagkey)
-				return
+				return fmt.Errorf("duplicate tagkey(%s)", tagkey)
 			}
 		}
 
-		ginx.Dangerous(target.AddTags(f.Tags))
+		err = target.AddTags(f.Tags)
+		if err != nil {
+			return err
+		}
 	}
-
-	ginx.NewRender(c).Message(nil)
+	return nil
 }
 
-func targetUnbindTags(c *gin.Context) {
+func targetUnbindTagsByFE(c *gin.Context) {
 	var f targetTagsForm
 	ginx.BindJSON(c, &f)
 
@@ -116,18 +138,37 @@ func targetUnbindTags(c *gin.Context) {
 
 	checkTargetPerm(c, f.Idents)
 
+	ginx.NewRender(c).Message(targetUnbindTags(f))
+}
+
+func targetUnbindTagsByService(c *gin.Context) {
+	var f targetTagsForm
+	ginx.BindJSON(c, &f)
+
+	if len(f.Idents) == 0 {
+		ginx.Bomb(http.StatusBadRequest, "idents empty")
+	}
+
+	ginx.NewRender(c).Message(targetUnbindTags(f))
+}
+
+func targetUnbindTags(f targetTagsForm) error {
 	for i := 0; i < len(f.Idents); i++ {
 		target, err := models.TargetGetByIdent(f.Idents[i])
-		ginx.Dangerous(err)
+		if err != nil {
+			return err
+		}
 
 		if target == nil {
 			continue
 		}
 
-		ginx.Dangerous(target.DelTags(f.Tags))
+		err = target.DelTags(f.Tags)
+		if err != nil {
+			return err
+		}
 	}
-
-	ginx.NewRender(c).Message(nil)
+	return nil
 }
 
 type targetNoteForm struct {
@@ -148,6 +189,17 @@ func targetUpdateNote(c *gin.Context) {
 	ginx.NewRender(c).Message(models.TargetUpdateNote(f.Idents, f.Note))
 }
 
+func targetUpdateNoteByService(c *gin.Context) {
+	var f targetNoteForm
+	ginx.BindJSON(c, &f)
+
+	if len(f.Idents) == 0 {
+		ginx.Bomb(http.StatusBadRequest, "idents empty")
+	}
+
+	ginx.NewRender(c).Message(models.TargetUpdateNote(f.Idents, f.Note))
+}
+
 type targetBgidForm struct {
 	Idents []string `json:"idents" binding:"required"`
 	Bgid   int64    `json:"bgid"`

src/webapi/webapi.go

@@ -101,11 +101,7 @@ func (a Webapi) initialize() (func(), error) {
 	}
 
 	// init database
-	if err = storage.InitDB(storage.DBConfig{
-		Gorm:     config.C.Gorm,
-		MySQL:    config.C.MySQL,
-		Postgres: config.C.Postgres,
-	}); err != nil {
+	if err = storage.InitDB(config.C.DB); err != nil {
 		return nil, err
 	}