Add coll and segState lebels to DataCoordNumStoredRows monitoring (#26299)

Signed-off-by: Nbigsheeper <yihao.dai@zilliz.com>

internal/datacoord/meta.go

@@ -111,7 +111,6 @@ func (m *meta) reloadFromKV() error {
 	metrics.DataCoordNumSegments.WithLabelValues(metrics.FlushedSegmentLabel).Set(0)
 	metrics.DataCoordNumSegments.WithLabelValues(metrics.FlushingSegmentLabel).Set(0)
 	metrics.DataCoordNumSegments.WithLabelValues(metrics.DroppedSegmentLabel).Set(0)
-	metrics.DataCoordNumStoredRows.WithLabelValues().Set(0)
 	numStoredRows := int64(0)
 	for _, segment := range segments {
 		m.segments.SetSegment(segment.ID, NewSegmentInfo(segment))
@@ -138,7 +137,6 @@ func (m *meta) reloadFromKV() error {
 			metrics.FlushedSegmentFileNum.WithLabelValues(metrics.DeleteFileLabel).Observe(float64(deleteFileNum))
 		}
 	}
-	metrics.DataCoordNumStoredRows.WithLabelValues().Set(float64(numStoredRows))
 	metrics.DataCoordNumStoredRowsCounter.WithLabelValues().Add(float64(numStoredRows))
 
 	channelCPs, err := m.catalog.ListChannelCheckpoint(m.ctx)
@@ -274,6 +272,7 @@ func (m *meta) GetCollectionBinlogSize() (int64, map[UniqueID]int64) {
 	m.RLock()
 	defer m.RUnlock()
 	collectionBinlogSize := make(map[UniqueID]int64)
+	collectionRowsNum := make(map[UniqueID]map[commonpb.SegmentState]int64)
 	segments := m.segments.GetSegments()
 	var total int64
 	for _, segment := range segments {
@@ -283,6 +282,15 @@ func (m *meta) GetCollectionBinlogSize() (int64, map[UniqueID]int64) {
 			collectionBinlogSize[segment.GetCollectionID()] += segmentSize
 			metrics.DataCoordStoredBinlogSize.WithLabelValues(
 				fmt.Sprint(segment.GetCollectionID()), fmt.Sprint(segment.GetID())).Set(float64(segmentSize))
+			if _, ok := collectionRowsNum[segment.GetCollectionID()]; !ok {
+				collectionRowsNum[segment.GetCollectionID()] = make(map[commonpb.SegmentState]int64)
+			}
+			collectionRowsNum[segment.GetCollectionID()][segment.GetState()] += segment.GetNumOfRows()
+		}
+	}
+	for collection, statesRows := range collectionRowsNum {
+		for state, rows := range statesRows {
+			metrics.DataCoordNumStoredRows.WithLabelValues(fmt.Sprint(collection), state.String()).Set(float64(rows))
 		}
 	}
 	return total, collectionBinlogSize
@@ -1316,7 +1324,6 @@ func (s *segMetricMutation) commit() {
 	for state, change := range s.stateChange {
 		metrics.DataCoordNumSegments.WithLabelValues(state).Add(float64(change))
 	}
-	metrics.DataCoordNumStoredRows.WithLabelValues().Add(float64(s.rowCountChange))
 	metrics.DataCoordNumStoredRowsCounter.WithLabelValues().Add(float64(s.rowCountAccChange))
 }
 

internal/datacoord/services.go

@@ -527,6 +527,7 @@ func (s *Server) DropVirtualChannel(ctx context.Context, req *datapb.DropVirtual
 		return resp, nil
 	}
 
+	var collectionID int64
 	segments := make([]*SegmentInfo, 0, len(req.GetSegments()))
 	for _, seg2Drop := range req.GetSegments() {
 		info := &datapb.SegmentInfo{
@@ -542,6 +543,7 @@ func (s *Server) DropVirtualChannel(ctx context.Context, req *datapb.DropVirtual
 		}
 		segment := NewSegmentInfo(info)
 		segments = append(segments, segment)
+		collectionID = seg2Drop.GetCollectionID()
 	}
 
 	err := s.meta.UpdateDropChannelSegmentInfo(channel, segments)
@@ -558,6 +560,8 @@ func (s *Server) DropVirtualChannel(ctx context.Context, req *datapb.DropVirtual
 	}
 	s.segmentManager.DropSegmentsOfChannel(ctx, channel)
 
+	metrics.CleanupDataCoordNumStoredRows(collectionID)
+
 	// no compaction triggered in Drop procedure
 	resp.Status.ErrorCode = commonpb.ErrorCode_Success
 	return resp, nil

pkg/metrics/datacoord_metrics.go

@@ -19,9 +19,9 @@ package metrics
 import (
 	"fmt"
 
-	"github.com/prometheus/client_golang/prometheus"
-
+	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -74,8 +74,11 @@ var (
 			Namespace: milvusNamespace,
 			Subsystem: typeutil.DataCoordRole,
 			Name:      "stored_rows_num",
-			Help:      "number of stored rows",
-		}, []string{})
+			Help:      "number of stored rows of healthy segment",
+		}, []string{
+			collectionIDLabelName,
+			segmentStateLabelName,
+		})
 
 	DataCoordNumStoredRowsCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
@@ -248,3 +251,12 @@ func CleanupDataCoordSegmentMetrics(collectionID int64, segmentID int64) {
 		segmentIDLabelName:    fmt.Sprint(segmentID),
 	})
 }
+
+func CleanupDataCoordNumStoredRows(collectionID int64) {
+	for _, state := range commonpb.SegmentState_name {
+		DataCoordNumStoredRows.Delete(prometheus.Labels{
+			collectionIDLabelName: fmt.Sprint(collectionID),
+			segmentStateLabelName: fmt.Sprint(state),
+		})
+	}
+}