package metricsserver import ( "fmt" "strings" "testing" "time" "io/ioutil" "github.com/google/go-cmp/cmp" jsoniter "github.com/json-iterator/go" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/informers" fakek8s "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" metricsV1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" fakemetricsclient "k8s.io/metrics/pkg/client/clientset/versioned/fake" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" ) // mergeResourceLists will merge resoure lists. When two lists have the same resourece, the value from // the last list will be present in the result func mergeResourceLists(resourceLists ...v1.ResourceList) v1.ResourceList { result := v1.ResourceList{} for _, rl := range resourceLists { for resource, quantity := range rl { result[resource] = quantity } } return result } func getResourceList(cpu, memory string) v1.ResourceList { res := v1.ResourceList{} if cpu != "" { res[v1.ResourceCPU] = resource.MustParse(cpu) } if memory != "" { res[v1.ResourceMemory] = resource.MustParse(memory) } return res } var nodeCapacity = mergeResourceLists(getResourceList("8", "8Gi")) var node1 = &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "edgenode-1", Labels: map[string]string{ "node-role.kubernetes.io/edge": "", }, }, Status: v1.NodeStatus{ Capacity: nodeCapacity, }, } var node2 = &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "edgenode-2", Labels: map[string]string{ "node-role.kubernetes.io/edge": "", }, }, Status: v1.NodeStatus{ Capacity: nodeCapacity, }, } var pod1 = &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "kubeedge", }, } var pod2 = &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "kubeedge", }, } const ( layout = "2006-01-02T15:04:05.000Z" str = "2021-03-25T12:34:56.789Z" ) var ( metricsTime, _ = time.Parse(layout, str) nodeMetric1 = metricsV1beta1.NodeMetrics{ ObjectMeta: metav1.ObjectMeta{ Name: "edgenode-1", Labels: map[string]string{ "node-role.kubernetes.io/edge": "", }, }, Timestamp: metav1.Time{Time: metricsTime}, Window: metav1.Duration{Duration: time.Minute}, Usage: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity( int64(1000), resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity( int64(1024*1024), resource.BinarySI), }, } nodeMetric2 = metricsV1beta1.NodeMetrics{ ObjectMeta: metav1.ObjectMeta{ Name: "edgenode-2", Labels: map[string]string{ "node-role.kubernetes.io/edge": "", }, }, Timestamp: metav1.Time{Time: metricsTime}, Window: metav1.Duration{Duration: time.Minute}, Usage: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity( int64(2000), resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity( int64(2*1024*1024), resource.BinarySI), }, } podMetric1 = metricsV1beta1.PodMetrics{ TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "kubeedge", }, Timestamp: metav1.Time{Time: metricsTime}, Window: metav1.Duration{Duration: time.Minute}, Containers: []metricsV1beta1.ContainerMetrics{ metricsV1beta1.ContainerMetrics{ Name: "containers-1", Usage: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity( 1, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity( int64(1024*1024), resource.DecimalSI), }, }, }, } podMetric2 = metricsV1beta1.PodMetrics{ TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "kubeedge", }, Timestamp: metav1.Time{Time: metricsTime}, Window: metav1.Duration{Duration: time.Minute}, Containers: []metricsV1beta1.ContainerMetrics{ metricsV1beta1.ContainerMetrics{ Name: "containers-1", Usage: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity( 1, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity( int64(1024*1024), resource.DecimalSI), }, }, metricsV1beta1.ContainerMetrics{ Name: "containers-2", Usage: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity( 1, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity( int64(1024*1024), resource.DecimalSI), }, }, }, } ) func TestGetNamedMetrics(t *testing.T) { nodeMetricsTests := []struct { metrics []string filter string expected string }{ { metrics: []string{"node_cpu_usage", "node_memory_usage_wo_cache"}, filter: ".*", expected: "metrics-vector-1.json", }, { metrics: []string{"node_cpu_usage", "node_cpu_utilisation"}, filter: "edgenode-2", expected: "metrics-vector-2.json", }, { metrics: []string{"node_memory_usage_wo_cache", "node_memory_utilisation"}, filter: "edgenode-1|edgenode-2", expected: "metrics-vector-3.json", }, } podMetricsTests := []struct { metrics []string filter string expected string podName string namespaceName string }{ { metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"}, filter: "pod1$", expected: "metrics-vector-4.json", podName: "", namespaceName: "", }, { metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"}, filter: "default/pod2$", expected: "metrics-vector-5.json", podName: "", namespaceName: "", }, { metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"}, filter: "", expected: "metrics-vector-6.json", podName: "pod1", namespaceName: "kubeedge", }, } fakeK8sClient := fakek8s.NewSimpleClientset(node1, node2, pod1, pod2) informer := informers.NewSharedInformerFactory(fakeK8sClient, 0) informer.Core().V1().Nodes().Informer().GetIndexer().Add(node1) informer.Core().V1().Nodes().Informer().GetIndexer().Add(node2) informer.Core().V1().Pods().Informer().GetIndexer().Add(pod1) informer.Core().V1().Pods().Informer().GetIndexer().Add(pod2) fakeMetricsclient := &fakemetricsclient.Clientset{} fakeMetricsclient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) { metrics := &metricsV1beta1.NodeMetricsList{} metrics.Items = append(metrics.Items, nodeMetric1) metrics.Items = append(metrics.Items, nodeMetric2) return true, metrics, nil }) fakeMetricsclient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { metrics := &metricsV1beta1.PodMetricsList{} metrics.Items = append(metrics.Items, podMetric1) metrics.Items = append(metrics.Items, podMetric2) return true, metrics, nil }) // test for node edge for i, tt := range nodeMetricsTests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { expected := make([]monitoring.Metric, 0) err := jsonFromFile(tt.expected, &expected) if err != nil { t.Fatal(err) } client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient) result := client.GetNamedMetrics(tt.metrics, time.Now(), monitoring.NodeOption{ResourceFilter: tt.filter}) if diff := cmp.Diff(result, expected); diff != "" { t.Fatalf("%T differ (-got, +want): %s", expected, diff) } }) } //test for pods on the node edges for i, tt := range podMetricsTests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { expected := make([]monitoring.Metric, 0) err := jsonFromFile(tt.expected, &expected) if err != nil { t.Fatal(err) } if tt.podName == "pod1" || strings.Contains(tt.filter, "pod1") { fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { return true, &podMetric1, nil }) } else { fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { return true, &podMetric2, nil }) } client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient) result := client.GetNamedMetrics(tt.metrics, time.Now(), monitoring.PodOption{ResourceFilter: tt.filter, PodName: tt.podName, NamespaceName: tt.namespaceName}) if diff := cmp.Diff(result, expected); diff != "" { t.Fatalf("%T differ (-got, +want): %s", expected, diff) } }) } } func TestGetNamedMetricsOverTime(t *testing.T) { nodeMetricsTests := []struct { metrics []string filter string expected string }{ { metrics: []string{"node_cpu_usage", "node_memory_usage_wo_cache"}, filter: ".*", expected: "metrics-matrix-1.json", }, { metrics: []string{"node_cpu_usage", "node_cpu_utilisation"}, filter: "edgenode-2", expected: "metrics-matrix-2.json", }, { metrics: []string{"node_memory_usage_wo_cache", "node_memory_utilisation"}, filter: "edgenode-1|edgenode-2", expected: "metrics-matrix-3.json", }, } podMetricsTests := []struct { metrics []string filter string expected string podName string namespaceName string }{ { metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"}, filter: "pod1$", expected: "metrics-matrix-4.json", podName: "", namespaceName: "", }, { metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"}, filter: "default/pod2$", expected: "metrics-matrix-5.json", podName: "", namespaceName: "", }, { metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"}, filter: "", expected: "metrics-matrix-6.json", podName: "pod1", namespaceName: "kubeedge", }, } fakeK8sClient := fakek8s.NewSimpleClientset(node1, node2, pod1, pod2) informer := informers.NewSharedInformerFactory(fakeK8sClient, 0) informer.Core().V1().Nodes().Informer().GetIndexer().Add(node1) informer.Core().V1().Nodes().Informer().GetIndexer().Add(node2) informer.Core().V1().Nodes().Informer().GetIndexer().Add(pod1) informer.Core().V1().Nodes().Informer().GetIndexer().Add(pod2) fakeMetricsclient := &fakemetricsclient.Clientset{} fakeMetricsclient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) { metrics := &metricsV1beta1.NodeMetricsList{} metrics.Items = append(metrics.Items, nodeMetric1) metrics.Items = append(metrics.Items, nodeMetric2) return true, metrics, nil }) fakeMetricsclient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { metrics := &metricsV1beta1.PodMetricsList{} metrics.Items = append(metrics.Items, podMetric1) metrics.Items = append(metrics.Items, podMetric2) return true, metrics, nil }) for i, tt := range nodeMetricsTests { fakeMetricsclient.Fake.ClearActions() t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { expected := make([]monitoring.Metric, 0) err := jsonFromFile(tt.expected, &expected) if err != nil { t.Fatal(err) } client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient) result := client.GetNamedMetricsOverTime(tt.metrics, time.Now().Add(-time.Minute*3), time.Now(), time.Minute, monitoring.NodeOption{ResourceFilter: tt.filter}) if diff := cmp.Diff(result, expected); diff != "" { t.Fatalf("%T differ (-got, +want): %s", expected, diff) } }) } for i, tt := range podMetricsTests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { expected := make([]monitoring.Metric, 0) err := jsonFromFile(tt.expected, &expected) if err != nil { t.Fatal(err) } if tt.podName == "pod1" || strings.Contains(tt.filter, "pod1") { fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { return true, &podMetric1, nil }) } else { fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { return true, &podMetric2, nil }) } client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient) result := client.GetNamedMetricsOverTime(tt.metrics, time.Now().Add(-time.Minute*3), time.Now(), time.Minute, monitoring.PodOption{ResourceFilter: tt.filter, PodName: tt.podName, NamespaceName: tt.namespaceName}) if diff := cmp.Diff(result, expected); diff != "" { t.Fatalf("%T differ (-got, +want): %s", expected, diff) } }) } } func jsonFromFile(expectedFile string, expectedJsonPtr interface{}) error { json, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", expectedFile)) if err != nil { return err } err = jsoniter.Unmarshal(json, expectedJsonPtr) if err != nil { return err } return nil }