From 4559c49e86d12f2e62373e19868c4095ff12b421 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Fri, 31 Aug 2018 20:01:01 +0800 Subject: [PATCH] Change alarm notification to batch mode. --- .../server/core/alarm/provider/AlarmCore.java | 16 +++-- .../core/alarm/provider/RunningRule.java | 13 ++-- .../core/alarm/provider/WebhookCallback.java | 2 +- .../core/alarm/provider/RunningRuleTest.java | 62 ++++++------------- .../oap/server/core/alarm/AlarmCallback.java | 4 +- .../oap/server/core/alarm/AlarmMessage.java | 5 ++ .../core/alarm/AlarmStandardPersistence.java | 4 +- 7 files changed, 50 insertions(+), 56 deletions(-) diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java index 40d62bc210..5d02b5556e 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.skywalking.oap.server.core.alarm.AlarmCallback; +import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; import org.joda.time.LocalDateTime; import org.joda.time.Minutes; @@ -61,8 +62,9 @@ public class AlarmCore { public void start(List allCallbacks) { LocalDateTime now = LocalDateTime.now(); lastExecuteTime = now; - runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> runningRule.start(now, allCallbacks))); - Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> + runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> runningRule.start(now))); + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { + List alarmMessageList = new ArrayList<>(30); runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> { LocalDateTime checkTime = LocalDateTime.now(); int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes(); @@ -72,11 +74,17 @@ public class AlarmCore { * Don't run in the first quarter per min, avoid to trigger alarm. */ if (checkTime.getSecondOfMinute() > 15) { - runningRule.check(); + AlarmMessage alarmMessage = runningRule.check(); + if (alarmMessage != AlarmMessage.NONE) { + alarmMessageList.add(alarmMessage); + } // Set the last execute time, and make sure the second is `00`, such as: 18:30:00 lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute()); } } - })), 10, 10, TimeUnit.SECONDS); + })); + + allCallbacks.forEach(callback -> callback.doAlarm(alarmMessageList)); + }, 10, 10, TimeUnit.SECONDS); } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java index e13be47bf4..2d3d86f97b 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java @@ -55,7 +55,6 @@ public class RunningRule { private Window window; private volatile boolean isStarted = false; private volatile IndicatorValueType valueType; - private volatile List allCallbacks; private Scope targetScope; public RunningRule(AlarmRule alarmRule) { @@ -108,8 +107,7 @@ public class RunningRule { * * @param current */ - public void start(LocalDateTime current, List allCallbacks) { - this.allCallbacks = allCallbacks; + public void start(LocalDateTime current) { window.start(current); isStarted = true; } @@ -126,7 +124,7 @@ public class RunningRule { /** * Check the conditions, decide to whether trigger alarm. */ - public void check() { + public AlarmMessage check() { boolean isMatched = window.isMatch(); /** @@ -138,7 +136,7 @@ public class RunningRule { if (isMatched) { counter++; if (counter >= countThreshold && silenceCountdown < 1) { - triggerAlarm(); + return triggerAlarm(); } else { silenceCountdown--; } @@ -148,15 +146,16 @@ public class RunningRule { counter--; } } + return AlarmMessage.NONE; } /** * Trigger alarm callbacks. */ - private void triggerAlarm() { + private AlarmMessage triggerAlarm() { silenceCountdown = silencePeriod; AlarmMessage message = new AlarmMessage(); - allCallbacks.forEach(callback -> callback.doAlarm(message)); + return message; } /** diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java index 4ec9a40e28..7655b42292 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java @@ -32,7 +32,7 @@ public class WebhookCallback implements AlarmCallback { this.remoteEndpoints = remoteEndpoints; } - @Override public void doAlarm(AlarmMessage alarmMessage) { + @Override public void doAlarm(List alarmMessage) { } } diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java index 2c1f2da8f0..e5695c7087 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.core.alarm.provider; import java.util.LinkedList; +import java.util.List; import org.apache.skywalking.oap.server.core.alarm.AlarmCallback; import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; import org.apache.skywalking.oap.server.core.alarm.MetaInAlarm; @@ -55,7 +56,7 @@ public class RunningRuleTest { RunningRule runningRule = new RunningRule(alarmRule); LocalDateTime startTime = TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301440"); - runningRule.start(startTime, new LinkedList<>()); + runningRule.start(startTime); RunningRule.Window window = Whitebox.getInternalState(runningRule, "window"); LocalDateTime endTime = Whitebox.getInternalState(window, "endTime"); @@ -80,15 +81,7 @@ public class RunningRuleTest { RunningRule runningRule = new RunningRule(alarmRule); LocalDateTime startTime = TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301440"); - final boolean[] isAlarm = {false}; - AlarmCallback assertCallback = new AlarmCallback() { - @Override public void doAlarm(AlarmMessage alarmMessage) { - isAlarm[0] = true; - } - }; - LinkedList callbackList = new LinkedList<>(); - callbackList.add(assertCallback); - runningRule.start(startTime, callbackList); + runningRule.start(startTime); long timeInPeriod1 = 201808301434L; long timeInPeriod2 = 201808301436L; @@ -98,15 +91,14 @@ public class RunningRuleTest { runningRule.in(getMetaInAlarm(), getIndicator(timeInPeriod3, 74)); // check at 201808301440 - runningRule.check(); + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); runningRule.moveTo(TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301441")); // check at 201808301441 - runningRule.check(); + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); runningRule.moveTo(TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301442")); // check at 201808301442 - runningRule.check(); + Assert.assertNotEquals(AlarmMessage.NONE, runningRule.check()); - Assert.assertTrue(isAlarm[0]); } @Test @@ -125,13 +117,13 @@ public class RunningRuleTest { final boolean[] isAlarm = {false}; AlarmCallback assertCallback = new AlarmCallback() { - @Override public void doAlarm(AlarmMessage alarmMessage) { + @Override public void doAlarm(List alarmMessage) { isAlarm[0] = true; } }; LinkedList callbackList = new LinkedList<>(); callbackList.add(assertCallback); - runningRule.start(startTime, callbackList); + runningRule.start(startTime); long timeInPeriod1 = 201808301434L; long timeInPeriod2 = 201808301436L; @@ -145,15 +137,13 @@ public class RunningRuleTest { runningRule.in(getMetaInAlarm(), getIndicator(timeInPeriod5, 95)); // check at 201808301440 - runningRule.check(); + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); runningRule.moveTo(TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301442")); // check at 201808301441 - runningRule.check(); + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); runningRule.moveTo(TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301443")); // check at 201808301442 - runningRule.check(); - - Assert.assertFalse((boolean)isAlarm[0]); + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); } @Test @@ -170,16 +160,7 @@ public class RunningRuleTest { RunningRule runningRule = new RunningRule(alarmRule); LocalDateTime startTime = TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301440"); - final Object[] isAlarm = {false, 0}; - AlarmCallback assertCallback = new AlarmCallback() { - @Override public void doAlarm(AlarmMessage alarmMessage) { - isAlarm[0] = true; - isAlarm[1] = (int)isAlarm[1] + 1; - } - }; - LinkedList callbackList = new LinkedList<>(); - callbackList.add(assertCallback); - runningRule.start(startTime, callbackList); + runningRule.start(startTime); long timeInPeriod1 = 201808301434L; long timeInPeriod2 = 201808301436L; @@ -189,22 +170,19 @@ public class RunningRuleTest { runningRule.in(getMetaInAlarm(), getIndicator(timeInPeriod3, 74)); // check at 201808301440 - runningRule.check(); //check matches, no alarm + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); //check matches, no alarm runningRule.moveTo(TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301441")); // check at 201808301441 - runningRule.check(); //check matches, no alarm + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); //check matches, no alarm runningRule.moveTo(TIME_BUCKET_FORMATTER.parseLocalDateTime("201808301442")); // check at 201808301442 + Assert.assertNotEquals(AlarmMessage.NONE, runningRule.check()); //alarm + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); //silence, no alarm + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); //silence, no alarm runningRule.check(); //alarm - runningRule.check(); //silence, no alarm - runningRule.check(); //silence, no alarm - runningRule.check(); //alarm - runningRule.check(); //silence, no alarm - runningRule.check(); //silence, no alarm - runningRule.check(); //alarm - - Assert.assertTrue((boolean)isAlarm[0]); - Assert.assertEquals(3, (int)isAlarm[1]); + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); //silence, no alarm + Assert.assertEquals(AlarmMessage.NONE, runningRule.check()); //silence, no alarm + Assert.assertNotEquals(AlarmMessage.NONE, runningRule.check()); //alarm } private MetaInAlarm getMetaInAlarm() { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmCallback.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmCallback.java index 80a6193198..784d8f2dd5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmCallback.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmCallback.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.core.alarm; +import java.util.List; + /** * Alarm call back will be called by alarm implementor, * after it decided alarm should be sent. @@ -25,5 +27,5 @@ package org.apache.skywalking.oap.server.core.alarm; * @author wusheng */ public interface AlarmCallback { - void doAlarm(AlarmMessage alarmMessage); + void doAlarm(List alarmMessage); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmMessage.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmMessage.java index f7cf355119..7a8f28eb4a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmMessage.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmMessage.java @@ -30,4 +30,9 @@ import lombok.Setter; @Setter(AccessLevel.PUBLIC) @Getter(AccessLevel.PUBLIC) public class AlarmMessage { + public static AlarmMessage NONE = new NoAlarm(); + + private static class NoAlarm extends AlarmMessage { + + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java index 4a695ee4cf..0f3037b425 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java @@ -18,11 +18,13 @@ package org.apache.skywalking.oap.server.core.alarm; +import java.util.List; + /** * Save the alarm info into storage for UI query. */ public class AlarmStandardPersistence implements AlarmCallback { - @Override public void doAlarm(AlarmMessage alarmMessage) { + @Override public void doAlarm(List alarmMessage) { //TODO Peng-yongsheng /** * This is just a callback entrance. -- GitLab