perf: optimize Groovy-based DSL with static compilation

Groovy naturally supports many dynamic features that we don't benefit for now but cost performance loss, in this patch we compile our Groovy-based DSL scripts statically to optimize performance.

.github/workflows/e2e.log.yaml

@@ -23,6 +23,9 @@ on:
       - '!**.md'
   schedule:
     - cron: '0 18 * * *'
+  push:
+    branches:
+      - perf/groovy # TODO
 
 env:
   SW_AGENT_JDK_VERSION: 8

oap-server/analyzer/log-analyzer/pom.xml

@@ -42,6 +42,10 @@
             <groupId>org.codehaus.groovy</groupId>
             <artifactId>groovy</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
     </dependencies>
 
 </project>

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java

@@ -19,14 +19,18 @@
 package org.apache.skywalking.oap.log.analyzer.dsl;
 
 import groovy.lang.GroovyShell;
+import groovy.transform.CompileStatic;
 import groovy.util.DelegatingScript;
+import java.util.Collections;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.log.analyzer.dsl.spec.SkyWalkingDelegatingScript;
 import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
 import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.module.ModuleStartException;
 import org.codehaus.groovy.control.CompilerConfiguration;
+import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer;
 
 @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
 public class DSL {
@@ -38,7 +42,14 @@ public class DSL {
                          final LogAnalyzerModuleConfig config,
                          final String dsl) throws ModuleStartException {
         final CompilerConfiguration cc = new CompilerConfiguration();
-        cc.setScriptBaseClass(DelegatingScript.class.getName());
+        final ASTTransformationCustomizer customizer =
+            new ASTTransformationCustomizer(CompileStatic.class);
+        customizer.setAnnotationParameters(Collections.singletonMap(
+            "extensions",
+            Collections.singletonList(PrecompiledExtension.class.getName())
+        ));
+        cc.addCompilationCustomizers(customizer);
+        cc.setScriptBaseClass(SkyWalkingDelegatingScript.class.getName());
 
         final GroovyShell sh = new GroovyShell(cc);
         final DelegatingScript script = (DelegatingScript) sh.parse(dsl);

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/PrecompiledExtension.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.log.analyzer.dsl;
+
+import org.apache.skywalking.apm.network.logging.v3.LogData;
+import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
+import org.apache.skywalking.apm.network.logging.v3.LogTags;
+import org.apache.skywalking.apm.network.logging.v3.TraceContext;
+import org.codehaus.groovy.ast.expr.ConstantExpression;
+import org.codehaus.groovy.ast.expr.Expression;
+import org.codehaus.groovy.ast.expr.PropertyExpression;
+import org.codehaus.groovy.ast.expr.VariableExpression;
+import org.codehaus.groovy.transform.stc.AbstractTypeCheckingExtension;
+import org.codehaus.groovy.transform.stc.StaticTypeCheckingVisitor;
+
+import static org.codehaus.groovy.ast.ClassHelper.makeCached;
+
+public class PrecompiledExtension extends AbstractTypeCheckingExtension {
+
+    public PrecompiledExtension(final StaticTypeCheckingVisitor typeCheckingVisitor) {
+        super(typeCheckingVisitor);
+    }
+
+    @Override
+    public boolean handleUnresolvedProperty(final PropertyExpression pexp) {
+        final Expression exp = pexp.getObjectExpression();
+
+        if (exp.getText().startsWith("parsed")) {
+            makeDynamic(pexp);
+            setHandled(true);
+            return true;
+        }
+
+        if (exp.getText().startsWith("log")) {
+            if (handleLogVariable(pexp)) {
+                return true;
+            }
+        }
+
+        return super.handleUnresolvedProperty(pexp);
+    }
+
+    private boolean handleLogVariable(final PropertyExpression pexp) {
+        final Expression exp = pexp.getObjectExpression();
+        final Expression p = pexp.getProperty();
+
+        if (exp instanceof VariableExpression) {
+            final VariableExpression v = (VariableExpression) exp;
+            if (v.getName().equals("log")) {
+                storeType(v, makeCached(LogData.Builder.class));
+            }
+            if (p instanceof ConstantExpression) {
+                final ConstantExpression c = (ConstantExpression) p;
+                switch (c.getText()) {
+                    case "body":
+                        storeType(pexp, makeCached(LogDataBody.class));
+                        break;
+                    case "traceContext":
+                        storeType(pexp, makeCached(TraceContext.class));
+                        break;
+                    case "tags":
+                        storeType(pexp, makeCached(LogTags.class));
+                        break;
+                }
+            }
+            setHandled(true);
+            return true;
+        }
+
+        return false;
+    }
+}
+

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/SkyWalkingDelegatingScript.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.log.analyzer.dsl.spec;
+
+import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
+import groovy.util.DelegatingScript;
+import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
+
+public class SkyWalkingDelegatingScript extends DelegatingScript {
+    @Override
+    public Object run() {
+        return null;
+    }
+
+    public void filter(@DelegatesTo(value = FilterSpec.class, strategy = Closure.DELEGATE_ONLY) Closure<?> closure) {
+        closure.setDelegate(getDelegate());
+        closure.call();
+    }
+
+    public void json(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+        closure.setDelegate(getDelegate());
+        closure.call();
+    }
+
+    public void text(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+        closure.setDelegate(getDelegate());
+        closure.call();
+    }
+
+    public void extractor(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+        closure.setDelegate(getDelegate());
+        closure.call();
+    }
+
+    public void sink(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+        closure.setDelegate(getDelegate());
+        closure.call();
+    }
+}

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java

@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.extractor;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
 import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -54,7 +55,8 @@ public class ExtractorSpec extends AbstractSpec {
                          final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
         super(moduleManager, moduleConfig);
 
-        final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
+        final MeterSystem meterSystem =
+            moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
 
         metricConverts = moduleConfig.malConfigs()
                                      .stream()
@@ -93,7 +95,7 @@ public class ExtractorSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void tag(final Map<String, Object> kv) {
+    public void tag(final Map<String, ?> kv) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -108,7 +110,8 @@ public class ExtractorSpec extends AbstractSpec {
                        kv.entrySet()
                          .stream()
                          .filter(it -> isNotBlank(it.getKey()))
-                         .filter(it -> nonNull(it.getValue()) && isNotBlank(Objects.toString(it.getValue())))
+                         .filter(it -> nonNull(it.getValue()) &&
+                             isNotBlank(Objects.toString(it.getValue())))
                          .map(it -> {
                              final Object val = it.getValue();
                              String valStr = Objects.toString(val);
@@ -176,7 +179,7 @@ public class ExtractorSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void metrics(final Closure<Void> cl) {
+    public void metrics(@DelegatesTo(SampleBuilder.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -188,8 +191,8 @@ public class ExtractorSpec extends AbstractSpec {
 
         metricConverts.forEach(it -> it.toMeter(
             ImmutableMap.<String, SampleFamily>builder()
-                .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
-                .build()
+                        .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
+                        .build()
         ));
     }
 
@@ -198,11 +201,15 @@ public class ExtractorSpec extends AbstractSpec {
         private final Sample.SampleBuilder sampleBuilder = Sample.builder();
 
         @SuppressWarnings("unused")
-        public Sample.SampleBuilder labels(final Map<String, String> labels) {
-            final Map<String, String> filtered = labels.entrySet()
-                                                       .stream()
-                                                       .filter(it -> isNotBlank(it.getKey()) && isNotBlank(it.getValue()))
-                                                       .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        public Sample.SampleBuilder labels(final Map<String, ?> labels) {
+            final Map<String, String> filtered =
+                labels.entrySet()
+                      .stream()
+                      .filter(it -> isNotBlank(it.getKey()) && nonNull(it.getValue()))
+                      .collect(
+                          Collectors.toMap(Map.Entry::getKey,
+                                           it -> Objects.toString(it.getValue()))
+                      );
             return sampleBuilder.labels(ImmutableMap.copyOf(filtered));
         }
     }

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java

@@ -18,10 +18,10 @@
 
 package org.apache.skywalking.oap.log.analyzer.dsl.spec.filter;
 
-import com.google.gson.reflect.TypeToken;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.protobuf.TextFormat;
 import groovy.lang.Closure;
-import java.lang.reflect.Type;
+import groovy.lang.DelegatesTo;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -57,14 +57,14 @@ public class FilterSpec extends AbstractSpec {
 
     private final SinkSpec sink;
 
-    private final Type parsedType;
+    private final TypeReference<Map<String, Object>> parsedType;
 
     public FilterSpec(final ModuleManager moduleManager,
                       final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
         super(moduleManager, moduleConfig);
 
-        parsedType = new TypeToken<Map<String, Object>>() {
-        }.getType();
+        parsedType = new TypeReference<Map<String, Object>>() {
+        };
 
         factories = Arrays.asList(
             new RecordAnalysisListener.Factory(moduleManager(), moduleConfig()),
@@ -81,7 +81,7 @@ public class FilterSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void text(final Closure<Void> cl) {
+    public void text(@DelegatesTo(TextParserSpec.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -90,7 +90,7 @@ public class FilterSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void json(final Closure<Void> cl) {
+    public void json(@DelegatesTo(JsonParserSpec.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -99,7 +99,8 @@ public class FilterSpec extends AbstractSpec {
 
         final LogData.Builder logData = BINDING.get().log();
         try {
-            final Map<String, Object> parsed = jsonParser.create().fromJson(
+
+            final Map<String, Object> parsed = jsonParser.create().readValue(
                 logData.getBody().getJson().getJson(), parsedType
             );
 
@@ -111,8 +112,8 @@ public class FilterSpec extends AbstractSpec {
         }
     }
 
-    @SuppressWarnings({"unused", "unchecked"})
-    public void yaml(final Closure<Void> cl) {
+    @SuppressWarnings({"unused"})
+    public void yaml(@DelegatesTo(YamlParserSpec.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -121,7 +122,7 @@ public class FilterSpec extends AbstractSpec {
 
         final LogData.Builder logData = BINDING.get().log();
         try {
-            final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
+            final Map<String, Object> parsed = yamlParser.create().load(
                 logData.getBody().getYaml().getYaml()
             );
 
@@ -134,7 +135,7 @@ public class FilterSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void extractor(final Closure<Void> cl) {
+    public void extractor(@DelegatesTo(ExtractorSpec.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -143,7 +144,7 @@ public class FilterSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void sink(final Closure<Void> cl) {
+    public void sink(@DelegatesTo(SinkSpec.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -166,7 +167,7 @@ public class FilterSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void filter(final Closure<Void> cl) {
+    public void filter(final Closure<?> cl) {
         cl.call();
     }
 }

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java

@@ -18,26 +18,32 @@
 
 package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
 
-import lombok.Getter;
-import lombok.Setter;
 import lombok.experimental.Accessors;
 import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
 import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 
-@Accessors(fluent = true)
+@Accessors
 public class AbstractParserSpec extends AbstractSpec {
     /**
      * Whether the filter chain should abort when parsing the logs failed.
      *
-     * Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns.
+     * Failing to parse the logs means either parsing throws exceptions or the logs not matching the
+     * desired patterns.
      */
-    @Getter
-    @Setter
     private boolean abortOnFailure = true;
 
     public AbstractParserSpec(final ModuleManager moduleManager,
                               final LogAnalyzerModuleConfig moduleConfig) {
         super(moduleManager, moduleConfig);
     }
+
+    @SuppressWarnings("unused") // used in user LAL scripts
+    public void abortOnFailure(final boolean abortOnFailure) {
+        this.abortOnFailure = abortOnFailure;
+    }
+
+    public boolean abortOnFailure() {
+        return this.abortOnFailure;
+    }
 }

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java

@@ -18,28 +18,23 @@
 
 package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 
 public class JsonParserSpec extends AbstractParserSpec {
-    private final GsonBuilder gsonBuilder;
-
-    private final Gson gson;
+    private final ObjectMapper mapper;
 
     public JsonParserSpec(final ModuleManager moduleManager,
                           final LogAnalyzerModuleConfig moduleConfig) {
         super(moduleManager, moduleConfig);
 
-        gsonBuilder = new GsonBuilder();
-
-        // We just create a gson instance in advance for now (for the sake of performance),
+        // We just create a mapper instance in advance for now (for the sake of performance),
         // when we want to provide some extra options, we'll move this into method "create" then.
-        gson = gsonBuilder.create();
+        mapper = new ObjectMapper();
     }
 
-    public Gson create() {
-        return gson;
+    public ObjectMapper create() {
+        return mapper;
     }
 }

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java

@@ -31,13 +31,13 @@ public class TextParserSpec extends AbstractParserSpec {
     }
 
     @SuppressWarnings("unused")
-    public boolean regexp(final String regexp) {
-        return regexp(Pattern.compile(regexp));
+    public void regexp(final String regexp) {
+        regexp(Pattern.compile(regexp));
     }
 
-    public boolean regexp(final Pattern pattern) {
+    public void regexp(final Pattern pattern) {
         if (BINDING.get().shouldAbort()) {
-            return false;
+            return;
         }
         final LogData.Builder log = BINDING.get().log();
         final Matcher matcher = pattern.matcher(log.getBody().getText().getText());
@@ -47,7 +47,6 @@ public class TextParserSpec extends AbstractParserSpec {
         } else if (abortOnFailure()) {
             BINDING.get().abort();
         }
-        return matched;
     }
 
     public boolean grok(final String grok) {

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java

@@ -19,6 +19,8 @@
 package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink;
 
 import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
+import groovy.lang.GString;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
@@ -28,7 +30,7 @@ import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 
 public class SamplerSpec extends AbstractSpec {
-    private final Map<String, Sampler> samplers;
+    private final Map<GString, Sampler> samplers;
     private final RateLimitingSampler.ResetHandler rlsResetHandler;
 
     public SamplerSpec(final ModuleManager moduleManager,
@@ -40,7 +42,7 @@ public class SamplerSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void rateLimit(final String id, final Closure<Void> cl) {
+    public void rateLimit(final GString id, @DelegatesTo(RateLimitingSampler.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java

@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink;
 
 import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
 import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
 import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
@@ -35,7 +36,7 @@ public class SinkSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void sampler(final Closure<Void> cl) {
+    public void sampler(@DelegatesTo(SamplerSpec.class) final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -44,7 +45,7 @@ public class SinkSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void enforcer(final Closure<Void> cl) {
+    public void enforcer(final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
@@ -52,7 +53,7 @@ public class SinkSpec extends AbstractSpec {
     }
 
     @SuppressWarnings("unused")
-    public void dropper(final Closure<Void> cl) {
+    public void dropper(final Closure<?> cl) {
         if (BINDING.get().shouldAbort()) {
             return;
         }

oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.log.analyzer.dsl;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import org.apache.skywalking.apm.network.logging.v3.LogData;
+import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.config.ConfigService;
+import org.apache.skywalking.oap.server.core.source.SourceReceiver;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder;
+import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder;
+import org.apache.skywalking.oap.server.library.module.ModuleStartException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.powermock.reflect.Whitebox;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(Parameterized.class)
+public class DSLTest {
+    @Parameterized.Parameters(name = "{index}: {0}")
+    public static Collection<Object[]> data() {
+        return Arrays.asList(
+            new String[] {
+                "parser",
+                "filter {\n" +
+                    "  json {\n" +
+                    "    abortOnFailure false // for test purpose, we want to persist all logs\n" +
+                    "  }\n" +
+                    "  text {\n" +
+                    "    abortOnFailure false // for test purpose, we want to persist all logs\n" +
+                    "    regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$" +
+                    "  }\n" +
+                    "  yaml {\n" +
+                    "    abortOnFailure false // for test purpose, we want to persist all logs\n" +
+                    "  }" +
+                    "}",
+                },
+            new String[] {
+                "extractor",
+                "filter {\n" +
+                    "  extractor {\n" +
+                    "    service \"test\"\n" +
+                    "    instance \"test\"\n" +
+                    "    endpoint \"test\"\n" +
+                    "    traceId \"123\"\n" +
+                    "    segmentId \"123\"\n" +
+                    "    spanId \"123\"\n" +
+                    "    timestamp \"123\"\n" +
+                    "    metrics {\n" +
+                    "      name \"metricsName\"\n" +
+                    "      value 123\n" +
+                    "      timestamp \"123\"\n" +
+                    "      labels \"k1\": \"v1\"\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "}",
+                },
+            new String[] {
+                "sink",
+                "filter {\n" +
+                    "  sink {\n" +
+                    "    enforcer {\n" +
+                    "    }\n" +
+                    "    dropper {\n" +
+                    "    }\n" +
+                    "    sampler {\n" +
+                    "      if (parsed?.commonProperties?.responseFlags) {\n" +
+                    "        // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" +
+                    "        // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" +
+                    "        rateLimit(\"${log.service}:${log.body.json.json}:${log.tags.getData(0).key}:${parsed?.commonProperties?.responseFlags}\") {\n" +
+                    "          qps 100\n" +
+                    "        }\n" +
+                    "      } else {\n" +
+                    "        // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" +
+                    "        // e.g. checkoutservice:500, checkoutservice:404.\n" +
+                    "        rateLimit(\"${log.service}:${log.body?.type}:${log.traceContext?.traceId}:${parsed?.response?.responseCode}\") {\n" +
+                    "          qps 100\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "}",
+                },
+            new String[] {
+                "e2e",
+                "filter {\n" +
+                    "  text {\n" +
+                    "    abortOnFailure false // for test purpose, we want to persist all logs\n" +
+                    "    regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$\n" +
+                    "  }\n" +
+                    "  extractor {\n" +
+                    "    metrics {\n" +
+                    "      timestamp \"${log.timestamp}\"\n" +
+                    "      labels level: parsed.level, service: log.service, instance: log.serviceInstance\n" +
+                    "      name \"log_count\"\n" +
+                    "      value 1\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "  sink {\n" +
+                    "  }\n" +
+                    "}\n"
+            }
+        );
+    }
+
+    @Parameterized.Parameter()
+    public String name;
+
+    @Parameterized.Parameter(1)
+    public String script;
+
+    final ModuleManager manager = mock(ModuleManager.class);
+
+    @Before
+    public void setup() {
+        Whitebox.setInternalState(manager, "isInPrepareStage", false);
+        when(manager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class));
+        when(manager.find(CoreModule.NAME).provider()).thenReturn(mock(ModuleServiceHolder.class));
+        when(manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class))
+            .thenReturn(mock(SourceReceiver.class));
+        when(manager.find(CoreModule.NAME).provider().getService(
+            ConfigService.class)).thenReturn(mock(ConfigService.class));
+        when(manager.find(CoreModule.NAME).provider().getService(
+            ConfigService.class).getSearchableLogsTags()).thenReturn(anyString());
+    }
+
+    @Test
+    public void testDslStaticCompile() throws ModuleStartException {
+        final DSL dsl = DSL.of(manager, new LogAnalyzerModuleConfig(), script);
+        Whitebox.setInternalState(
+            Whitebox.getInternalState(dsl, "filterSpec"), "factories", Collections.emptyList()
+        );
+
+        dsl.bind(new Binding().log(LogData.newBuilder().build()));
+        dsl.evaluate();
+    }
+}

oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml

+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Refer to examples in config-examples/log-mal.yaml

oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4

@@ -39,7 +39,7 @@ SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory';
 SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool';
 SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC';
 SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread';
-SRC_SERVICE_INSTANCE_JVM_CLASS:'ServiceInstanceJVMClass';
+SRC_SERVICE_INSTANCE_JVM_CLASS: 'ServiceInstanceJVMClass';
 SRC_DATABASE_ACCESS: 'DatabaseAccess';
 SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU';
 SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';

oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java

@@ -37,10 +37,12 @@ public abstract class MockModuleManager extends ModuleManager {
         moduleProviderHolderMap.put(name, provider);
     }
 
+    @Override
     public boolean has(String moduleName) {
         return moduleProviderHolderMap.containsKey(moduleName);
     }
 
+    @Override
     public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException {
         if (!moduleProviderHolderMap.containsKey(moduleName)) {
             throw new ModuleNotFoundRuntimeException("ModuleProviderHolder[" + moduleName + "] cannot found in MOCK.");

test/e2e/e2e-test/docker/log/lal.yaml

@@ -23,7 +23,7 @@ rules:
         }
         extractor {
           metrics {
-            timestamp log.timestamp
+            timestamp log.timestamp as Long
             labels level: parsed.level, service: log.service, instance: log.serviceInstance
             name "log_count"
             value 1