未验证 提交 9de93119 编写于 作者: Z Zhenxu Ke 提交者: GitHub

Enhance gRPC log appender to allow layout pattern (#6403)

上级 317a6580
......@@ -25,6 +25,17 @@ inputs:
runs:
using: "composite"
steps:
- name: Check Unintended Changes
shell: bash
run: |
echo "::group::Check sw.version"
sudo apt install -y -q xmlstarlet
SW_VERSION=$(xmlstarlet sel -N pom=http://maven.apache.org/POM/4.0.0 -t -v "/pom:project/pom:properties/pom:sw.version" test/e2e/pom.xml)
if [[ "$(echo $(echo $SW_VERSION))" != "" ]]; then
echo "Please don't submit the change of sw.version in test/e2e/pom.xml"
exit 1
fi
echo "::endgroup::"
- name: Check License
shell: bash
run: |
......@@ -53,8 +64,14 @@ runs:
- name: Run E2E Test
shell: bash
run: |
echo "::group::Install SNAPSHOT apm-application-toolkit"
./mvnw -DskipTests -N install
./mvnw -f apm-application-toolkit -DskipTests -am install
echo "::endgroup::"
echo "::group::Run E2E Test ${{ inputs.test_class }}"
./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=${{ inputs.test_class }}
SW_VERSION=$(./mvnw -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false -Dsw.version=${SW_VERSION} verify -Dit.test=${{ inputs.test_class }}
echo "::endgroup::"
- name: Report Coverage
shell: bash
......
......@@ -29,6 +29,17 @@ inputs:
runs:
using: "composite"
steps:
- name: Check Unintended Changes
shell: bash
run: |
echo "::group::Check sw.version"
sudo apt install -y -q xmlstarlet
SW_VERSION=$(xmlstarlet sel -N pom=http://maven.apache.org/POM/4.0.0 -t -v "/pom:project/pom:properties/pom:sw.version" test/e2e/pom.xml)
if [[ "$(echo $(echo $SW_VERSION))" != "" ]]; then
echo "Please don't submit the change of sw.version in test/e2e/pom.xml"
exit 1
fi
echo "::endgroup::"
- name: Check License
shell: bash
run: |
......
......@@ -48,6 +48,17 @@ jobs:
with:
submodules: true
- name: Check Unintended Changes
run: |
echo "::group::Check sw.version"
sudo apt install -y -q xmlstarlet
SW_VERSION=$(xmlstarlet sel -N pom=http://maven.apache.org/POM/4.0.0 -t -v "/pom:project/pom:properties/pom:sw.version" test/e2e/pom.xml)
if [[ "$(echo $(echo $SW_VERSION))" != "" ]]; then
echo "::error Please don't submit the change of sw.version in test/e2e/pom.xml"
exit 1
fi
echo "::endgroup::"
- name: Set Skip Env Var
uses: ./.github/actions/skip
......@@ -132,7 +143,12 @@ jobs:
export WEBAPP_HOST=127.0.0.1
export WEBAPP_PORT=8080
./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.mesh.ALSE2E
export SW_VERSION=$(./mvnw -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
./mvnw -DskipTests -N install
./mvnw -f apm-application-toolkit -DskipTests -am install
./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false -Dsw.version=${SW_VERSION} verify -Dit.test=org.apache.skywalking.e2e.mesh.ALSE2E
- name: Logs
if: ${{ failure() }}
......@@ -162,6 +178,17 @@ jobs:
with:
submodules: true
- name: Check Unintended Changes
run: |
echo "::group::Check sw.version"
sudo apt install -y -q xmlstarlet
SW_VERSION=$(xmlstarlet sel -N pom=http://maven.apache.org/POM/4.0.0 -t -v "/pom:project/pom:properties/pom:sw.version" test/e2e/pom.xml)
if [[ "$(echo $(echo $SW_VERSION))" != "" ]]; then
echo "Please don't submit the change of sw.version in test/e2e/pom.xml"
exit 1
fi
echo "::endgroup::"
- name: Set Skip Env Var
uses: ./.github/actions/skip
......@@ -246,7 +273,12 @@ jobs:
export WEBAPP_HOST=127.0.0.1
export WEBAPP_PORT=8080
./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.mesh.MetricsServiceE2E
export SW_VERSION=$(./mvnw -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
./mvnw -DskipTests -N install
./mvnw -f apm-application-toolkit -DskipTests -am install
./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false -Dsw.version=${SW_VERSION} verify -Dit.test=org.apache.skywalking.e2e.mesh.MetricsServiceE2E
- name: Logs
if: ${{ failure() }}
......
......@@ -13,6 +13,7 @@ Release Notes.
* Add net.bytebuddy.agent.builder.AgentBuilder.RedefinitionStrategy.Listener to show detail message when redefine errors occur.
* Fix ClassCastException of log4j gRPC reporter.
* Fix NPE when Kafka reporter activated.
* Enhance gRPC log appender to allow layout pattern.
#### OAP-Backend
* Allow user-defined `JAVA_OPTS` in the startup script.
......
......@@ -19,9 +19,18 @@
package org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Layout;
import org.apache.log4j.spi.LoggingEvent;
public class GRPCLogClientAppender extends AppenderSkeleton {
public GRPCLogClientAppender() {
}
public GRPCLogClientAppender(Layout layout) {
this.setLayout(layout);
}
@Override
protected void append(LoggingEvent loggingEvent) {
......@@ -34,6 +43,6 @@ public class GRPCLogClientAppender extends AppenderSkeleton {
@Override
public boolean requiresLayout() {
return false;
return true;
}
}
......@@ -18,9 +18,14 @@
package org.apache.skywalking.apm.toolkit.log.log4j.v2.x.log;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import org.apache.logging.log4j.core.Filter;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
import org.apache.logging.log4j.core.appender.OutputStreamManager;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
......@@ -29,25 +34,45 @@ import org.apache.logging.log4j.core.config.plugins.PluginElement;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
@Plugin(name = "GRPCLogClientAppender", category = "Core", elementType = "appender")
public class GRPCLogClientAppender extends AbstractAppender {
public class GRPCLogClientAppender extends AbstractOutputStreamAppender<OutputStreamManager> {
private static final OutputStream DISCARDED_STREAM = new OutputStream() {
@Override
public void write(final int b) throws IOException {
// discarded
}
};
private GRPCLogClientAppender(final String name, final Filter filter, final boolean ignoreExceptions) {
super(name, filter, null, ignoreExceptions);
protected GRPCLogClientAppender(final String name,
final Layout<? extends Serializable> layout,
final Filter filter,
final boolean ignoreExceptions) {
super(
name,
layout,
filter,
ignoreExceptions,
true,
getManager0(layout)
);
}
@Override
public void append(LogEvent logEvent) {
public void append(final LogEvent event) {
}
@PluginFactory
public static GRPCLogClientAppender createAppender(@PluginAttribute("name") final String name,
@PluginElement("Layout") final Layout<? extends Serializable> layout,
@PluginElement("Filter") final Filter filter,
@PluginConfiguration final Configuration config,
@PluginAttribute("ignoreExceptions") final String ignore) {
String appenderName = name == null ? "gRPCLogClientAppender" : name;
final boolean ignoreExceptions = "true".equalsIgnoreCase(ignore) || !"false".equalsIgnoreCase(ignore);
return new GRPCLogClientAppender(appenderName, filter, ignoreExceptions);
return new GRPCLogClientAppender(appenderName, layout, filter, ignoreExceptions);
}
private static OutputStreamManager getManager0(final Layout<? extends Serializable> layout) {
return OutputStreamManager.getManager("Discard", new Object(), (s, o) -> new OutputStreamManager(DISCARDED_STREAM, "Discard", layout, false) {
});
}
}
......@@ -18,12 +18,21 @@
package org.apache.skywalking.apm.toolkit.log.logback.v1.x.log;
import ch.qos.logback.core.AppenderBase;
import ch.qos.logback.core.OutputStreamAppender;
import java.io.IOException;
import java.io.OutputStream;
public class GRPCLogClientAppender<E> extends AppenderBase<E> {
public class GRPCLogClientAppender<E> extends OutputStreamAppender<E> {
public GRPCLogClientAppender() {
setOutputStream(new OutputStream() {
@Override
public void write(final int b) throws IOException {
// discarded
}
});
}
@Override
protected void append(E eventObject) {
protected void subAppend(final E event) {
}
}
......@@ -20,7 +20,8 @@ package org.apache.skywalking.apm.toolkit.activation.log.log4j.v1.x.log;
import java.lang.reflect.Method;
import java.util.Objects;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.context.ContextManager;
......@@ -35,7 +36,6 @@ import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
import org.apache.skywalking.apm.network.logging.v3.LogTags;
import org.apache.skywalking.apm.network.logging.v3.TextLog;
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
import org.apache.log4j.spi.LoggingEvent;
public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundInterceptor {
......@@ -52,7 +52,7 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
}
LoggingEvent event = (LoggingEvent) allArguments[0];
if (Objects.nonNull(event)) {
client.produce(transform(event));
client.produce(transform((AppenderSkeleton) objInst, event));
}
}
......@@ -71,10 +71,12 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
/**
* transforms {@link LoggingEvent} to {@link LogData}
*
*
* @param appender the real {@link AppenderSkeleton appender}
* @param event {@link LoggingEvent}
* @return {@link LogData} with filtered trace context in order to reduce the cost on the network
*/
private LogData transform(LoggingEvent event) {
private LogData transform(final AppenderSkeleton appender, LoggingEvent event) {
LogData.Builder builder = LogData.newBuilder()
.setTimestamp(event.getTimeStamp())
.setService(Config.Agent.SERVICE_NAME)
......@@ -93,7 +95,7 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
.setKey("thread").setValue(event.getThreadName()).build())
.build())
.setBody(LogDataBody.newBuilder().setType(LogDataBody.ContentCase.TEXT.name())
.setText(TextLog.newBuilder().setText(transformLogText(event)).build()).build());
.setText(TextLog.newBuilder().setText(transformLogText(appender, event)).build()).build());
return -1 == ContextManager.getSpanId() ? builder.build()
: builder.setTraceContext(TraceContext.newBuilder()
.setTraceId(ContextManager.getGlobalTraceId())
......@@ -102,8 +104,11 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
.build()).build();
}
private String transformLogText(final LoggingEvent event) {
final String throwableString = Objects.isNull(event.getThrowableInformation()) ? "" :
private String transformLogText(final AppenderSkeleton appender, final LoggingEvent event) {
if (appender.getLayout() != null) {
return appender.getLayout().format(event);
}
final String throwableString = Objects.isNull(event.getThrowableInformation()) ? "" :
ThrowableTransformer.INSTANCE.convert2String(event.getThrowableInformation().getThrowable(), 2048);
return event.getMessage() + "\n" + throwableString;
}
......
......@@ -21,8 +21,8 @@ package org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.log;
import java.lang.reflect.Method;
import java.util.Objects;
import java.util.Optional;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.context.ContextManager;
......@@ -54,7 +54,7 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
}
LogEvent event = (LogEvent) allArguments[0];
if (Objects.nonNull(event)) {
client.produce(transform(event));
client.produce(transform((AbstractAppender) objInst, event));
}
}
......@@ -73,10 +73,12 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
/**
* transforms {@link LogEvent} to {@link LogData}
*
*
* @param appender the real {@link AbstractAppender appender}
* @param event {@link LogEvent}
* @return {@link LogData} with filtered trace context in order to reduce the cost on the network
*/
private LogData transform(LogEvent event) {
private LogData transform(final AbstractAppender appender, LogEvent event) {
LogTags.Builder logTags = LogTags.newBuilder()
.addData(KeyStringValuePair.newBuilder()
.setKey("level").setValue(event.getLevel().toString()).build())
......@@ -105,7 +107,7 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
.setServiceInstance(Config.Agent.INSTANCE_NAME)
.setTags(logTags.build())
.setBody(LogDataBody.newBuilder().setType(LogDataBody.ContentCase.TEXT.name())
.setText(TextLog.newBuilder().setText(transformLogText(event)).build()).build());
.setText(TextLog.newBuilder().setText(transformLogText(appender, event)).build()).build());
return -1 == ContextManager.getSpanId() ? builder.build()
: builder.setTraceContext(TraceContext.newBuilder()
.setTraceId(ContextManager.getGlobalTraceId())
......@@ -114,8 +116,11 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
.build()).build();
}
private String transformLogText(final LogEvent event) {
private String transformLogText(final AbstractAppender appender, final LogEvent event) {
if (ToolkitConfig.Plugin.Toolkit.Log.TRANSMIT_FORMATTED) {
if (appender.getLayout() != null) {
return new String(appender.getLayout().toByteArray(event));
}
return event.getMessage().getFormattedMessage() + "\n" + ThrowableTransformer.INSTANCE.convert2String(event.getThrown(), 2048);
} else {
return event.getMessage().getFormat();
......
......@@ -39,7 +39,7 @@ public class GRPCLogAppenderActivation extends ClassInstanceMethodsEnhancePlugin
"org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log.GRPCLogAppenderInterceptor";
public static final String ENHANCE_CLASS =
"org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender";
public static final String ENHANCE_METHOD = "append";
public static final String ENHANCE_METHOD = "subAppend";
@Override
protected ClassMatch enhanceClass() {
......
......@@ -18,12 +18,13 @@
package org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.ThrowableProxy;
import ch.qos.logback.core.OutputStreamAppender;
import java.lang.reflect.Method;
import java.util.Objects;
import java.util.Optional;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.context.ContextManager;
......@@ -38,14 +39,13 @@ import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
import org.apache.skywalking.apm.network.logging.v3.LogTags;
import org.apache.skywalking.apm.network.logging.v3.TextLog;
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import org.apache.skywalking.apm.toolkit.logging.common.log.ToolkitConfig;
public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundInterceptor {
private LogReportServiceClient client;
@SuppressWarnings("unchecked")
@Override
public void beforeMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class<?>[] argumentsTypes,
MethodInterceptResult result) throws Throwable {
......@@ -57,7 +57,7 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
}
ILoggingEvent event = (ILoggingEvent) allArguments[0];
if (Objects.nonNull(event)) {
client.produce(transform(event));
client.produce(transform((OutputStreamAppender<ILoggingEvent>) objInst, event));
}
}
......@@ -76,10 +76,11 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
/**
* transforms {@link ILoggingEvent} to {@link LogData}
*
* @param appender the real {@link OutputStreamAppender appender}
* @param event {@link ILoggingEvent}
* @return {@link LogData} with filtered trace context in order to reduce the cost on the network
*/
private LogData transform(ILoggingEvent event) {
private LogData transform(final OutputStreamAppender<ILoggingEvent> appender, ILoggingEvent event) {
LogTags.Builder logTags = LogTags.newBuilder()
.addData(KeyStringValuePair.newBuilder()
.setKey("level").setValue(event.getLevel().toString()).build())
......@@ -110,7 +111,7 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
.setServiceInstance(Config.Agent.INSTANCE_NAME)
.setTags(logTags.build())
.setBody(LogDataBody.newBuilder().setType(LogDataBody.ContentCase.TEXT.name())
.setText(TextLog.newBuilder().setText(transformLogText(event)).build()).build());
.setText(TextLog.newBuilder().setText(transformLogText(appender, event)).build()).build());
return -1 == ContextManager.getSpanId() ? builder.build()
: builder.setTraceContext(TraceContext.newBuilder()
.setTraceId(ContextManager.getGlobalTraceId())
......@@ -119,18 +120,9 @@ public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundIntercep
.build()).build();
}
private String transformLogText(final ILoggingEvent event) {
final IThrowableProxy throwableProxy = event.getThrowableProxy();
if (!(throwableProxy instanceof ThrowableProxy)) {
if (ToolkitConfig.Plugin.Toolkit.Log.TRANSMIT_FORMATTED) {
return event.getFormattedMessage();
} else {
return event.getMessage();
}
}
private String transformLogText(final OutputStreamAppender<ILoggingEvent> appender, final ILoggingEvent event) {
if (ToolkitConfig.Plugin.Toolkit.Log.TRANSMIT_FORMATTED) {
final Throwable throwable = ((ThrowableProxy) throwableProxy).getThrowable();
return event.getFormattedMessage() + "\n" + ThrowableTransformer.INSTANCE.convert2String(throwable, 2048);
return new String(appender.getEncoder().encode(event));
} else {
return event.getMessage();
}
......
......@@ -34,7 +34,6 @@
<artifactId>e2e-service-provider</artifactId>
<properties>
<sw.version>8.4.0</sw.version>
<log4j.version>1.2.17</log4j.version>
<log4j2.version>2.7</log4j2.version>
<logback.version>1.2.3</logback.version>
......
......@@ -13,4 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
log4j.rootLogger=info,CustomAppender
log4j.appender.CustomAppender=org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log.GRPCLogClientAppender
\ No newline at end of file
log4j.appender.CustomAppender=org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log.GRPCLogClientAppender
log4j.appender.CustomAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.CustomAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
......@@ -20,7 +20,9 @@
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
<GRPCLogClientAppender name="grpc-log"/>
<GRPCLogClientAppender name="grpc-log">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</GRPCLogClientAppender>
</Appenders>
......@@ -31,4 +33,4 @@
<AppenderRef ref="grpc-log"/>
</Root>
</Loggers>
</Configuration>
\ No newline at end of file
</Configuration>
......@@ -25,10 +25,16 @@
</encoder>
</appender>
<appender name="grpc-log" class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender"/>
<appender name="grpc-log" class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.mdc.TraceIdMDCPatternLogbackLayout">
<Pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%X{tid}] [%thread] %-5level %logger{36} -%msg%n</Pattern>
</layout>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="grpc-log"/>
<appender-ref ref="stdout"/>
</root>
</configuration>
\ No newline at end of file
</configuration>
......@@ -44,6 +44,14 @@
</modules>
<properties>
<sw.version>
<!--
Please manually set the SkyWalking version here to build in the IDE
(if you build from command line, just use ./mvnw -Dsw.version=x.y.z),
but make sure not to check it into the code base / git
-->
</sw.version>
<java.version>1.8</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册