Agent supports the collection of JVM arguments and jar dependencies information. (#7069)

CHANGES.md

@@ -9,6 +9,7 @@ Release Notes.
 
 #### Java Agent
 * Supports modifying span attributes in async mode.
+* Agent supports the collection of JVM arguments and jar dependency information.
 
 #### OAP-Backend
 * Disable Spring sleuth meter analyzer by default.

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/jvm/LoadedLibraryCollector.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.agent.core.jvm;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.skywalking.apm.agent.core.logging.api.ILog;
+import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
+import org.apache.skywalking.apm.agent.core.util.CollectionUtil;
+import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
+
+public class LoadedLibraryCollector {
+
+    private static final ILog LOGGER = LogManager.getLogger(LoadedLibraryCollector.class);
+    private static final  String JAR_SEPARATOR = "!";
+    private static Set<ClassLoader> CURRENT_URL_CLASSLOADER_SET = new HashSet<>();
+    private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create();
+    /**
+     * Prevent OOM in special scenes
+     */
+    private static int CURRENT_URL_CLASSLOADER_SET_MAX_SIZE = 50;
+
+    public static void registerURLClassLoader(ClassLoader classLoader) {
+        if (CURRENT_URL_CLASSLOADER_SET.size() < CURRENT_URL_CLASSLOADER_SET_MAX_SIZE && classLoader instanceof URLClassLoader) {
+            CURRENT_URL_CLASSLOADER_SET.add(classLoader);
+        }
+    }
+
+    /**
+     * Build the required JVM information to add to the instance properties
+     */
+    public static List<KeyStringValuePair> buildJVMInfo() {
+        List<KeyStringValuePair> jvmInfo = new ArrayList<>();
+        jvmInfo.add(KeyStringValuePair.newBuilder().setKey("Start Time").setValue(getVmStartTime()).build());
+        jvmInfo.add(KeyStringValuePair.newBuilder().setKey("JVM Arguments").setValue(GSON.toJson(getVmArgs())).build());
+        List<String> libJarNames = getLibJarNames();
+        jvmInfo.add(KeyStringValuePair.newBuilder().setKey("Jar Dependencies").setValue(GSON.toJson(libJarNames)).build());
+        return jvmInfo;
+    }
+
+    private static String getVmStartTime() {
+        long startTime = ManagementFactory.getRuntimeMXBean().getStartTime();
+        return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(startTime));
+    }
+
+    private static List<String> getVmArgs() {
+        List<String> vmArgs = ManagementFactory.getRuntimeMXBean().getInputArguments();
+        List<String> sortedVmArgs = new ArrayList<>(vmArgs);
+        Collections.sort(sortedVmArgs);
+        return sortedVmArgs;
+    }
+
+    private static List<String> getLibJarNames() {
+        List<URL> classLoaderUrls = loadClassLoaderUrls();
+        return extractLibJarNamesFromURLs(classLoaderUrls);
+    }
+
+    private static List<URL> loadClassLoaderUrls() {
+        List<URL> classLoaderUrls = new ArrayList<>();
+        for (ClassLoader classLoader : CURRENT_URL_CLASSLOADER_SET) {
+            try {
+                URLClassLoader webappClassLoader = (URLClassLoader) classLoader;
+                URL[] urls = webappClassLoader.getURLs();
+                classLoaderUrls.addAll(Arrays.asList(urls));
+            } catch (Exception e) {
+                LOGGER.warn("Load classloader urls exception: {}", e.getMessage());
+            }
+        }
+        return classLoaderUrls;
+    }
+
+    private static List<String> extractLibJarNamesFromURLs(List<URL> urls) {
+        Set<String> libJarNames = new HashSet<>();
+        for (URL url : urls) {
+            try {
+                String libJarName = extractLibJarName(url);
+                if (libJarName.endsWith(".jar")) {
+                    libJarNames.add(libJarName);
+                }
+            } catch (Exception e) {
+                LOGGER.warn("Extracting library name exception: {}", e.getMessage());
+            }
+        }
+        List<String> sortedLibJarNames = new ArrayList<>(libJarNames.size());
+        if (!CollectionUtil.isEmpty(libJarNames)) {
+            sortedLibJarNames.addAll(libJarNames);
+            Collections.sort(sortedLibJarNames);
+        }
+        return sortedLibJarNames;
+    }
+
+    private static String extractLibJarName(URL url) {
+        String protocol = url.getProtocol();
+        if (protocol.equals("file")) {
+            return extractNameFromFile(url.toString());
+        } else if (protocol.equals("jar")) {
+            return extractNameFromJar(url.toString());
+        } else {
+            return "";
+        }
+    }
+
+    private static String extractNameFromFile(String fileUri) {
+        int lastIndexOfSeparator = fileUri.lastIndexOf(File.separator);
+        if (lastIndexOfSeparator < 0) {
+            return fileUri;
+        } else {
+            return fileUri.substring(lastIndexOfSeparator + 1);
+        }
+    }
+
+    private static String extractNameFromJar(String jarUri) {
+        String uri = jarUri.substring(0, jarUri.lastIndexOf(JAR_SEPARATOR));
+        return extractNameFromFile(uri);
+    }
+
+}
\ No newline at end of file

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/ServiceManagementClient.java

@@ -31,6 +31,7 @@ import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
 import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
 import org.apache.skywalking.apm.agent.core.commands.CommandService;
 import org.apache.skywalking.apm.agent.core.conf.Config;
+import org.apache.skywalking.apm.agent.core.jvm.LoadedLibraryCollector;
 import org.apache.skywalking.apm.agent.core.logging.api.ILog;
 import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
 import org.apache.skywalking.apm.agent.core.os.OSUtil;
@@ -117,6 +118,7 @@ public class ServiceManagementClient implements BootService, Runnable, GRPCChann
                                                                         .addAllProperties(OSUtil.buildOSInfo(
                                                                             Config.OsInfo.IPV4_LIST_SIZE))
                                                                         .addAllProperties(SERVICE_INSTANCE_PROPERTIES)
+                                                                        .addAllProperties(LoadedLibraryCollector.buildJVMInfo())
                                                                         .build());
                     } else {
                         final Commands commands = managementServiceBlockingStub.withDeadlineAfter(

apm-sniffer/apm-agent/src/main/java/org/apache/skywalking/apm/agent/SkyWalkingAgent.java

@@ -36,6 +36,7 @@ import org.apache.skywalking.apm.agent.core.boot.AgentPackageNotFoundException;
 import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
 import org.apache.skywalking.apm.agent.core.conf.Config;
 import org.apache.skywalking.apm.agent.core.conf.SnifferConfigInitializer;
+import org.apache.skywalking.apm.agent.core.jvm.LoadedLibraryCollector;
 import org.apache.skywalking.apm.agent.core.logging.api.ILog;
 import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
 import org.apache.skywalking.apm.agent.core.plugin.AbstractClassEnhancePluginDefine;
@@ -151,6 +152,7 @@ public class SkyWalkingAgent {
                                                 final TypeDescription typeDescription,
                                                 final ClassLoader classLoader,
                                                 final JavaModule module) {
+            LoadedLibraryCollector.registerURLClassLoader(classLoader);
             List<AbstractClassEnhancePluginDefine> pluginDefines = pluginFinder.find(typeDescription);
             if (pluginDefines.size() > 0) {
                 DynamicType.Builder<?> newBuilder = builder;

apm-sniffer/optional-reporter-plugins/kafka-reporter-plugin/src/main/java/org/apache/skywalking/apm/agent/core/kafka/KafkaServiceManagementServiceClient.java

@@ -33,6 +33,7 @@ import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
 import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor;
 import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
 import org.apache.skywalking.apm.agent.core.conf.Config;
+import org.apache.skywalking.apm.agent.core.jvm.LoadedLibraryCollector;
 import org.apache.skywalking.apm.agent.core.logging.api.ILog;
 import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
 import org.apache.skywalking.apm.agent.core.os.OSUtil;
@@ -101,6 +102,7 @@ public class KafkaServiceManagementServiceClient implements BootService, Runnabl
                                                             .addAllProperties(OSUtil.buildOSInfo(
                                                                 Config.OsInfo.IPV4_LIST_SIZE))
                                                             .addAllProperties(SERVICE_INSTANCE_PROPERTIES)
+                                                            .addAllProperties(LoadedLibraryCollector.buildJVMInfo())
                                                             .build();
             producer.send(new ProducerRecord<>(topic, TOPIC_KEY_REGISTER + instance.getServiceInstance(),
                                                Bytes.wrap(instance.toByteArray())

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java

@@ -69,7 +69,7 @@ public class InstanceTraffic extends Metrics {
     private long lastPingTimestamp;
     @Setter
     @Getter
-    @Column(columnName = PROPERTIES, storageOnly = true)
+    @Column(columnName = PROPERTIES, storageOnly = true, length = 50000)
     private JsonObject properties;
 
     @Override

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java

@@ -18,6 +18,7 @@
 
 package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql;
 
+import com.google.gson.JsonObject;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -134,6 +135,12 @@ public class MySQLTableInstaller extends H2TableInstaller {
             } else {
                 return storageName + " VARCHAR(" + column.getLength() + ")";
             }
+        } else if (JsonObject.class.equals(type)) {
+            if (column.getLength() > 16383) {
+                return storageName + " MEDIUMTEXT";
+            } else {
+                return storageName + " VARCHAR(" + column.getLength() + ")";
+            }
         }
         return super.getColumn(column);
     }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/postgresql/PostgreSQLTableInstaller.java

@@ -52,7 +52,11 @@ public class PostgreSQLTableInstaller extends MySQLTableInstaller {
         } else if (byte[].class.equals(type)) {
             return storageName + " TEXT";
         } else if (JsonObject.class.equals(type)) {
-            return storageName + " VARCHAR(" + column.getLength() + ")";
+            if (column.getLength() > 16383) {
+                return storageName + " TEXT";
+            } else {
+                return storageName + " VARCHAR(" + column.getLength() + ")";
+            }
         } else if (List.class.isAssignableFrom(type)) {
             final Type elementType = ((ParameterizedType) genericType).getActualTypeArguments()[0];
             String oneColumnType = transform(column, (Class<?>) elementType, elementType);
@@ -81,6 +85,12 @@ public class PostgreSQLTableInstaller extends MySQLTableInstaller {
             } else {
                 return storageName + " VARCHAR(" + column.getLength() + ")";
             }
+        } else if (JsonObject.class.equals(type)) {
+            if (column.getLength() > 16383) {
+                return storageName + " TEXT";
+            } else {
+                return storageName + " VARCHAR(" + column.getLength() + ")";
+            }
         }
         return super.getColumn(column);
     }

test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaMeterE2E.java

@@ -111,7 +111,7 @@ public class KafkaMeterE2E extends SkyWalkingTestAdapter {
 
         LOGGER.info("instances: {}", instances);
 
-        load("expected/simple/instances.yml").as(InstancesMatcher.class).verify(instances);
+        load("expected/meter/instances.yml").as(InstancesMatcher.class).verify(instances);
 
         return instances;
     }

test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java

@@ -108,7 +108,7 @@ public class MeterE2E extends SkyWalkingTestAdapter {
 
         LOGGER.info("instances: {}", instances);
 
-        load("expected/simple/instances.yml").as(InstancesMatcher.class).verify(instances);
+        load("expected/meter/instances.yml").as(InstancesMatcher.class).verify(instances);
 
         return instances;
     }

test/e2e/e2e-test/src/test/resources/expected/cluster/instances.yml

@@ -23,5 +23,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
-      - name: ipv4s
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
         value: not null
+      - name: Jar Dependencies
+        value: not null
+      - name: ipv4s
+        value: not null
\ No newline at end of file

test/e2e/e2e-test/src/test/resources/expected/cluster/providerInstances.yml

@@ -23,6 +23,12 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null
   - key: not null
@@ -34,5 +40,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null

test/e2e/e2e-test/src/test/resources/expected/gateway/instances.yml

@@ -23,5 +23,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null

test/e2e/e2e-test/src/test/resources/expected/gateway/providerInstances.yml

@@ -23,6 +23,12 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null
   - key: not null
@@ -34,5 +40,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null

test/e2e/e2e-test/src/test/resources/expected/go/instances-java.yml

@@ -23,5 +23,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null
\ No newline at end of file

test/e2e/e2e-test/src/test/resources/expected/lua/instances.yml

@@ -23,5 +23,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null

test/e2e/e2e-test/src/test/resources/expected/meter/instances.yml

@@ -16,12 +16,3 @@
 instances:
   - key: not null
     label: not null
-    attributes:
-      - name: OS Name
-        value: not null
-      - name: hostname
-        value: not null
-      - name: Process No.
-        value: gt 0
-      - name: ipv4s
-        value: not null

test/e2e/e2e-test/src/test/resources/expected/profile/instances.yml

@@ -23,5 +23,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null

test/e2e/e2e-test/src/test/resources/expected/simple/instances.yml

@@ -16,12 +16,3 @@
 instances:
   - key: not null
     label: not null
-    attributes:
-      - name: OS Name
-        value: not null
-      - name: hostname
-        value: not null
-      - name: Process No.
-        value: gt 0
-      - name: ipv4s
-        value: not null

test/e2e/e2e-test/src/test/resources/expected/storage/instances.yml

@@ -23,5 +23,11 @@ instances:
         value: not null
       - name: Process No.
         value: gt 0
+      - name: Start Time
+        value: not null
+      - name: JVM Arguments
+        value: not null
+      - name: Jar Dependencies
+        value: not null
       - name: ipv4s
         value: not null