Collect metrics on number of logs lines printed (#2735)

### Motivation

Expose metrics in Prometheus regarding the number of log messages printed by each process.

This will add metrics like: 

```
# TYPE log4j2_appender_total counter
log4j2_appender_total{cluster="standalone",level="debug"} 0.0
log4j2_appender_total{cluster="standalone",level="warn"} 1.0
log4j2_appender_total{cluster="standalone",level="trace"} 0.0
log4j2_appender_total{cluster="standalone",level="error"} 0.0
log4j2_appender_total{cluster="standalone",level="fatal"} 0.0
log4j2_appender_total{cluster="standalone",level="info"} 181.0
```

conf/log4j2.yaml

@@ -22,6 +22,7 @@ Configuration:
   status: INFO
   monitorInterval: 30
   name: pulsar
+  packages: io.prometheus.client.log4j2
 
   Properties:
     Property:
@@ -43,7 +44,7 @@ Configuration:
       language: JavaScript
       path: ./conf/log4j2-scripts/filter.js
       charset: UTF-8
-  
+
   Appenders:
 
     # Console
@@ -81,6 +82,9 @@ Configuration:
             IfLastModified:
               age: 30d
 
+    Prometheus:
+      name: Prometheus
+
     # Routing
     Routing:
       name: RoutingAppender
@@ -129,10 +133,12 @@ Configuration:
     # Default root logger configuration
     Root:
       level: info
-      additivity: false
+      additivity: true
       AppenderRef:
         - ref: "${sys:pulsar.log.appender}"
           level: "${sys:pulsar.log.level}"
+        - ref: Prometheus
+          level: info
 
     Logger:
       - name: org.apache.bookkeeper.bookie.BookieShell
@@ -146,7 +152,7 @@ Configuration:
         additivity: false
         AppenderRef:
           - ref: Console
-    
+
     # Logger to inject filter script
 #     - name: org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl
 #       level: debug
@@ -158,4 +164,4 @@ Configuration:
 #           onMisMatch: DENY
 #           ScriptRef:
 #             ref: filter.js
-        
+

distribution/server/pom.xml

@@ -99,6 +99,11 @@
       <artifactId>prometheus-metrics-provider</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_log4j2</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-zookeeper</artifactId>

distribution/server/src/assemble/LICENSE.bin.txt

@@ -357,6 +357,7 @@ The Apache Software License, Version 2.0
     - io.prometheus-simpleclient_common-0.0.23.jar
     - io.prometheus-simpleclient_hotspot-0.0.23.jar
     - io.prometheus-simpleclient_servlet-0.0.23.jar
+    - io.prometheus-simpleclient_log4j2-0.0.23.jar
  * Bean Validation API -- javax.validation-validation-api-1.1.0.Final.jar
  * Log4J
     - log4j-log4j-1.2.17.jar

pom.xml

@@ -674,6 +674,12 @@ flexible messaging model and an intuitive client API.</description>
         <version>${prometheus.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>io.prometheus</groupId>
+        <artifactId>simpleclient_log4j2</artifactId>
+        <version>${prometheus.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>io.prometheus</groupId>
         <artifactId>simpleclient_servlet</artifactId>