未验证 提交 9cc5e76d 编写于 作者: wu-sheng's avatar wu-sheng 提交者: GitHub

Support service inventory extension fields in json format (#2148)

* Add properties for service inventory

* Fix code bugs

* Provide develop doc for inventory extension.
上级 20aa7f9b
...@@ -37,6 +37,8 @@ and private plugin developer should read this. ...@@ -37,6 +37,8 @@ and private plugin developer should read this.
- [Storage extension development guide](storage-extention.md). Help potential contributors to build a new - [Storage extension development guide](storage-extention.md). Help potential contributors to build a new
storage implementor besides the official. storage implementor besides the official.
- [Customize analysis by oal script](write-oal.md). Guide you to use oal script to make your own metric available. - [Customize analysis by oal script](write-oal.md). Guide you to use oal script to make your own metric available.
- [Backend Inventory entity extension](inventory-extension.md). If you want to extend SkyWalking inventory entities, and
want to push upstream back to our Apache OSS repo, please read these principles.
### UI developer ### UI developer
Our UI is constituted by static pages and web container. Our UI is constituted by static pages and web container.
......
# Backend Inventory Entity Extension
SkyWalking includes four inventory entities.
- Service Inventory
- Service Instance Inventory
- Endpoint Inventory
- Network Address Inventory
All metric, topology, trace and alarm are related to these entity IDs.
For understanding the **Service**, **Service Instance** and **Endpoint** concepts,
please read [Project Overview](../concepts-and-designs/overview.md#why-use-skywalking).
For **Network Address Inventory**, it represents all network address, in IP:port, hostname, domain name
formats, which are detected by language agents or other probes.
## Extension
Right now, only **Service Inventory** extension is already supported in backend core.
Service provides field `properties` in Json format, which is usually used for specific service
rather than normal business services, such as Database, Cache, MQ, etc.
For keeping code consistent and friendly in query and visualization, the Json properties
need to follow the rules.
### Database
1. NodeType == **Database(1)**
1. Json properties include following keys.
- `database`. Database name, such as MySQL, PostgreSQL
- `db.type`. Database type, such as sql db, redis db.
- `db.instance`. Database instance name.
...@@ -82,6 +82,17 @@ public class NetworkAddressInventory extends RegisterSource { ...@@ -82,6 +82,17 @@ public class NetworkAddressInventory extends RegisterSource {
return true; return true;
} }
public NetworkAddressInventory getClone() {
NetworkAddressInventory inventory = new NetworkAddressInventory();
inventory.setSequence(getSequence());
inventory.setRegisterTime(getRegisterTime());
inventory.setHeartbeatTime(getHeartbeatTime());
inventory.setName(name);
inventory.setNodeType(nodeType);
return inventory;
}
@Override public void combine(RegisterSource registerSource) { @Override public void combine(RegisterSource registerSource) {
super.combine(registerSource); super.combine(registerSource);
NetworkAddressInventory inventory = (NetworkAddressInventory)registerSource; NetworkAddressInventory inventory = (NetworkAddressInventory)registerSource;
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.register; package org.apache.skywalking.oap.server.core.register;
import com.google.gson.*;
import java.util.*; import java.util.*;
import lombok.*; import lombok.*;
import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.Const;
...@@ -46,6 +47,7 @@ public class ServiceInventory extends RegisterSource { ...@@ -46,6 +47,7 @@ public class ServiceInventory extends RegisterSource {
public static final String NODE_TYPE = "node_type"; public static final String NODE_TYPE = "node_type";
public static final String MAPPING_SERVICE_ID = "mapping_service_id"; public static final String MAPPING_SERVICE_ID = "mapping_service_id";
public static final String MAPPING_LAST_UPDATE_TIME = "mapping_last_update_time"; public static final String MAPPING_LAST_UPDATE_TIME = "mapping_last_update_time";
public static final String PROPERTIES = "properties";
@Setter @Getter @Column(columnName = NAME, matchQuery = true) private String name = Const.EMPTY_STRING; @Setter @Getter @Column(columnName = NAME, matchQuery = true) private String name = Const.EMPTY_STRING;
@Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress; @Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress;
...@@ -53,6 +55,8 @@ public class ServiceInventory extends RegisterSource { ...@@ -53,6 +55,8 @@ public class ServiceInventory extends RegisterSource {
@Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType; @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType;
@Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceId; @Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceId;
@Setter @Getter @Column(columnName = MAPPING_LAST_UPDATE_TIME) private long mappingLastUpdateTime; @Setter @Getter @Column(columnName = MAPPING_LAST_UPDATE_TIME) private long mappingLastUpdateTime;
@Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop;
@Getter private JsonObject properties;
public NodeType getServiceNodeType() { public NodeType getServiceNodeType() {
return NodeType.get(this.nodeType); return NodeType.get(this.nodeType);
...@@ -86,6 +90,24 @@ public class ServiceInventory extends RegisterSource { ...@@ -86,6 +90,24 @@ public class ServiceInventory extends RegisterSource {
return result; return result;
} }
public void setProperties(JsonObject properties) {
this.properties = properties;
if (properties != null && properties.keySet().size() > 0) {
this.prop = properties.toString();
}
}
private void setProp(String prop) {
this.prop = prop;
if (!Strings.isNullOrEmpty(prop)) {
this.properties = new Gson().fromJson(prop, JsonObject.class);
}
}
public boolean hasProperties() {
return prop != null && prop.length() > 0;
}
public ServiceInventory getClone() { public ServiceInventory getClone() {
ServiceInventory inventory = new ServiceInventory(); ServiceInventory inventory = new ServiceInventory();
inventory.setSequence(getSequence()); inventory.setSequence(getSequence());
...@@ -97,6 +119,7 @@ public class ServiceInventory extends RegisterSource { ...@@ -97,6 +119,7 @@ public class ServiceInventory extends RegisterSource {
inventory.setAddressId(addressId); inventory.setAddressId(addressId);
inventory.setMappingLastUpdateTime(mappingLastUpdateTime); inventory.setMappingLastUpdateTime(mappingLastUpdateTime);
inventory.setMappingServiceId(mappingServiceId); inventory.setMappingServiceId(mappingServiceId);
inventory.setProp(prop);
return inventory; return inventory;
} }
...@@ -133,6 +156,7 @@ public class ServiceInventory extends RegisterSource { ...@@ -133,6 +156,7 @@ public class ServiceInventory extends RegisterSource {
remoteBuilder.addDataLongs(getMappingLastUpdateTime()); remoteBuilder.addDataLongs(getMappingLastUpdateTime());
remoteBuilder.addDataStrings(Strings.isNullOrEmpty(name) ? Const.EMPTY_STRING : name); remoteBuilder.addDataStrings(Strings.isNullOrEmpty(name) ? Const.EMPTY_STRING : name);
remoteBuilder.addDataStrings(Strings.isNullOrEmpty(prop) ? Const.EMPTY_STRING : prop);
return remoteBuilder; return remoteBuilder;
} }
...@@ -148,6 +172,8 @@ public class ServiceInventory extends RegisterSource { ...@@ -148,6 +172,8 @@ public class ServiceInventory extends RegisterSource {
setMappingLastUpdateTime(remoteData.getDataLongs(2)); setMappingLastUpdateTime(remoteData.getDataLongs(2));
setName(remoteData.getDataStrings(0)); setName(remoteData.getDataStrings(0));
setProp(remoteData.getDataStrings(1));
} }
@Override public int remoteHashCode() { @Override public int remoteHashCode() {
...@@ -158,6 +184,7 @@ public class ServiceInventory extends RegisterSource { ...@@ -158,6 +184,7 @@ public class ServiceInventory extends RegisterSource {
super.combine(registerSource); super.combine(registerSource);
ServiceInventory serviceInventory = (ServiceInventory)registerSource; ServiceInventory serviceInventory = (ServiceInventory)registerSource;
nodeType = serviceInventory.nodeType; nodeType = serviceInventory.nodeType;
setProp(serviceInventory.getProp());
if (Const.NONE != serviceInventory.getMappingServiceId() && serviceInventory.getMappingLastUpdateTime() >= this.getMappingLastUpdateTime()) { if (Const.NONE != serviceInventory.getMappingServiceId() && serviceInventory.getMappingLastUpdateTime() >= this.getMappingLastUpdateTime()) {
this.mappingServiceId = serviceInventory.getMappingServiceId(); this.mappingServiceId = serviceInventory.getMappingServiceId();
this.mappingLastUpdateTime = serviceInventory.getMappingLastUpdateTime(); this.mappingLastUpdateTime = serviceInventory.getMappingLastUpdateTime();
...@@ -177,6 +204,7 @@ public class ServiceInventory extends RegisterSource { ...@@ -177,6 +204,7 @@ public class ServiceInventory extends RegisterSource {
inventory.setRegisterTime((Long)dbMap.get(REGISTER_TIME)); inventory.setRegisterTime((Long)dbMap.get(REGISTER_TIME));
inventory.setHeartbeatTime((Long)dbMap.get(HEARTBEAT_TIME)); inventory.setHeartbeatTime((Long)dbMap.get(HEARTBEAT_TIME));
inventory.setMappingLastUpdateTime((Long)dbMap.get(MAPPING_LAST_UPDATE_TIME)); inventory.setMappingLastUpdateTime((Long)dbMap.get(MAPPING_LAST_UPDATE_TIME));
inventory.setProp((String)dbMap.get(PROPERTIES));
return inventory; return inventory;
} }
...@@ -191,6 +219,7 @@ public class ServiceInventory extends RegisterSource { ...@@ -191,6 +219,7 @@ public class ServiceInventory extends RegisterSource {
map.put(REGISTER_TIME, storageData.getRegisterTime()); map.put(REGISTER_TIME, storageData.getRegisterTime());
map.put(HEARTBEAT_TIME, storageData.getHeartbeatTime()); map.put(HEARTBEAT_TIME, storageData.getHeartbeatTime());
map.put(MAPPING_LAST_UPDATE_TIME, storageData.getMappingLastUpdateTime()); map.put(MAPPING_LAST_UPDATE_TIME, storageData.getMappingLastUpdateTime());
map.put(PROPERTIES, storageData.getProp());
return map; return map;
} }
} }
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.register.service; package org.apache.skywalking.oap.server.core.register.service;
import com.google.gson.JsonObject;
import org.apache.skywalking.oap.server.core.register.NodeType; import org.apache.skywalking.oap.server.core.register.NodeType;
import org.apache.skywalking.oap.server.library.module.Service; import org.apache.skywalking.oap.server.library.module.Service;
...@@ -25,7 +26,7 @@ import org.apache.skywalking.oap.server.library.module.Service; ...@@ -25,7 +26,7 @@ import org.apache.skywalking.oap.server.library.module.Service;
* @author peng-yongsheng * @author peng-yongsheng
*/ */
public interface INetworkAddressInventoryRegister extends Service { public interface INetworkAddressInventoryRegister extends Service {
int getOrCreate(String networkAddress); int getOrCreate(String networkAddress, JsonObject properties);
int get(String networkAddress); int get(String networkAddress);
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.register.service; package org.apache.skywalking.oap.server.core.register.service;
import com.google.gson.JsonObject;
import org.apache.skywalking.oap.server.library.module.Service; import org.apache.skywalking.oap.server.library.module.Service;
/** /**
...@@ -25,9 +26,11 @@ import org.apache.skywalking.oap.server.library.module.Service; ...@@ -25,9 +26,11 @@ import org.apache.skywalking.oap.server.library.module.Service;
*/ */
public interface IServiceInventoryRegister extends Service { public interface IServiceInventoryRegister extends Service {
int getOrCreate(String serviceName); int getOrCreate(String serviceName, JsonObject properties);
int getOrCreate(int addressId, String serviceName); int getOrCreate(int addressId, String serviceName, JsonObject properties);
void updateProperties(int serviceId, JsonObject properties);
void heartbeat(int serviceId, long heartBeatTime); void heartbeat(int serviceId, long heartBeatTime);
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.register.service; package org.apache.skywalking.oap.server.core.register.service;
import com.google.gson.JsonObject;
import java.util.Objects; import java.util.Objects;
import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.*;
import org.apache.skywalking.oap.server.core.cache.*; import org.apache.skywalking.oap.server.core.cache.*;
...@@ -73,11 +74,11 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory ...@@ -73,11 +74,11 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory
return this.serviceInstanceInventoryRegister; return this.serviceInstanceInventoryRegister;
} }
@Override public int getOrCreate(String networkAddress) { @Override public int getOrCreate(String networkAddress, JsonObject properties) {
int addressId = getNetworkAddressInventoryCache().getAddressId(networkAddress); int addressId = getNetworkAddressInventoryCache().getAddressId(networkAddress);
if (addressId != Const.NONE) { if (addressId != Const.NONE) {
int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress); int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress, properties);
if (serviceId != Const.NONE) { if (serviceId != Const.NONE) {
int serviceInstanceId = getServiceInstanceInventoryRegister().getOrCreate(serviceId, addressId, System.currentTimeMillis()); int serviceInstanceId = getServiceInstanceInventoryRegister().getOrCreate(serviceId, addressId, System.currentTimeMillis());
...@@ -107,6 +108,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory ...@@ -107,6 +108,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory
@Override public void heartbeat(int addressId, long heartBeatTime) { @Override public void heartbeat(int addressId, long heartBeatTime) {
NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId); NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId);
if (Objects.nonNull(networkAddress)) { if (Objects.nonNull(networkAddress)) {
networkAddress = networkAddress.getClone();
networkAddress.setHeartbeatTime(heartBeatTime); networkAddress.setHeartbeatTime(heartBeatTime);
InventoryProcess.INSTANCE.in(networkAddress); InventoryProcess.INSTANCE.in(networkAddress);
...@@ -119,7 +121,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory ...@@ -119,7 +121,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory
NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId); NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId);
if (!this.compare(networkAddress, nodeType)) { if (!this.compare(networkAddress, nodeType)) {
NetworkAddressInventory newNetworkAddress = getNetworkAddressInventoryCache().get(addressId); NetworkAddressInventory newNetworkAddress = networkAddress.getClone();
newNetworkAddress.setNetworkAddressNodeType(nodeType); newNetworkAddress.setNetworkAddressNodeType(nodeType);
newNetworkAddress.setHeartbeatTime(System.currentTimeMillis()); newNetworkAddress.setHeartbeatTime(System.currentTimeMillis());
...@@ -128,6 +130,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory ...@@ -128,6 +130,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory
ServiceInventory newServiceInventory = getServiceInventoryCache().get(getServiceInventoryCache().getServiceId(networkAddress.getSequence())); ServiceInventory newServiceInventory = getServiceInventoryCache().get(getServiceInventoryCache().getServiceId(networkAddress.getSequence()));
if (!this.compare(newServiceInventory, nodeType)) { if (!this.compare(newServiceInventory, nodeType)) {
newServiceInventory = newServiceInventory.getClone();
newServiceInventory.setServiceNodeType(nodeType); newServiceInventory.setServiceNodeType(nodeType);
newServiceInventory.setHeartbeatTime(System.currentTimeMillis()); newServiceInventory.setHeartbeatTime(System.currentTimeMillis());
......
...@@ -18,10 +18,11 @@ ...@@ -18,10 +18,11 @@
package org.apache.skywalking.oap.server.core.register.service; package org.apache.skywalking.oap.server.core.register.service;
import com.google.gson.JsonObject;
import java.util.Objects; import java.util.Objects;
import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.*;
import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
import org.apache.skywalking.oap.server.core.register.ServiceInventory; import org.apache.skywalking.oap.server.core.register.*;
import org.apache.skywalking.oap.server.core.register.worker.InventoryProcess; import org.apache.skywalking.oap.server.core.register.worker.InventoryProcess;
import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.util.BooleanUtils; import org.apache.skywalking.oap.server.library.util.BooleanUtils;
...@@ -50,7 +51,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { ...@@ -50,7 +51,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
return serviceInventoryCache; return serviceInventoryCache;
} }
@Override public int getOrCreate(String serviceName) { @Override public int getOrCreate(String serviceName, JsonObject properties) {
int serviceId = getServiceInventoryCache().getServiceId(serviceName); int serviceId = getServiceInventoryCache().getServiceId(serviceName);
if (serviceId == Const.NONE) { if (serviceId == Const.NONE) {
...@@ -64,13 +65,14 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { ...@@ -64,13 +65,14 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
serviceInventory.setHeartbeatTime(now); serviceInventory.setHeartbeatTime(now);
serviceInventory.setMappingServiceId(Const.NONE); serviceInventory.setMappingServiceId(Const.NONE);
serviceInventory.setMappingLastUpdateTime(now); serviceInventory.setMappingLastUpdateTime(now);
serviceInventory.setProperties(properties);
InventoryProcess.INSTANCE.in(serviceInventory); InventoryProcess.INSTANCE.in(serviceInventory);
} }
return serviceId; return serviceId;
} }
@Override public int getOrCreate(int addressId, String serviceName) { @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) {
int serviceId = getServiceInventoryCache().getServiceId(addressId); int serviceId = getServiceInventoryCache().getServiceId(addressId);
if (serviceId == Const.NONE) { if (serviceId == Const.NONE) {
...@@ -89,9 +91,23 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { ...@@ -89,9 +91,23 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
return serviceId; return serviceId;
} }
@Override public void updateProperties(int serviceId, JsonObject properties) {
ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId);
if (Objects.nonNull(serviceInventory)) {
serviceInventory = serviceInventory.getClone();
serviceInventory.setProperties(properties);
serviceInventory.setMappingLastUpdateTime(System.currentTimeMillis());
InventoryProcess.INSTANCE.in(serviceInventory);
} else {
logger.warn("Service {} properties update, but not found in storage.");
}
}
@Override public void heartbeat(int serviceId, long heartBeatTime) { @Override public void heartbeat(int serviceId, long heartBeatTime) {
ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId); ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId);
if (Objects.nonNull(serviceInventory)) { if (Objects.nonNull(serviceInventory)) {
serviceInventory = serviceInventory.getClone();
serviceInventory.setHeartbeatTime(heartBeatTime); serviceInventory.setHeartbeatTime(heartBeatTime);
InventoryProcess.INSTANCE.in(serviceInventory); InventoryProcess.INSTANCE.in(serviceInventory);
......
...@@ -44,7 +44,7 @@ public class ServiceMeshMetricDataDecorator { ...@@ -44,7 +44,7 @@ public class ServiceMeshMetricDataDecorator {
boolean isRegistered = true; boolean isRegistered = true;
sourceServiceId = origin.getSourceServiceId(); sourceServiceId = origin.getSourceServiceId();
if (sourceServiceId == Const.NONE) { if (sourceServiceId == Const.NONE) {
sourceServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getSourceServiceName()); sourceServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getSourceServiceName(), null);
if (sourceServiceId != Const.NONE) { if (sourceServiceId != Const.NONE) {
getNewDataBuilder().setSourceServiceId(sourceServiceId); getNewDataBuilder().setSourceServiceId(sourceServiceId);
} else { } else {
...@@ -65,7 +65,7 @@ public class ServiceMeshMetricDataDecorator { ...@@ -65,7 +65,7 @@ public class ServiceMeshMetricDataDecorator {
} }
destServiceId = origin.getDestServiceId(); destServiceId = origin.getDestServiceId();
if (destServiceId == Const.NONE) { if (destServiceId == Const.NONE) {
destServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getDestServiceName()); destServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getDestServiceName(), null);
if (destServiceId != Const.NONE) { if (destServiceId != Const.NONE) {
getNewDataBuilder().setDestServiceId(destServiceId); getNewDataBuilder().setDestServiceId(destServiceId);
} else { } else {
......
...@@ -47,7 +47,7 @@ public class ApplicationRegisterHandler extends ApplicationRegisterServiceGrpc.A ...@@ -47,7 +47,7 @@ public class ApplicationRegisterHandler extends ApplicationRegisterServiceGrpc.A
ApplicationMapping.Builder builder = ApplicationMapping.newBuilder(); ApplicationMapping.Builder builder = ApplicationMapping.newBuilder();
String serviceName = request.getApplicationCode(); String serviceName = request.getApplicationCode();
int serviceId = serviceInventoryRegister.getOrCreate(serviceName); int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null);
if (serviceId != Const.NONE) { if (serviceId != Const.NONE) {
KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(serviceName).setValue(serviceId).build(); KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(serviceName).setValue(serviceId).build();
......
...@@ -50,7 +50,7 @@ public class NetworkAddressRegisterServiceHandler extends NetworkAddressRegister ...@@ -50,7 +50,7 @@ public class NetworkAddressRegisterServiceHandler extends NetworkAddressRegister
NetworkAddressMappings.Builder builder = NetworkAddressMappings.newBuilder(); NetworkAddressMappings.Builder builder = NetworkAddressMappings.newBuilder();
for (String networkAddress : addressesList) { for (String networkAddress : addressesList) {
int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null);
if (addressId != Const.NONE) { if (addressId != Const.NONE) {
KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(networkAddress).setValue(addressId).build(); KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(networkAddress).setValue(addressId).build();
......
...@@ -57,7 +57,7 @@ public class ApplicationRegisterServletHandler extends JettyJsonHandler { ...@@ -57,7 +57,7 @@ public class ApplicationRegisterServletHandler extends JettyJsonHandler {
JsonArray applicationCodes = gson.fromJson(req.getReader(), JsonArray.class); JsonArray applicationCodes = gson.fromJson(req.getReader(), JsonArray.class);
for (int i = 0; i < applicationCodes.size(); i++) { for (int i = 0; i < applicationCodes.size(); i++) {
String applicationCode = applicationCodes.get(i).getAsString(); String applicationCode = applicationCodes.get(i).getAsString();
int applicationId = serviceInventoryRegister.getOrCreate(applicationCode); int applicationId = serviceInventoryRegister.getOrCreate(applicationCode, null);
JsonObject mapping = new JsonObject(); JsonObject mapping = new JsonObject();
mapping.addProperty(APPLICATION_CODE, applicationCode); mapping.addProperty(APPLICATION_CODE, applicationCode);
mapping.addProperty(APPLICATION_ID, applicationId); mapping.addProperty(APPLICATION_ID, applicationId);
......
...@@ -62,7 +62,7 @@ public class NetworkAddressRegisterServletHandler extends JettyJsonHandler { ...@@ -62,7 +62,7 @@ public class NetworkAddressRegisterServletHandler extends JettyJsonHandler {
logger.debug("network getAddress register, network getAddress: {}", networkAddress); logger.debug("network getAddress register, network getAddress: {}", networkAddress);
} }
int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null);
JsonObject mapping = new JsonObject(); JsonObject mapping = new JsonObject();
mapping.addProperty(ADDRESS_ID, addressId); mapping.addProperty(ADDRESS_ID, addressId);
mapping.addProperty(NETWORK_ADDRESS, networkAddress); mapping.addProperty(NETWORK_ADDRESS, networkAddress);
......
...@@ -63,7 +63,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem ...@@ -63,7 +63,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("Register service, service code: {}", serviceName); logger.debug("Register service, service code: {}", serviceName);
} }
int serviceId = serviceInventoryRegister.getOrCreate(serviceName); int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null);
if (serviceId != Const.NONE) { if (serviceId != Const.NONE) {
KeyIntValuePair value = KeyIntValuePair.newBuilder().setKey(serviceName).setValue(serviceId).build(); KeyIntValuePair value = KeyIntValuePair.newBuilder().setKey(serviceName).setValue(serviceId).build();
...@@ -149,7 +149,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem ...@@ -149,7 +149,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem
NetAddressMapping.Builder builder = NetAddressMapping.newBuilder(); NetAddressMapping.Builder builder = NetAddressMapping.newBuilder();
request.getAddressesList().forEach(networkAddress -> { request.getAddressesList().forEach(networkAddress -> {
int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null);
if (addressId != Const.NONE) { if (addressId != Const.NONE) {
builder.addAddressIds(KeyIntValuePair.newBuilder().setKey(networkAddress).setValue(addressId)); builder.addAddressIds(KeyIntValuePair.newBuilder().setKey(networkAddress).setValue(addressId));
...@@ -186,7 +186,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem ...@@ -186,7 +186,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem
return; return;
} }
networkAddressId = networkAddressInventoryRegister.getOrCreate(address); networkAddressId = networkAddressInventoryRegister.getOrCreate(address, null);
if (networkAddressId == Const.NONE) { if (networkAddressId == Const.NONE) {
return; return;
} }
......
...@@ -18,9 +18,9 @@ ...@@ -18,9 +18,9 @@
package org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator; package org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator;
import org.apache.skywalking.apm.network.language.agent.SpanLayer; import java.util.*;
import org.apache.skywalking.apm.network.language.agent.SpanObject; import org.apache.skywalking.apm.network.common.KeyStringValuePair;
import org.apache.skywalking.apm.network.language.agent.SpanType; import org.apache.skywalking.apm.network.language.agent.*;
import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2; import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2;
import static java.util.Objects.isNull; import static java.util.Objects.isNull;
...@@ -281,6 +281,14 @@ public class SpanDecorator implements StandardBuilder { ...@@ -281,6 +281,14 @@ public class SpanDecorator implements StandardBuilder {
return referenceDecorators[index]; return referenceDecorators[index];
} }
public List<KeyStringValuePair> getAllTags() {
if (isOrigin) {
return isV2 ? spanObjectV2.getTagsList() : convert(spanObject.getTagsList());
} else {
return isV2 ? spanBuilderV2.getTagsList() : convert(spanBuilder.getTagsList());
}
}
@Override public void toBuilder() { @Override public void toBuilder() {
if (this.isOrigin) { if (this.isOrigin) {
this.isOrigin = false; this.isOrigin = false;
...@@ -292,4 +300,16 @@ public class SpanDecorator implements StandardBuilder { ...@@ -292,4 +300,16 @@ public class SpanDecorator implements StandardBuilder {
standardBuilder.toBuilder(); standardBuilder.toBuilder();
} }
} }
private List<KeyStringValuePair> convert(List<KeyWithStringValue> list) {
List<KeyStringValuePair> result = new ArrayList<>();
if (list != null) {
list.forEach(element -> {
result.add(KeyStringValuePair.newBuilder()
.setKey(element.getKey())
.setValue(element.getValue()).build());
});
}
return result;
}
} }
...@@ -88,7 +88,7 @@ public class ReferenceIdExchanger implements IdExchanger<ReferenceDecorator> { ...@@ -88,7 +88,7 @@ public class ReferenceIdExchanger implements IdExchanger<ReferenceDecorator> {
} }
if (standardBuilder.getNetworkAddressId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getNetworkAddress())) { if (standardBuilder.getNetworkAddressId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getNetworkAddress())) {
int networkAddressId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getNetworkAddress()); int networkAddressId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getNetworkAddress(), null);
if (networkAddressId == 0) { if (networkAddressId == 0) {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
......
...@@ -19,9 +19,14 @@ ...@@ -19,9 +19,14 @@
package org.apache.skywalking.oap.server.receiver.trace.provider.parser.standardization; package org.apache.skywalking.oap.server.receiver.trace.provider.parser.standardization;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.gson.JsonObject;
import java.util.List;
import org.apache.skywalking.apm.network.common.KeyStringValuePair;
import org.apache.skywalking.apm.network.language.agent.SpanLayer;
import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.*;
import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
import org.apache.skywalking.oap.server.core.register.NodeType; import org.apache.skywalking.oap.server.core.register.*;
import org.apache.skywalking.oap.server.core.register.service.*; import org.apache.skywalking.oap.server.core.register.service.*;
import org.apache.skywalking.oap.server.core.source.DetectPoint; import org.apache.skywalking.oap.server.core.source.DetectPoint;
import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleManager;
...@@ -36,6 +41,8 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> { ...@@ -36,6 +41,8 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
private static final Logger logger = LoggerFactory.getLogger(SpanIdExchanger.class); private static final Logger logger = LoggerFactory.getLogger(SpanIdExchanger.class);
private static SpanIdExchanger EXCHANGER; private static SpanIdExchanger EXCHANGER;
private final ServiceInventoryCache serviceInventoryCacheDAO;
private final IServiceInventoryRegister serviceInventoryRegister;
private final IEndpointInventoryRegister endpointInventoryRegister; private final IEndpointInventoryRegister endpointInventoryRegister;
private final INetworkAddressInventoryRegister networkAddressInventoryRegister; private final INetworkAddressInventoryRegister networkAddressInventoryRegister;
private final IComponentLibraryCatalogService componentLibraryCatalogService; private final IComponentLibraryCatalogService componentLibraryCatalogService;
...@@ -48,6 +55,8 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> { ...@@ -48,6 +55,8 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
} }
private SpanIdExchanger(ModuleManager moduleManager) { private SpanIdExchanger(ModuleManager moduleManager) {
this.serviceInventoryCacheDAO = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class);
this.serviceInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IServiceInventoryRegister.class);
this.endpointInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IEndpointInventoryRegister.class); this.endpointInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IEndpointInventoryRegister.class);
this.networkAddressInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(INetworkAddressInventoryRegister.class); this.networkAddressInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(INetworkAddressInventoryRegister.class);
this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME).provider().getService(IComponentLibraryCatalogService.class); this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME).provider().getService(IComponentLibraryCatalogService.class);
...@@ -69,10 +78,11 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> { ...@@ -69,10 +78,11 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
} }
} }
int peerId = 0;
if (standardBuilder.getPeerId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getPeer())) { if (standardBuilder.getPeerId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getPeer())) {
int peerId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getPeer()); peerId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getPeer(), buildServiceProperties(standardBuilder));
if (peerId == 0) { if (peerId == Const.NONE) {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("peer: {} in service: {} exchange failed", standardBuilder.getPeer(), serviceId); logger.debug("peer: {} in service: {} exchange failed", standardBuilder.getPeer(), serviceId);
} }
...@@ -81,13 +91,27 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> { ...@@ -81,13 +91,27 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
standardBuilder.toBuilder(); standardBuilder.toBuilder();
standardBuilder.setPeerId(peerId); standardBuilder.setPeerId(peerId);
standardBuilder.setPeer(Const.EMPTY_STRING); standardBuilder.setPeer(Const.EMPTY_STRING);
}
}
if (peerId != Const.NONE) {
int spanLayerValue = standardBuilder.getSpanLayerValue();
networkAddressInventoryRegister.update(peerId, NodeType.fromSpanLayerValue(spanLayerValue));
int spanLayerValue = standardBuilder.getSpanLayerValue(); /**
networkAddressInventoryRegister.update(peerId, NodeType.fromSpanLayerValue(spanLayerValue)); * In some case, conjecture node, such as Database node, could be registered by agents.
* At here, if the target service properties need to be updated,
* it will only be updated at the first time for now.
*/
if (SpanLayer.Database.equals(standardBuilder.getSpanLayer())) {
ServiceInventory newServiceInventory = serviceInventoryCacheDAO.get(serviceInventoryCacheDAO.getServiceId(peerId));
if (!newServiceInventory.hasProperties()) {
serviceInventoryRegister.updateProperties(newServiceInventory.getSequence(), buildServiceProperties(standardBuilder));
}
} }
} }
if (standardBuilder.getOperationNameId() == 0) { if (standardBuilder.getOperationNameId() == Const.NONE) {
String endpointName = Strings.isNullOrEmpty(standardBuilder.getOperationName()) ? Const.DOMAIN_OPERATION_NAME : standardBuilder.getOperationName(); String endpointName = Strings.isNullOrEmpty(standardBuilder.getOperationName()) ? Const.DOMAIN_OPERATION_NAME : standardBuilder.getOperationName();
int endpointId = endpointInventoryRegister.getOrCreate(serviceId, endpointName, DetectPoint.fromSpanType(standardBuilder.getSpanType())); int endpointId = endpointInventoryRegister.getOrCreate(serviceId, endpointName, DetectPoint.fromSpanType(standardBuilder.getSpanType()));
...@@ -104,4 +128,28 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> { ...@@ -104,4 +128,28 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
} }
return true; return true;
} }
private JsonObject buildServiceProperties(SpanDecorator standardBuilder) {
JsonObject properties = new JsonObject();
if (SpanLayer.Database.equals(standardBuilder.getSpanLayer())) {
List<KeyStringValuePair> tags = standardBuilder.getAllTags();
tags.forEach(tag -> {
if ("db.type".equals(tag.getKey())) {
properties.addProperty("type", tag.getValue());
} else if ("db.instance".equals(tag.getKey())) {
properties.addProperty("instance", tag.getValue());
}
});
String componentName;
int id = standardBuilder.getComponentId();
if (id != Const.NONE) {
componentName = componentLibraryCatalogService.getServerNameBasedOnComponent(id);
} else {
componentName = "UNKNOWN";
}
properties.addProperty("database", componentName);
}
return properties;
}
} }
...@@ -48,7 +48,7 @@ public class SpanProcessor { ...@@ -48,7 +48,7 @@ public class SpanProcessor {
// In Zipkin, the local service name represents the application owner. // In Zipkin, the local service name represents the application owner.
String applicationCode = span.localServiceName(); String applicationCode = span.localServiceName();
if (applicationCode != null) { if (applicationCode != null) {
int applicationId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(applicationCode); int applicationId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(applicationCode, null);
if (applicationId != 0) { if (applicationId != 0) {
CoreRegisterLinker.getServiceInstanceInventoryRegister().getOrCreate(applicationId, applicationCode, applicationCode, CoreRegisterLinker.getServiceInstanceInventoryRegister().getOrCreate(applicationId, applicationCode, applicationCode,
span.timestampAsLong(), span.timestampAsLong(),
......
...@@ -301,7 +301,7 @@ public class SegmentBuilder { ...@@ -301,7 +301,7 @@ public class SegmentBuilder {
private Segment addApp(String serviceCode, long registerTime) throws Exception { private Segment addApp(String serviceCode, long registerTime) throws Exception {
int serviceId = waitForExchange(() -> int serviceId = waitForExchange(() ->
CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(serviceCode), CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(serviceCode, null),
10 10
); );
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.receiver.zipkin.transform; package org.apache.skywalking.oap.server.receiver.zipkin.transform;
import com.google.gson.JsonObject;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
...@@ -27,7 +28,7 @@ import org.apache.skywalking.apm.network.language.agent.SpanType; ...@@ -27,7 +28,7 @@ import org.apache.skywalking.apm.network.language.agent.SpanType;
import org.apache.skywalking.apm.network.language.agent.v2.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v2.SegmentObject;
import org.apache.skywalking.apm.network.language.agent.v2.SegmentReference; import org.apache.skywalking.apm.network.language.agent.v2.SegmentReference;
import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2; import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2;
import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory; import org.apache.skywalking.oap.server.core.register.*;
import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister; import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister;
import org.apache.skywalking.oap.server.core.register.service.IServiceInventoryRegister; import org.apache.skywalking.oap.server.core.register.service.IServiceInventoryRegister;
import org.apache.skywalking.oap.server.receiver.zipkin.CoreRegisterLinker; import org.apache.skywalking.oap.server.receiver.zipkin.CoreRegisterLinker;
...@@ -52,7 +53,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { ...@@ -52,7 +53,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
public void testTransform() throws Exception { public void testTransform() throws Exception {
IServiceInventoryRegister applicationIDService = new IServiceInventoryRegister() { IServiceInventoryRegister applicationIDService = new IServiceInventoryRegister() {
@Override public int getOrCreate(String serviceName) { @Override public int getOrCreate(String serviceName, JsonObject properties) {
String key = "AppCode:" + serviceName; String key = "AppCode:" + serviceName;
if (applicationRegister.containsKey(key)) { if (applicationRegister.containsKey(key)) {
return applicationRegister.get(key); return applicationRegister.get(key);
...@@ -63,7 +64,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { ...@@ -63,7 +64,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
} }
} }
@Override public int getOrCreate(int addressId, String serviceName) { @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) {
String key = "Address:" + serviceName; String key = "Address:" + serviceName;
if (applicationRegister.containsKey(key)) { if (applicationRegister.containsKey(key)) {
return applicationRegister.get(key); return applicationRegister.get(key);
...@@ -74,6 +75,9 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { ...@@ -74,6 +75,9 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
} }
} }
@Override public void updateProperties(int serviceId, JsonObject properties) {
}
@Override public void heartbeat(int serviceId, long heartBeatTime) { @Override public void heartbeat(int serviceId, long heartBeatTime) {
} }
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册