未验证 提交 991d0833 编写于 作者: wu-sheng's avatar wu-sheng 提交者: GitHub

Support typeOfMetrics query (#4686)

* Add data type to the column definition to replace the simple isValue definition. And top N query is also requiring value column now.

* Support typeOfMetrics query.
上级 51ce763e
......@@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.query.AggregationQueryService;
import org.apache.skywalking.oap.server.core.query.AlarmQueryService;
import org.apache.skywalking.oap.server.core.query.LogQueryService;
import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
import org.apache.skywalking.oap.server.core.query.ProfileTaskQueryService;
import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
......@@ -90,6 +91,7 @@ public class CoreModule extends ModuleDefine {
private void addQueryService(List<Class> classes) {
classes.add(TopologyQueryService.class);
classes.add(MetricsMetadataQueryService.class);
classes.add(MetricsQueryService.class);
classes.add(TraceQueryService.class);
classes.add(LogQueryService.class);
......
......@@ -48,6 +48,7 @@ import org.apache.skywalking.oap.server.core.query.AggregationQueryService;
import org.apache.skywalking.oap.server.core.query.AlarmQueryService;
import org.apache.skywalking.oap.server.core.query.LogQueryService;
import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
import org.apache.skywalking.oap.server.core.query.ProfileTaskQueryService;
import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
......@@ -213,6 +214,7 @@ public class CoreModuleProvider extends ModuleProvider {
NetworkAddressAliasCache.class, new NetworkAddressAliasCache(moduleConfig));
this.registerServiceImplementation(TopologyQueryService.class, new TopologyQueryService(getManager()));
this.registerServiceImplementation(MetricsMetadataQueryService.class, new MetricsMetadataQueryService());
this.registerServiceImplementation(MetricsQueryService.class, new MetricsQueryService(getManager()));
this.registerServiceImplementation(TraceQueryService.class, new TraceQueryService(getManager()));
this.registerServiceImplementation(LogQueryService.class, new LogQueryService(getManager()));
......
......@@ -60,7 +60,7 @@ public abstract class ApdexMetrics extends Metrics implements IntValueHolder {
private int tNum;
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Avg)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
private int value;
@Entrance
......
......@@ -34,7 +34,7 @@ public abstract class CPMMetrics extends Metrics implements LongValueHolder {
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Avg)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
private long value;
@Getter
@Setter
......
......@@ -33,7 +33,7 @@ public abstract class CountMetrics extends Metrics implements LongValueHolder {
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Sum)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Sum)
private long value;
@Entrance
......
......@@ -44,7 +44,7 @@ public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHol
private long count;
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Avg)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
private double value;
@Entrance
......
......@@ -39,13 +39,9 @@ public abstract class HistogramMetrics extends Metrics {
public static final String DATASET = "dataset";
/**
* The special case when the column is isValue = true, but storageOnly = true, because it is {@link DataTable} type,
* this column can't be query by the aggregation way.
*/
@Getter
@Setter
@Column(columnName = DATASET, isValue = true, storageOnly = true)
@Column(columnName = DATASET, dataType = Column.ValueDataType.HISTOGRAM, storageOnly = true)
private DataTable dataset = new DataTable(30);
/**
......
......@@ -44,7 +44,7 @@ public abstract class LongAvgMetrics extends Metrics implements LongValueHolder
private long count;
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Avg)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
private long value;
@Entrance
......
......@@ -32,7 +32,7 @@ public abstract class MaxDoubleMetrics extends Metrics implements DoubleValueHol
@Getter
@Setter
@Column(columnName = VALUE, isValue = true)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
private double value;
@Entrance
......
......@@ -35,7 +35,7 @@ public abstract class MaxLongMetrics extends Metrics implements LongValueHolder
@Getter
@Setter
@Column(columnName = VALUE, isValue = true)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
private long value;
@Entrance
......
......@@ -32,7 +32,7 @@ public abstract class MinDoubleMetrics extends Metrics implements DoubleValueHol
@Getter
@Setter
@Column(columnName = VALUE, isValue = true)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
private double value = Double.MAX_VALUE;
@Entrance
......
......@@ -32,7 +32,7 @@ public abstract class MinLongMetrics extends Metrics implements LongValueHolder
@Getter
@Setter
@Column(columnName = VALUE, isValue = true)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
private long value = Long.MAX_VALUE;
@Entrance
......
......@@ -38,7 +38,7 @@ public abstract class PercentMetrics extends Metrics implements IntValueHolder {
private long total;
@Getter
@Setter
@Column(columnName = PERCENTAGE, isValue = true, function = Function.Avg)
@Column(columnName = PERCENTAGE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
private int percentage;
@Getter
@Setter
......
......@@ -47,13 +47,9 @@ public abstract class PercentileMetrics extends Metrics implements MultiIntValue
99
};
/**
* The special case when the column is isValue = true, but storageOnly = true, because it is {@link DataTable} type,
* this column can't be query by the aggregation way.
*/
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, storageOnly = true)
@Column(columnName = VALUE, dataType = Column.ValueDataType.LABELED_VALUE, storageOnly = true)
private DataTable percentileValues;
@Getter
@Setter
......
......@@ -43,7 +43,7 @@ public abstract class PxxMetrics extends Metrics implements IntValueHolder {
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Avg)
@Column(columnName = VALUE, dataType = Column.ValueDataType.HISTOGRAM, function = Function.Avg)
private int value;
@Getter
@Setter
......
......@@ -33,7 +33,7 @@ public abstract class SumMetrics extends Metrics implements LongValueHolder {
@Getter
@Setter
@Column(columnName = VALUE, isValue = true, function = Function.Sum)
@Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Sum)
private long value;
@Entrance
......
......@@ -39,7 +39,7 @@ public abstract class TopN extends Record implements ComparableStorageData {
private String statement;
@Getter
@Setter
@Column(columnName = LATENCY)
@Column(columnName = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD)
private long latency;
@Getter
@Setter
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.query;
import java.util.Optional;
import org.apache.skywalking.oap.server.core.query.enumeration.MetricsType;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.library.module.Service;
/**
* MetricsMetadataQueryService provides the metadata of metrics to other modules.
*/
public class MetricsMetadataQueryService implements Service {
public MetricsType typeOfMetrics(String metricsName) {
final Optional<ValueColumnMetadata.ValueColumn> valueColumn
= ValueColumnMetadata.INSTANCE.readValueColumnDefinition(metricsName);
if (valueColumn.isPresent()) {
switch (valueColumn.get().getDataType()) {
case COMMON_VALUE:
return MetricsType.REGULAR_VALUE;
case LABELED_VALUE:
return MetricsType.LABELED_VALUE;
case HISTOGRAM:
return MetricsType.HEATMAP;
case SAMPLED_RECORD:
return MetricsType.SAMPLED_RECORD;
case NOT_VALUE:
default:
return MetricsType.UNKNOWN;
}
} else {
return MetricsType.UNKNOWN;
}
}
}
......@@ -24,6 +24,7 @@ import org.apache.skywalking.oap.server.core.query.input.Duration;
import org.apache.skywalking.oap.server.core.query.input.TopNCondition;
import org.apache.skywalking.oap.server.core.query.type.SelectedRecord;
import org.apache.skywalking.oap.server.core.storage.StorageModule;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.Service;
......@@ -46,6 +47,7 @@ public class TopNRecordsQueryService implements Service {
}
public List<SelectedRecord> readSampledRecords(TopNCondition condition, Duration duration) throws IOException {
return getTopNRecordsQueryDAO().readSampledRecords(condition, duration);
return getTopNRecordsQueryDAO().readSampledRecords(
condition, ValueColumnMetadata.INSTANCE.getValueCName(condition.getName()), duration);
}
}
......@@ -22,6 +22,7 @@ import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import lombok.Getter;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.storage.model.IModelOverride;
......@@ -37,18 +38,13 @@ public @interface Column {
*/
String columnName();
/**
* The column value is used in metrics value query.
*/
boolean isValue() default false;
/**
* The function is used in aggregation query.
*/
Function function() default Function.None;
/**
* The default value of this column, when its {@link #isValue()} == true.
* The default value of this column, when its {@link #dataType()} != {@link ValueDataType#NOT_VALUE}.
*/
int defaultValue() default 0;
......@@ -68,4 +64,50 @@ public @interface Column {
* @since 7.1.0
*/
int length() default 200;
/**
* Column with data type != {@link ValueDataType#NOT_VALUE} represents this is a value column. Indicate it would be
* queried by UI/CLI.
*
* @return the data type of this value column. The value column is the query related value Set {@link
* ValueDataType#NOT_VALUE} if this is not the value column, read {@link ValueDataType} for more details.
* @since 8.0.0
*/
ValueDataType dataType() default ValueDataType.NOT_VALUE;
/**
* ValueDataType represents the data structure of value column. The persistent way of the value column determine the
* available ways to query the data.
*/
enum ValueDataType {
/**
* NOT_VALUE represents this value wouldn't be queried directly through metrics v2 protocol. It could be never
* queried, or just through hard code to do so, uch as the lines of topology and service.
*/
NOT_VALUE(false),
/**
* COMMON_VALUE represents a single value, usually int or long.
*/
COMMON_VALUE(true),
/**
* LABELLED_VALUE represents this metrics have multiple values with different labels.
*/
LABELED_VALUE(true),
/**
* HISTOGRAM represents the values are grouped by the buckets, usually suitable for heatmap query.
*/
HISTOGRAM(true),
/**
* SAMPLED_RECORD represents the values are detail data, being persistent by following some sampled rules.
* Usually do topn query based on value column value ASC or DESC.
*/
SAMPLED_RECORD(true);
@Getter
private boolean isValue = false;
ValueDataType(final boolean isValue) {
this.isValue = isValue;
}
}
}
......@@ -20,6 +20,8 @@ package org.apache.skywalking.oap.server.core.storage.annotation;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.server.core.query.sql.Function;
......@@ -35,8 +37,12 @@ public enum ValueColumnMetadata {
/**
* Register the new metadata for the given model name.
*/
public void putIfAbsent(String modelName, String valueCName, Function function, int defaultValue) {
mapping.putIfAbsent(modelName, new ValueColumn(valueCName, function, defaultValue));
public void putIfAbsent(String modelName,
String valueCName,
Column.ValueDataType dataType,
Function function,
int defaultValue) {
mapping.putIfAbsent(modelName, new ValueColumn(valueCName, dataType, function, defaultValue));
}
/**
......@@ -57,6 +63,10 @@ public enum ValueColumnMetadata {
return findColumn(metricsName).defaultValue;
}
public Optional<ValueColumn> readValueColumnDefinition(String metricsName) {
return Optional.ofNullable(mapping.get(metricsName));
}
private ValueColumn findColumn(String metricsName) {
ValueColumn column = mapping.get(metricsName);
if (column == null) {
......@@ -65,9 +75,11 @@ public enum ValueColumnMetadata {
return column;
}
@Getter
@RequiredArgsConstructor
class ValueColumn {
public class ValueColumn {
private final String valueCName;
private final Column.ValueDataType dataType;
private final Function function;
private final int defaultValue;
}
......
......@@ -76,13 +76,13 @@ public class StorageModels implements IModelManager, INewModel, IModelOverride {
modelColumns.add(
new ModelColumn(
new ColumnName(modelName, column.columnName()), field.getType(), column.matchQuery(), column
.storageOnly(), column.isValue(), column.length()));
.storageOnly(), column.dataType().isValue(), column.length()));
if (log.isDebugEnabled()) {
log.debug("The field named {} with the {} type", column.columnName(), field.getType());
}
if (column.isValue()) {
if (column.dataType().isValue()) {
ValueColumnMetadata.INSTANCE.putIfAbsent(
modelName, column.columnName(), column.function(), column.defaultValue());
modelName, column.columnName(), column.dataType(), column.function(), column.defaultValue());
}
List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
......
......@@ -33,5 +33,7 @@ import org.apache.skywalking.oap.server.library.module.Service;
* @since 8.0.0
*/
public interface ITopNRecordsQueryDAO extends Service {
List<SelectedRecord> readSampledRecords(TopNCondition condition, Duration duration) throws IOException;
List<SelectedRecord> readSampledRecords(TopNCondition condition,
final String valueColumnName,
Duration duration) throws IOException;
}
......@@ -26,6 +26,6 @@ public class CoreModuleTest {
public void testOpenServiceList() {
CoreModule coreModule = new CoreModule();
Assert.assertEquals(27, coreModule.services().length);
Assert.assertEquals(28, coreModule.services().length);
}
}
......@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.query.AggregationQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
import org.apache.skywalking.oap.server.core.query.enumeration.MetricsType;
......@@ -44,11 +45,21 @@ public class MetricsQuery implements GraphQLQueryResolver {
private MetricsQueryService metricsQueryService;
private AggregationQueryService queryService;
private TopNRecordsQueryService topNRecordsQueryService;
private MetricsMetadataQueryService metricsMetadataQueryService;
public MetricsQuery(ModuleManager moduleManager) {
this.moduleManager = moduleManager;
}
private MetricsMetadataQueryService getMetricsMetadataQueryService() {
if (metricsMetadataQueryService == null) {
this.metricsMetadataQueryService = moduleManager.find(CoreModule.NAME)
.provider()
.getService(MetricsMetadataQueryService.class);
}
return metricsMetadataQueryService;
}
private AggregationQueryService getQueryService() {
if (queryService == null) {
this.queryService = moduleManager.find(CoreModule.NAME)
......@@ -80,7 +91,7 @@ public class MetricsQuery implements GraphQLQueryResolver {
* Metrics definition metadata query. Response the metrics type which determines the suitable query methods.
*/
public MetricsType typeOfMetrics(String name) throws IOException {
return MetricsType.UNKNOWN;
return getMetricsMetadataQueryService().typeOfMetrics(name);
}
/**
......@@ -118,9 +129,8 @@ public class MetricsQuery implements GraphQLQueryResolver {
/**
* Heatmap is bucket based value statistic result.
*
* @return heapmap including the latency distribution
* {@link HeatMap#getBuckets()}
* {@link HeatMap.HeatMapColumn#getValues()} follows this rule.
* @return heapmap including the latency distribution {@link HeatMap#getBuckets()} {@link
* HeatMap.HeatMapColumn#getValues()} follows this rule.
* <pre>
* key = 0, represents [0, 100), value = count of requests in the latency range.
* key = 100, represents [100, 200), value = count of requests in the latency range.
......
......@@ -45,6 +45,7 @@ public class TopNRecordsQueryEsDAO extends EsDAO implements ITopNRecordsQueryDAO
@Override
public List<SelectedRecord> readSampledRecords(final TopNCondition condition,
final String valueColumnName,
final Duration duration) throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
......@@ -59,7 +60,7 @@ public class TopNRecordsQueryEsDAO extends EsDAO implements ITopNRecordsQueryDAO
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(condition.getTopN())
.sort(TopN.LATENCY, condition.getOrder().equals(Order.DES) ? SortOrder.DESC : SortOrder.ASC);
.sort(valueColumnName, condition.getOrder().equals(Order.DES) ? SortOrder.DESC : SortOrder.ASC);
SearchResponse response = getClient().search(condition.getName(), sourceBuilder);
List<SelectedRecord> results = new ArrayList<>(condition.getTopN());
......@@ -68,7 +69,7 @@ public class TopNRecordsQueryEsDAO extends EsDAO implements ITopNRecordsQueryDAO
SelectedRecord record = new SelectedRecord();
record.setName((String) searchHit.getSourceAsMap().get(TopN.STATEMENT));
record.setRefId((String) searchHit.getSourceAsMap().get(TopN.TRACE_ID));
record.setValue(((Number) searchHit.getSourceAsMap().get(TopN.LATENCY)).toString());
record.setValue(((Number) searchHit.getSourceAsMap().get(valueColumnName)).toString());
results.add(record);
}
......
......@@ -52,6 +52,7 @@ public class TopNRecordsQuery implements ITopNRecordsQueryDAO {
@Override
public List<SelectedRecord> readSampledRecords(final TopNCondition condition,
final String valueColumnName,
final Duration duration) throws IOException {
String function = InfluxConstants.SORT_ASC;
// Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`.
......@@ -62,7 +63,7 @@ public class TopNRecordsQuery implements ITopNRecordsQueryDAO {
}
WhereQueryImpl query = select()
.function(function, TopN.LATENCY, condition.getTopN())
.function(function, valueColumnName, condition.getTopN())
.column(TopN.STATEMENT)
.column(TopN.TRACE_ID)
.from(client.getDatabase(), condition.getName())
......
......@@ -43,6 +43,7 @@ public class H2TopNRecordsQueryDAO implements ITopNRecordsQueryDAO {
@Override
public List<SelectedRecord> readSampledRecords(final TopNCondition condition,
final String valueColumnName,
final Duration duration) throws IOException {
StringBuilder sql = new StringBuilder("select * from " + condition.getName() + " where ");
List<Object> parameters = new ArrayList<>(10);
......@@ -58,7 +59,7 @@ public class H2TopNRecordsQueryDAO implements ITopNRecordsQueryDAO {
sql.append(" and ").append(TopN.TIME_BUCKET).append(" <= ?");
parameters.add(duration.getEndTimeBucket());
sql.append(" order by ").append(TopN.LATENCY);
sql.append(" order by ").append(valueColumnName);
if (condition.getOrder().equals(Order.DES)) {
sql.append(" desc ");
} else {
......@@ -74,7 +75,7 @@ public class H2TopNRecordsQueryDAO implements ITopNRecordsQueryDAO {
SelectedRecord record = new SelectedRecord();
record.setName(resultSet.getString(TopN.STATEMENT));
record.setRefId(resultSet.getString(TopN.TRACE_ID));
record.setValue(resultSet.getString(TopN.LATENCY));
record.setValue(resultSet.getString(valueColumnName));
results.add(record);
}
}
......
......@@ -36,6 +36,7 @@ import org.apache.skywalking.oap.server.core.query.AggregationQueryService;
import org.apache.skywalking.oap.server.core.query.AlarmQueryService;
import org.apache.skywalking.oap.server.core.query.LogQueryService;
import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
import org.apache.skywalking.oap.server.core.query.ProfileTaskQueryService;
import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
......@@ -132,6 +133,7 @@ public class MockCoreModuleProvider extends CoreModuleProvider {
NetworkAddressAliasCache.class, new NetworkAddressAliasCache(moduleConfig));
this.registerServiceImplementation(TopologyQueryService.class, new TopologyQueryService(getManager()));
this.registerServiceImplementation(MetricsMetadataQueryService.class, new MetricsMetadataQueryService());
this.registerServiceImplementation(MetricsQueryService.class, new MetricsQueryService(getManager()));
this.registerServiceImplementation(TraceQueryService.class, new TraceQueryService(getManager()));
this.registerServiceImplementation(LogQueryService.class, new LogQueryService(getManager()));
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册