Skip to content

microservices-platform
microservices-platform

zhangjian1949 / microservices-platform
与 Fork 源项目一致

Fork自 zlt2000 / microservices-platform

通知 6
Star 0
Fork 0
microservices-platform
microservices-platform

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 588cd5b0 编写于 作者: zlt2000's avatar zlt2000
浏览文件
操作

优化搜索中心search-center，客户端由TransportClient改为HighLevelClient

上级 3798d544
隐藏空白更改
内联 并排
Showing with 259 addition and 516 deletion
zlt-business/search-center/search-server/pom.xml
浏览文件 @ 588cd5b0
......@@ -14,14 +14,22 @@
<groupId>com.zlt</groupId>
<artifactId>zlt-config</artifactId>
</dependency>
<dependency>
<groupId>com.zlt</groupId>
<artifactId>zlt-common-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.zlt</groupId>
<artifactId>search-client</artifactId>
</dependency>
<dependency>
<groupId>com.zlt</groupId>
<artifactId>zlt-elasticsearch-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
......@@ -31,6 +39,7 @@
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-extension</artifactId>
......@@ -51,11 +60,6 @@
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
</dependencies>
<build>
......
zlt-business/search-center/search-server/src/main/java/com/central/admin/controller/IndexController.java
浏览文件 @ 588cd5b0
package com.central.admin.controller;
import com.central.admin.model.IndexDto;
import com.central.admin.model.IndexVo;
import com.central.admin.properties.IndexProperties;
import com.central.admin.service.IIndexService;
import com.central.common.model.PageResult;
......@@ -11,6 +10,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.*;
/**
......@@ -30,7 +30,7 @@ public class IndexController {
private IndexProperties indexProperties;
@PostMapping("/index")
public Result createIndex(@RequestBody IndexDto indexDto) {
public Result createIndex(@RequestBody IndexDto indexDto) throws IOException {
if (indexDto.getNumberOfShards() == null) {
indexDto.setNumberOfShards(1);
}
......@@ -45,7 +45,7 @@ public class IndexController {
* 索引列表
*/
@GetMapping("/indices")
public PageResult<IndexVo> list(@RequestParam(required = false) String queryStr) {
public PageResult<Map<String, String>> list(@RequestParam(required = false) String queryStr) throws IOException {
return indexService.list(queryStr, indexProperties.getShow());
}
......@@ -53,7 +53,7 @@ public class IndexController {
* 索引明细
*/
@GetMapping("/index")
public Result<Map<String, Object>> showIndex(String indexName) {
public Result<Map<String, Object>> showIndex(String indexName) throws IOException {
Map<String, Object> result = indexService.show(indexName);
return Result.succeed(result);
}
......@@ -62,7 +62,7 @@ public class IndexController {
* 删除索引
*/
@DeleteMapping("/index")
public Result deleteIndex(String indexName) {
public Result deleteIndex(String indexName) throws IOException {
indexService.delete(indexName);
return Result.succeed("操作成功");
}
......
zlt-business/search-center/search-server/src/main/java/com/central/admin/model/IndexVo.java 已删除 100644 → 0
浏览文件 @ 3798d544
package com.central.admin.model;
import lombok.Data;
/**
* @author zlt
*/
@Data
public class IndexVo {
/**
* 索引名
*/
private String indexName;
/**
* 文档数
*/
private Long docsCount;
/**
* 文档删除数
*/
private Long docsDeleted;
/**
* 索引大小(kb)
*/
private Double storeSizeInBytes;
/**
* 总查询数
*/
private Long queryCount;
/**
* 总查询耗时(s)
*/
private Double queryTimeInMillis;
}
\ No newline at end of file
zlt-business/search-center/search-server/src/main/java/com/central/admin/properties/IndexProperties.java
浏览文件 @ 588cd5b0
......@@ -18,5 +18,5 @@ public class IndexProperties {
/**
* 配置过滤的索引名：默认只显示这些索引
*/
private String[] show;
private String show;
}
zlt-business/search-center/search-server/src/main/java/com/central/admin/service/IIndexService.java
浏览文件 @ 588cd5b0
package com.central.admin.service;
import com.central.admin.model.IndexDto;
import com.central.admin.model.IndexVo;
import com.central.common.model.PageResult;
import java.io.IOException;
import java.util.Map;
/**
......@@ -16,24 +16,24 @@ public interface IIndexService {
/**
* 创建索引
*/
void create(IndexDto indexDto);
boolean create(IndexDto indexDto) throws IOException;
/**
* 删除索引
* @param indexName 索引名
*/
void delete(String indexName);
boolean delete(String indexName) throws IOException;
/**
* 索引列表
* @param queryStr 搜索字符串
* @param indices 默认显示的索引名
*/
PageResult<IndexVo> list(String queryStr, String... indices);
PageResult<Map<String, String>> list(String queryStr, String indices) throws IOException;
/**
* 显示索引明细
* @param indexName 索引名
*/
Map<String, Object> show(String indexName);
Map<String, Object> show(String indexName) throws IOException;
}
zlt-business/search-center/search-server/src/main/java/com/central/admin/service/impl/IndexServiceImpl.java
浏览文件 @ 588cd5b0
......@@ -4,26 +4,30 @@ import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.central.admin.model.IndexDto;
import com.central.admin.model.IndexVo;
import com.central.admin.service.IIndexService;
import com.central.common.model.PageResult;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.search.stats.SearchStats;
import org.elasticsearch.index.shard.DocsStats;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -38,62 +42,49 @@ import java.util.Map;
@Service
public class IndexServiceImpl implements IIndexService {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
private ElasticsearchRestTemplate elasticsearchRestTemplate;
private ObjectMapper mapper = new ObjectMapper();
@Override
public void create(IndexDto indexDto) {
// setting
Settings settings = Settings.builder()
public boolean create(IndexDto indexDto) throws IOException {
CreateIndexRequest request = new CreateIndexRequest(indexDto.getIndexName());
request.settings(Settings.builder()
.put("index.number_of_shards", indexDto.getNumberOfShards())
.put("index.number_of_replicas", indexDto.getNumberOfReplicas())
.build();
CreateIndexRequestBuilder builder = elasticsearchTemplate.getClient().admin()
.indices()
.prepareCreate(indexDto.getIndexName())
.setSettings(settings);
);
if (StrUtil.isNotEmpty(indexDto.getType()) && StrUtil.isNotEmpty(indexDto.getMappingsSource())) {
//mappings
builder.addMapping(indexDto.getType(), indexDto.getMappingsSource(), XContentType.JSON);
request.mapping(indexDto.getType(), indexDto.getMappingsSource(), XContentType.JSON);
}
builder.get();
CreateIndexResponse response = elasticsearchRestTemplate.getClient()
.indices()
.create(request, RequestOptions.DEFAULT);
return response.isAcknowledged();
}
@Override
public void delete(String indexName) {
elasticsearchTemplate.getClient().admin().indices()
.prepareDelete(indexName)
.execute().actionGet();
public boolean delete(String indexName) throws IOException {
DeleteIndexRequest request = new DeleteIndexRequest(indexName);
DeleteIndexResponse response = elasticsearchRestTemplate.getClient().indices().delete(request, RequestOptions.DEFAULT);
return response.isAcknowledged();
}
@Override
public PageResult<IndexVo> list(String queryStr, String... indices) {
IndicesStatsRequestBuilder indicesBuilder = elasticsearchTemplate.getClient().admin().indices()
.prepareStats(indices);
if (StrUtil.isNotEmpty(queryStr)) {
indicesBuilder.setIndices(queryStr);
}
List<IndexVo> indexList = new ArrayList<>();
try {
IndicesStatsResponse response = indicesBuilder.execute().actionGet();
Map<String, IndexStats> indicesMap = response.getIndices();
for(IndexStats stat : indicesMap.values()) {
IndexVo vo = new IndexVo();
vo.setIndexName(stat.getIndex());
//获取文档数据
DocsStats docsStats = stat.getTotal().getDocs();
vo.setDocsCount(docsStats.getCount());
vo.setDocsDeleted(docsStats.getDeleted());
//获取存储数据
vo.setStoreSizeInBytes(getKB(stat.getTotal().getStore().getSizeInBytes()));
//获取查询数据
SearchStats.Stats searchStats = stat.getTotal().getSearch().getTotal();
vo.setQueryCount(searchStats.getQueryCount());
vo.setQueryTimeInMillis(searchStats.getQueryTimeInMillis() / 1000D);
public PageResult<Map<String, String>> list(String queryStr, String indices) throws IOException {
Response response = elasticsearchRestTemplate.getClient().getLowLevelClient()
.performRequest(new Request(
"GET",
"/_cat/indices?h=health,status,index,docsCount,docsDeleted,storeSize&s=cds:desc&format=json&index="+StrUtil.nullToEmpty(indices)
));
indexList.add(vo);
}
} catch (IndexNotFoundException e) {}
return PageResult.<IndexVo>builder().data(indexList).code(0).build();
List<Map<String, String>> listOfIndicesFromEs = null;
if (response != null) {
String rawBody = EntityUtils.toString(response.getEntity());
TypeReference<List<HashMap<String, String>>> typeRef = new TypeReference<List<HashMap<String, String>>>() {};
listOfIndicesFromEs = mapper.readValue(rawBody, typeRef);
}
return PageResult.<Map<String, String>>builder().data(listOfIndicesFromEs).code(0).build();
}
/**
......@@ -107,28 +98,24 @@ public class IndexServiceImpl implements IIndexService {
}
@Override
public Map<String, Object> show(String indexName) {
ImmutableOpenMap<String, IndexMetaData> stat = elasticsearchTemplate.getClient().admin().cluster()
.prepareState().setIndices(indexName).execute().actionGet()
.getState()
.getMetaData()
.getIndices();
public Map<String, Object> show(String indexName) throws IOException {
GetIndexRequest request = new GetIndexRequest();
request.indices(indexName);
GetIndexResponse getIndexResponse = elasticsearchRestTemplate.getClient()
.indices().get(request, RequestOptions.DEFAULT);
ImmutableOpenMap<String, MappingMetaData> mappOpenMap = getIndexResponse.getMappings().get(indexName);
List<AliasMetaData> indexAliases = getIndexResponse.getAliases().get(indexName);
IndexMetaData indexMetaData = stat.get(indexName);
//获取settings数据
String settingsStr = indexMetaData.getSettings().toString();
String settingsStr = getIndexResponse.getSettings().get(indexName).toString();
Object settingsObj = null;
if (StrUtil.isNotEmpty(settingsStr)) {
settingsObj = JSONObject.parse(settingsStr);
}
ImmutableOpenMap<String, MappingMetaData> mappOpenMap = indexMetaData.getMappings();
ImmutableOpenMap<String, AliasMetaData> aliasesOpenMap = indexMetaData.getAliases();
Map<String, Object> result = new HashMap<>(1);
Map<String, Object> indexMap = new HashMap<>(3);
Map<String, Object> mappMap = new HashMap<>(mappOpenMap.size());
Map<String, Object> aliasesMap = new HashMap<>(aliasesOpenMap.size());
indexMap.put("aliases", aliasesMap);
List<String> aliasesList = new ArrayList<>(indexAliases.size());
indexMap.put("aliases", aliasesList);
indexMap.put("settings", settingsObj);
indexMap.put("mappings", mappMap);
result.put(indexName, indexMap);
......@@ -139,9 +126,8 @@ public class IndexServiceImpl implements IIndexService {
mappMap.put(key.value, dataMap);
}
//获取aliases数据
for (ObjectCursor<String> key : aliasesOpenMap.keys()) {
AliasMetaData data = aliasesOpenMap.get(key.value);
aliasesMap.put(key.value, data.alias());
for (AliasMetaData aliases : indexAliases) {
aliasesList.add(aliases.getAlias());
}
return result;
}
......
zlt-business/search-center/search-server/src/main/java/com/central/search/controller/AggregationController.java
浏览文件 @ 588cd5b0
......@@ -3,9 +3,9 @@ package com.central.search.controller;
import com.central.search.service.IAggregationService;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.Map;
/**
......@@ -19,8 +19,11 @@ import java.util.Map;
@Api(tags = "搜索模块api")
@RequestMapping("/agg")
public class AggregationController {
@Autowired
private IAggregationService aggregationService;
private final IAggregationService aggregationService;
public AggregationController(IAggregationService aggregationService) {
this.aggregationService = aggregationService;
}
/**
* 访问统计聚合查询
......@@ -28,7 +31,7 @@ public class AggregationController {
* @param routing es的路由
*/
@GetMapping("/requestStat/{indexName}/{routing}")
public Map<String, Object> requestStatAgg(@PathVariable String indexName, @PathVariable String routing) {
public Map<String, Object> requestStatAgg(@PathVariable String indexName, @PathVariable String routing) throws IOException {
return aggregationService.requestStatAgg(indexName, routing);
}
}
\ No newline at end of file
zlt-business/search-center/search-server/src/main/java/com/central/search/controller/SearchController.java
浏览文件 @ 588cd5b0
......@@ -5,12 +5,13 @@ import com.central.common.model.*;
import com.central.search.model.SearchDto;
import com.central.search.service.ISearchService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
/**
* 通用搜索
*
......@@ -21,8 +22,11 @@ import lombok.extern.slf4j.Slf4j;
@Api(tags = "搜索模块api")
@RequestMapping("/search")
public class SearchController {
@Autowired
private ISearchService searchService;
private final ISearchService searchService;
public SearchController(ISearchService searchService) {
this.searchService = searchService;
}
/**
* 查询文档列表
......@@ -30,7 +34,7 @@ public class SearchController {
* @param searchDto 搜索Dto
*/
@PostMapping("/{indexName}")
public PageResult<JSONObject> strQuery(@PathVariable String indexName, @RequestBody(required = false) SearchDto searchDto) {
public PageResult<JSONObject> strQuery(@PathVariable String indexName, @RequestBody(required = false) SearchDto searchDto) throws IOException {
if (searchDto == null) {
searchDto = new SearchDto();
}
......
zlt-business/search-center/search-server/src/main/java/com/central/search/service/IAggregationService.java
浏览文件 @ 588cd5b0
package com.central.search.service;
import java.io.IOException;
import java.util.Map;
/**
......@@ -13,5 +14,5 @@ public interface IAggregationService {
* @param routing es的路由
* @return
*/
Map<String, Object> requestStatAgg(String indexName, String routing);
Map<String, Object> requestStatAgg(String indexName, String routing) throws IOException;
}
zlt-business/search-center/search-server/src/main/java/com/central/search/service/ISearchService.java
浏览文件 @ 588cd5b0
......@@ -4,6 +4,8 @@ import com.alibaba.fastjson.JSONObject;
import com.central.common.model.PageResult;
import com.central.search.model.SearchDto;
import java.io.IOException;
/**
* @author zlt
* @date 2019/4/24
......@@ -15,5 +17,5 @@ public interface ISearchService {
* @param searchDto 搜索Dto
* @return
*/
PageResult<JSONObject> strQuery(String indexName, SearchDto searchDto);
PageResult<JSONObject> strQuery(String indexName, SearchDto searchDto) throws IOException;
}
zlt-business/search-center/search-server/src/main/java/com/central/search/service/impl/AggregationServiceImpl.java
浏览文件 @ 588cd5b0
......@@ -4,22 +4,24 @@ import cn.hutool.core.util.StrUtil;
import com.central.common.constant.CommonConstant;
import com.central.search.model.AggItemVo;
import com.central.search.service.IAggregationService;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange;
import org.elasticsearch.search.aggregations.bucket.histogram.*;
import org.elasticsearch.search.aggregations.bucket.range.ParsedDateRange;
import org.elasticsearch.search.aggregations.bucket.range.Range;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
......@@ -36,8 +38,11 @@ import java.util.Map;
*/
@Service
public class AggregationServiceImpl implements IAggregationService {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
private final ElasticsearchRestTemplate elasticsearchRestTemplate;
public AggregationServiceImpl(ElasticsearchRestTemplate elasticsearchRestTemplate) {
this.elasticsearchRestTemplate = elasticsearchRestTemplate;
}
/**
* 访问统计聚合查询，需要es里面提供以下结构的数据
......@@ -107,116 +112,112 @@ public class AggregationServiceImpl implements IAggregationService {
* }
*/
@Override
public Map<String, Object> requestStatAgg(String indexName, String routing) {
public Map<String, Object> requestStatAgg(String indexName, String routing) throws IOException {
DateTime currDt = DateTime.now();
LocalDate localDate = LocalDate.now();
LocalDateTime curDateTime = LocalDateTime.now();
SearchRequestBuilder searchRequestBuilder = elasticsearchTemplate.getClient().prepareSearch(indexName)
.setRouting(routing)
.addAggregation(
//聚合查询当天的数据
AggregationBuilders
.dateRange("currDate")
.field("timestamp")
.addRange(
currDt.withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0).withMillisOfSecond(0), currDt.plusDays(1)
)
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
)
.addAggregation(
//聚合查询7天内的数据
AggregationBuilders
.dateRange("curr24Hour")
.field("timestamp")
.addRange(currDt.minusDays(1), currDt)
.subAggregation(
//聚合并且按小时分组查询当天内的数据
AggregationBuilders
.dateHistogram("statDate")
.field("timestamp")
.dateHistogramInterval(new DateHistogramInterval("90m"))
.format(CommonConstant.DATETIME_FORMAT)
//时区相差8小时
.timeZone(DateTimeZone.forOffsetHours(8))
.minDocCount(0L)
.extendedBounds(new ExtendedBounds(
curDateTime.minusDays(1).format(DateTimeFormatter.ofPattern(CommonConstant.DATETIME_FORMAT)),
curDateTime.format(DateTimeFormatter.ofPattern(CommonConstant.DATETIME_FORMAT))
))
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
)
)
.addAggregation(
//聚合查询7天内的数据
AggregationBuilders
.dateRange("currWeek")
.field("timestamp")
.addRange(currDt.minusDays(7), currDt)
.subAggregation(
//聚合并且按日期分组查询7天内的数据
AggregationBuilders
.dateHistogram("statWeek")
.field("timestamp")
.dateHistogramInterval(DateHistogramInterval.DAY)
.format(CommonConstant.DATE_FORMAT)
//时区相差8小时
.timeZone(DateTimeZone.forOffsetHours(8))
.minDocCount(0L)
.extendedBounds(new ExtendedBounds(
localDate.minusDays(6).format(DateTimeFormatter.ofPattern(CommonConstant.DATE_FORMAT)),
localDate.format(DateTimeFormatter.ofPattern(CommonConstant.DATE_FORMAT))
))
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
)
)
.addAggregation(
//聚合查询30天内的数据
AggregationBuilders
.dateRange("currMonth")
.field("timestamp")
.addRange(currDt.minusDays(30), currDt)
)
.addAggregation(
//聚合查询浏览器的数据
AggregationBuilders
.terms("browser")
.field("browser.keyword")
)
.addAggregation(
//聚合查询操作系统的数据
AggregationBuilders
.terms("operatingSystem")
.field("operatingSystem.keyword")
)
.addAggregation(
//聚合查询1小时内的数据
AggregationBuilders
.dateRange("currHour")
.field("timestamp")
.addRange(
currDt.minusHours(1), currDt
)
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
)
.setSize(0);
SearchResponse response = searchRequestBuilder.get();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder).routing(routing);
searchSourceBuilder.aggregation(
//聚合查询当天的数据
AggregationBuilders
.dateRange("currDate")
.field("timestamp")
.addRange(
currDt.withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0).withMillisOfSecond(0), currDt.plusDays(1)
)
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
).aggregation(
//聚合查询7天内的数据
AggregationBuilders
.dateRange("curr24Hour")
.field("timestamp")
.addRange(currDt.minusDays(1), currDt)
.subAggregation(
//聚合并且按小时分组查询当天内的数据
AggregationBuilders
.dateHistogram("statDate")
.field("timestamp")
.dateHistogramInterval(new DateHistogramInterval("90m"))
.format(CommonConstant.DATETIME_FORMAT)
//时区相差8小时
.timeZone(DateTimeZone.forOffsetHours(8))
.minDocCount(0L)
.extendedBounds(new ExtendedBounds(
curDateTime.minusDays(1).format(DateTimeFormatter.ofPattern(CommonConstant.DATETIME_FORMAT)),
curDateTime.format(DateTimeFormatter.ofPattern(CommonConstant.DATETIME_FORMAT))
))
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
)
).aggregation(
//聚合查询7天内的数据
AggregationBuilders
.dateRange("currWeek")
.field("timestamp")
.addRange(currDt.minusDays(7), currDt)
.subAggregation(
//聚合并且按日期分组查询7天内的数据
AggregationBuilders
.dateHistogram("statWeek")
.field("timestamp")
.dateHistogramInterval(DateHistogramInterval.DAY)
.format(CommonConstant.DATE_FORMAT)
//时区相差8小时
.timeZone(DateTimeZone.forOffsetHours(8))
.minDocCount(0L)
.extendedBounds(new ExtendedBounds(
localDate.minusDays(6).format(DateTimeFormatter.ofPattern(CommonConstant.DATE_FORMAT)),
localDate.format(DateTimeFormatter.ofPattern(CommonConstant.DATE_FORMAT))
))
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
)
).aggregation(
//聚合查询30天内的数据
AggregationBuilders
.dateRange("currMonth")
.field("timestamp")
.addRange(currDt.minusDays(30), currDt)
).aggregation(
//聚合查询浏览器的数据
AggregationBuilders
.terms("browser")
.field("browser.keyword")
).aggregation(
//聚合查询操作系统的数据
AggregationBuilders
.terms("operatingSystem")
.field("operatingSystem.keyword")
).aggregation(
//聚合查询1小时内的数据
AggregationBuilders
.dateRange("currHour")
.field("timestamp")
.addRange(
currDt.minusHours(1), currDt
)
.subAggregation(
AggregationBuilders
.cardinality("uv")
.field("ip.keyword")
)
).size(0);
RestHighLevelClient client = elasticsearchRestTemplate.getClient();
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
Aggregations aggregations = response.getAggregations();
Map<String, Object> result = new HashMap<>(15);
if (aggregations != null) {
......@@ -234,8 +235,8 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值当天统计
*/
private void setCurrDate(Map<String, Object> result, Aggregations aggregations) {
InternalDateRange currDate = aggregations.get("currDate");
InternalDateRange.Bucket bucket = currDate.getBuckets().get(0);
ParsedDateRange currDate = aggregations.get("currDate");
Range.Bucket bucket = currDate.getBuckets().get(0);
Cardinality cardinality = bucket.getAggregations().get("uv");
result.put("currDate_pv", bucket.getDocCount());
result.put("currDate_uv", cardinality.getValue());
......@@ -244,8 +245,8 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值周统计
*/
private void setCurr24Hour(Map<String, Object> result, Aggregations aggregations) {
InternalDateRange curr24Hour = aggregations.get("curr24Hour");
InternalDateRange.Bucket bucket = curr24Hour.getBuckets().get(0);
ParsedDateRange curr24Hour = aggregations.get("curr24Hour");
Range.Bucket bucket = curr24Hour.getBuckets().get(0);
//赋值天趋势统计
setStatDate(result, bucket.getAggregations());
}
......@@ -253,8 +254,8 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值周统计
*/
private void setCurrWeek(Map<String, Object> result, Aggregations aggregations) {
InternalDateRange currWeek = aggregations.get("currWeek");
InternalDateRange.Bucket bucket = currWeek.getBuckets().get(0);
ParsedDateRange currWeek = aggregations.get("currWeek");
Range.Bucket bucket = currWeek.getBuckets().get(0);
result.put("currWeek_pv", bucket.getDocCount());
//赋值周趋势统计
setStatWeek(result, bucket.getAggregations());
......@@ -263,8 +264,8 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值月统计
*/
private void setCurrMonth(Map<String, Object> result, Aggregations aggregations) {
InternalDateRange currMonth = aggregations.get("currMonth");
InternalDateRange.Bucket bucket = currMonth.getBuckets().get(0);
ParsedDateRange currMonth = aggregations.get("currMonth");
Range.Bucket bucket = currMonth.getBuckets().get(0);
result.put("currMonth_pv", bucket.getDocCount());
}
/**
......@@ -288,12 +289,12 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值周趋势统计
*/
private void setStatWeek(Map<String, Object> result, Aggregations aggregations) {
InternalDateHistogram agg = aggregations.get("statWeek");
ParsedDateHistogram agg = aggregations.get("statWeek");
List<String> items = new ArrayList<>();
List<Long> uv = new ArrayList<>();
List<Long> pv = new ArrayList<>();
Cardinality cardinality;
for (InternalDateHistogram.Bucket bucket : agg.getBuckets()) {
for (Histogram.Bucket bucket : agg.getBuckets()) {
items.add(bucket.getKeyAsString());
pv.add(bucket.getDocCount());
......@@ -308,8 +309,8 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值小时内统计-当前在线数
*/
private void setCurrHour(Map<String, Object> result, Aggregations aggregations) {
InternalDateRange currDate = aggregations.get("currHour");
InternalDateRange.Bucket bucket = currDate.getBuckets().get(0);
ParsedDateRange currDate = aggregations.get("currHour");
Range.Bucket bucket = currDate.getBuckets().get(0);
Cardinality cardinality = bucket.getAggregations().get("uv");
result.put("currHour_uv", cardinality.getValue());
}
......@@ -317,12 +318,12 @@ public class AggregationServiceImpl implements IAggregationService {
* 赋值天趋势统计
*/
private void setStatDate(Map<String, Object> result, Aggregations aggregations) {
InternalDateHistogram agg = aggregations.get("statDate");
ParsedDateHistogram agg = aggregations.get("statDate");
List<String> items = new ArrayList<>();
List<Long> uv = new ArrayList<>();
List<Long> pv = new ArrayList<>();
Cardinality cardinality;
for (InternalDateHistogram.Bucket bucket : agg.getBuckets()) {
for (Histogram.Bucket bucket : agg.getBuckets()) {
items.add(getTimeByDatetimeStr(bucket.getKeyAsString()));
pv.add(bucket.getDocCount());
......
zlt-business/search-center/search-server/src/main/java/com/central/search/service/impl/SearchServiceImpl.java
浏览文件 @ 588cd5b0
......@@ -2,14 +2,15 @@ package com.central.search.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.central.common.model.PageResult;
import com.central.es.utils.SearchBuilder;
import com.central.search.model.SearchDto;
import com.central.search.service.ISearchService;
import com.central.search.util.SearchBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
/**
* 通用搜索
*
......@@ -18,8 +19,11 @@ import org.springframework.stereotype.Service;
*/
@Service
public class SearchServiceImpl implements ISearchService {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
private final ElasticsearchRestTemplate elasticsearchRestTemplate;
public SearchServiceImpl(ElasticsearchRestTemplate elasticsearchRestTemplate) {
this.elasticsearchRestTemplate = elasticsearchRestTemplate;
}
/**
* StringQuery通用搜索
......@@ -28,8 +32,8 @@ public class SearchServiceImpl implements ISearchService {
* @return
*/
@Override
public PageResult<JSONObject> strQuery(String indexName, SearchDto searchDto) {
return SearchBuilder.builder(elasticsearchTemplate, indexName)
public PageResult<JSONObject> strQuery(String indexName, SearchDto searchDto) throws IOException {
return SearchBuilder.builder(elasticsearchRestTemplate, indexName)
.setStringQuery(searchDto.getQueryStr())
.addSort(searchDto.getSortCol(), SortOrder.DESC)
.setIsHighlight(searchDto.getIsHighlighter())
......
zlt-business/search-center/search-server/src/main/java/com/central/search/util/SearchBuilder.java 已删除 100644 → 0
浏览文件 @ 3798d544
package com.central.search.util;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.central.common.model.PageResult;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.beanutils.PropertyUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* ES查询Builder
*
* @author zlt
* @date 2019/4/23
*/
@Setter
@Getter
public class SearchBuilder {
/**
* 高亮前缀
*/
private static final String HIGHLIGHTER_PRE_TAGS = "<mark>";
/**
* 高亮后缀
*/
private static final String HIGHLIGHTER_POST_TAGS = "</mark>";
private SearchRequestBuilder searchBuilder;
private SearchBuilder(SearchRequestBuilder searchBuilder) {
this.searchBuilder = searchBuilder;
}
/**
* 生成SearchBuilder实例
* @param elasticsearchTemplate
* @param indexName
*/
public static SearchBuilder builder(ElasticsearchTemplate elasticsearchTemplate, String indexName) {
SearchRequestBuilder searchBuilder = elasticsearchTemplate.getClient().prepareSearch(indexName);
return new SearchBuilder(searchBuilder);
}
/**
* 生成queryStringQuery查询
* @param queryStr 查询关键字
*/
public SearchBuilder setStringQuery(String queryStr) {
QueryBuilder queryBuilder;
if (StrUtil.isNotEmpty(queryStr)) {
queryBuilder = QueryBuilders.queryStringQuery(queryStr);
} else {
queryBuilder = QueryBuilders.matchAllQuery();
}
searchBuilder.setQuery(queryBuilder);
return this;
}
/**
* 设置分页
* @param page 当前页数
* @param limit 每页显示数
*/
public SearchBuilder setPage(Integer page, Integer limit) {
if (page != null && limit != null) {
searchBuilder.setFrom((page - 1) * limit)
.setSize(limit);
}
return this;
}
/**
* 增加排序
* @param field 排序字段
* @param order 顺序方向
*/
public SearchBuilder addSort(String field, SortOrder order) {
if (StrUtil.isNotEmpty(field) && order != null) {
searchBuilder.addSort(field, order);
}
return this;
}
/**
* 设置高亮
* @param preTags 高亮处理前缀
* @param postTags 高亮处理后缀
*/
public SearchBuilder setHighlight(String field, String preTags, String postTags) {
if (StrUtil.isNotEmpty(field) && StrUtil.isNotEmpty(preTags) && StrUtil.isNotEmpty(postTags)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field(field)
.preTags(preTags)
.postTags(postTags);
searchBuilder.highlighter(highlightBuilder);
}
return this;
}
/**
* 设置是否需要高亮处理
* @param isHighlighter 是否需要高亮处理
*/
public SearchBuilder setIsHighlight(Boolean isHighlighter) {
if (BooleanUtil.isTrue(isHighlighter)) {
this.setHighlight("*"
, HIGHLIGHTER_PRE_TAGS, HIGHLIGHTER_POST_TAGS);
}
return this;
}
/**
* 设置查询路由
* @param routing 路由数组
*/
public SearchBuilder setRouting(String... routing) {
if (ArrayUtil.isNotEmpty(routing)) {
searchBuilder.setRouting(routing);
}
return this;
}
/**
* 返回结果 SearchResponse
*/
public SearchResponse get() {
return searchBuilder.execute().actionGet();
}
/**
* 返回列表结果 List<JSONObject>
*/
public List<JSONObject> getList() {
return getList(this.get().getHits());
}
/**
* 返回分页结果 PageResult<JSONObject>
*/
public PageResult<JSONObject> getPage() {
return this.getPage(null, null);
}
/**
* 返回分页结果 PageResult<JSONObject>
* @param page 当前页数
* @param limit 每页显示
*/
public PageResult<JSONObject> getPage(Integer page, Integer limit) {
this.setPage(page, limit);
SearchResponse response = this.get();
SearchHits searchHits = response.getHits();
long totalCnt = searchHits.getTotalHits();
List<JSONObject> list = getList(searchHits);
return PageResult.<JSONObject>builder().data(list).code(0).count(totalCnt).build();
}
/**
* 返回JSON列表数据
*/
private List<JSONObject> getList(SearchHits searchHits) {
List<JSONObject> list = new ArrayList<>();
if (searchHits != null) {
searchHits.forEach(item -> {
JSONObject jsonObject = JSON.parseObject(item.getSourceAsString());
jsonObject.put("id", item.getId());
Map<String, HighlightField> highlightFields = item.getHighlightFields();
if (highlightFields != null) {
populateHighLightedFields(jsonObject, highlightFields);
}
list.add(jsonObject);
});
}
return list;
}
/**
* 组装高亮字符
* @param result 目标对象
* @param highlightFields 高亮配置
*/
private <T> void populateHighLightedFields(T result, Map<String, HighlightField> highlightFields) {
for (HighlightField field : highlightFields.values()) {
try {
String name = field.getName();
if (!name.endsWith(".keyword")) {
PropertyUtils.setProperty(result, field.getName(), concat(field.fragments()));
}
} catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
throw new ElasticsearchException("failed to set highlighted value for field: " + field.getName()
+ " with value: " + Arrays.toString(field.getFragments()), e);
}
}
}
/**
* 拼凑数组为字符串
*/
private String concat(Text[] texts) {
StringBuffer sb = new StringBuffer();
for (Text text : texts) {
sb.append(text.toString());
}
return sb.toString();
}
}
zlt-business/search-center/search-server/src/main/resources/application.yml
浏览文件 @ 588cd5b0
spring:
data:
elasticsearch:
cluster-name: ${zlt.elasticsearch.cluster-name}
cluster-nodes: ${zlt.elasticsearch.cluster-nodes}:9300
elasticsearch:
rest:
uris: ${zlt.elasticsearch.cluster-nodes}:9200
uris: ${zlt.elasticsearch.uris}
username: ${zlt.elasticsearch.username}
password: ${zlt.elasticsearch.password}
zlt:
indices:
show: sys-log-*, mysql-slowlog-*, sys_user, point-log-*, audit-log-*
show: sys-log-*,mysql-slowlog-*,sys_user,point-log-*,audit-log-*
swagger:
enabled: true
title: 搜索中心
......
zlt-config/src/main/resources/application-dev.properties
浏览文件 @ 588cd5b0
......@@ -10,8 +10,9 @@ spring.redis.port=6379
spring.redis.timeout=5000
##### elasticsearch配置
zlt.elasticsearch.cluster-name=my-es
zlt.elasticsearch.cluster-nodes=192.168.28.130
zlt.elasticsearch.uris=192.168.28.130:9200
zlt.elasticsearch.username=elastic
zlt.elasticsearch.password=qEnNfKNujqNrOPD9q5kb
##### sentinel配置
zlt.sentinel.dashboard=192.168.28.130:6999
......
zlt-config/src/main/resources/application-prod.properties
浏览文件 @ 588cd5b0
......@@ -10,8 +10,9 @@ spring.redis.sentinel.nodes=192.168.28.130:26380,192.168.28.131:26381
spring.redis.password=1q2w3e4r
##### elasticsearch配置
zlt.elasticsearch.cluster-name=my-es
zlt.elasticsearch.cluster-nodes=192.168.28.130
zlt.elasticsearch.uris=192.168.28.130:9200
zlt.elasticsearch.username=
zlt.elasticsearch.password=
##### sentinel配置
zlt.sentinel.dashboard=192.168.28.130:6999
......
zlt-config/src/main/resources/application-test.properties
浏览文件 @ 588cd5b0
......@@ -10,8 +10,9 @@ spring.redis.port=6379
spring.redis.timeout=5000
##### elasticsearch配置
zlt.elasticsearch.cluster-name=my-es
zlt.elasticsearch.cluster-nodes=192.168.28.130
zlt.elasticsearch.uris=192.168.28.130:9200
zlt.elasticsearch.username=
zlt.elasticsearch.password=
##### sentinel配置
zlt.sentinel.dashboard=192.168.28.130:6999
......
zlt-gateway/sc-gateway/src/main/java/com/central/gateway/filter/RequestStatisticsFilter.java
浏览文件 @ 588cd5b0
......@@ -50,6 +50,8 @@ public class RequestStatisticsFilter implements GlobalFilter, Ordered {
return "CHROME";
} else if (browser.contains("FIREFOX")) {
return "FIREFOX";
} else if (browser.contains("SAFARI")) {
return "SAFARI";
}
}
return browser;
......
zlt-gateway/zuul-gateway/src/main/java/com/central/gateway/filter/pre/RequestStatisticsFilter.java
浏览文件 @ 588cd5b0
......@@ -57,6 +57,8 @@ public class RequestStatisticsFilter extends ZuulFilter {
return "CHROME";
} else if (browser.contains("FIREFOX")) {
return "FIREFOX";
} else if (browser.contains("SAFARI")) {
return "SAFARI";
}
}
return browser;
......
zlt-web/back-web/src/main/resources/static/pages/search/index_manager.html
浏览文件 @ 588cd5b0
......@@ -37,19 +37,12 @@
page: false,
cols: [[
{type: 'numbers'},
{field: 'indexName', sort: true, title: '索引名'},
{field: 'health', sort: true, title: '索引健康'},
{field: 'status', sort: true, title: '索引状态'},
{field: 'index', sort: true, title: '索引名'},
{field: 'docsCount', sort: true, title: '文档数'},
{field: 'docsDeleted', sort: true, title: '文档删除数'},
{
sort: true, templet: function (d) {
if (d.storeSizeInBytes) {
return parseFloat(d.storeSizeInBytes).toFixed(2);
}
return 0;
}, title: '索引大小(kb)'
},
{field: 'queryCount', sort: true, title: '总查询数'},
{field: 'queryTimeInMillis', sort: true, title: '总查询耗时(s)'},
{field: 'storeSize', sort: true, title: '索引大小'},
{fixed: 'right', align: 'center', toolbar: '#index-table-bar', title: '操作', width: 120}
]]
});
......@@ -87,7 +80,7 @@
//显示索引查看弹窗
let showViewModel = function (data) {
admin.putTempData('indexName', data.indexName);
admin.putTempData('indexName', data.index);
admin.popupCenter({
title: '查看索引',
path: 'pages/search/index_manager_view.html'
......@@ -99,7 +92,7 @@
layer.confirm('确定要删除吗？', function (i) {
layer.close(i);
layer.load(2);
admin.req('api-search/admin/index?indexName='+obj.data.indexName, {}, function (data) {
admin.req('api-search/admin/index?indexName='+obj.data.index, {}, function (data) {
layer.closeAll('loading');
if (data.resp_code == 0) {
layer.msg(data.resp_msg, {icon: 1, time: 500});
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或