Skip to content

E
easy-es

easy-es / easy-es

通知 96
Star 0
Fork 0
E
easy-es

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 ec4a8f9c 编写于 作者: FelixHPP's avatar FelixHPP
浏览文件
操作

feat: 扩展search API

上级 f81dc7a3
隐藏空白更改
内联 并排
Showing with 135 addition and 85 deletion
easy-es-core/src/main/java/indi/felix/easy/core/elastic/client/EasyEsService.java
浏览文件 @ ec4a8f9c
package indi.felix.easy.core.elastic.client;
import indi.felix.easy.core.elastic.function.SearchHitsFunction;
import indi.felix.easy.core.elastic.model.MappingModel;
import indi.felix.easy.core.elastic.model.SettingModel;
import indi.felix.easy.core.elastic.utool.Const;
import indi.felix.easy.core.elastic.utool.DelEsListener;
import javafx.util.Pair;
import org.elasticsearch.action.search.*;
import org.elasticsearch.client.*;
import org.elasticsearch.common.unit.TimeValue;
import indi.felix.easy.core.exceptions.ElasticsearchException;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
......@@ -19,14 +16,10 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import static indi.felix.easy.core.elastic.utool.Const.DOC_TYPE;
/**
* TODO 6.8版本以下不支持include_type_name
* host [http://localhost:9200], URI [/cat?master_timeout=30sinclude_type_name=falsetimeout=30s], status line [HTTP/1.1 400 Bad Request]
......@@ -166,14 +159,52 @@ public class EasyEsService extends AbstractService {
return response.getHits();
}
public SearchResponse searchResponsePage(String index, Integer form, Integer size) throws IOException {
SearchRequest searchRequest = new SearchRequest(index);
/**
* 分页查询
*
* @param index
* @param form
* @param size
* @return
*/
public SearchResponse searchResponsePage(String index, Integer form, Integer size) {
//构建搜索条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//QueryBuilders.matchAllQuery查询所有
MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
searchSourceBuilder.query(matchAllQueryBuilder);
return searchResponsePage(index, form, size, searchSourceBuilder);
}
/**
* 分页查询
*
* @param index
* @param form
* @param size
* @param query
* @return
*/
public SearchResponse searchResponsePage(String index, Integer form, Integer size, QueryBuilder query) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(query);
return searchResponsePage(index, form, size, searchSourceBuilder);
}
/**
* 分页查询
*
* @param index
* @param form
* @param size
* @param searchSourceBuilder
* @return
*/
public SearchResponse searchResponsePage(String index, Integer form, Integer size, SearchSourceBuilder searchSourceBuilder) {
SearchRequest searchRequest = new SearchRequest(index);
//构建分页
if (form == 1) {
searchSourceBuilder.from();
......@@ -183,25 +214,41 @@ public class EasyEsService extends AbstractService {
searchSourceBuilder.size(size);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
searchRequest.source(searchSourceBuilder);
SearchResponse search = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchResponse search = null;
try {
search = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error(">>> easy-es 请求出错，错误信息：{}", e.getMessage());
}
return search;
}
/**
* 根据分页查询数据 查询全部
*
* @param index 索引名称
* @param form 页数 如果查看第一页传1
* @param form 页数 如果查看第一页传1,
* @param size 数量 默认传10
* @return 查询好的SearchHit数组
* @throws IOException es连接异常
*/
public SearchHits searchPage(String index, Integer form, Integer size) throws IOException {
public SearchHits searchHisPage(String index, Integer form, Integer size) {
SearchResponse search = searchResponsePage(index, form, size);
/*List<Map<String, Object>> maps = toList(search.getHits());
PageInfo<Map<String, Object>> mapPageInfo = new PageInfo<>(maps);*/
//return mapPageInfo;
return search.getHits();
}
public SearchHits searchHisPage(String index, Integer form, Integer size, QueryBuilder query) {
SearchResponse search = searchResponsePage(index, form, size, query);
return search.getHits();
}
public SearchHits searchHisPage(String index, Integer form, Integer size, SearchSourceBuilder builder) {
SearchResponse search = searchResponsePage(index, form, size, builder);
return search.getHits();
}
......@@ -216,7 +263,7 @@ public class EasyEsService extends AbstractService {
* return map
* })
*/
public void scrollForResponse(String index, QueryBuilder query, Consumer<SearchResponse> consumer) throws IOException {
public void scrollForResponse(String index, QueryBuilder query, Consumer<SearchResponse> consumer) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//返回字段信息
// searchSourceBuilder.fetchSource(new String[]{"id", "x_dip","i_dport"}, new String[]{});
......@@ -231,59 +278,64 @@ public class EasyEsService extends AbstractService {
* @param index
* @param searchSourceBuilder
* @param consumer
* @throws IOException
*/
public void scrollForResponse(String index, SearchSourceBuilder searchSourceBuilder, Consumer<SearchResponse> consumer) throws IOException {
public void scrollForResponse(String index, SearchSourceBuilder searchSourceBuilder, Consumer<SearchResponse> consumer) {
//设定滚动时间间隔
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L));
SearchRequest searchRequest = new SearchRequest(index);//索引库
searchRequest.scroll(scroll);
if(searchSourceBuilder.size() < 1000){
if (searchSourceBuilder.size() < 1000) {
//设定每次返回多少条数据
searchSourceBuilder.size(1000);
}
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
consumer.accept(searchResponse);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
try {
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
consumer.accept(searchResponse);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
//遍历搜索命中的数据，直到没有数据
while (searchHits != null && searchHits.length > 0) {
//遍历搜索命中的数据，直到没有数据
while (searchHits != null && searchHits.length > 0) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
consumer.accept(searchResponse);
consumer.accept(searchResponse);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
//清除滚屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
//也可以选择setScrollIds()将多个scrollId一起使用
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = null;
try {
clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
//清除滚屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
//也可以选择setScrollIds()将多个scrollId一起使用
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
boolean succeeded = clearScrollResponse.isSucceeded();
} catch (Exception e) {
logger.error(">>> easy-es 请求出错，错误信息：{}", e.getMessage());
}
boolean succeeded = clearScrollResponse.isSucceeded();
}
public void scrollForHit(String index, QueryBuilder query, Consumer<SearchHit> consumer) throws IOException {
/**
* scroll 查询
*
* @param index
* @param query
* @param consumer
* @throws IOException
*/
public void scrollForHit(String index, QueryBuilder query, Consumer<SearchHit> consumer) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(query);
scrollForHit(index, searchSourceBuilder, consumer);
}
public void scrollForHit(String index, SearchSourceBuilder searchSourceBuilder, Consumer<SearchHit> consumer) throws IOException {
public void scrollForHit(String index, SearchSourceBuilder searchSourceBuilder, Consumer<SearchHit> consumer) {
scrollForResponse(index, searchSourceBuilder, searchResponse -> forEachHits(searchResponse, consumer));
}
......@@ -305,29 +357,38 @@ public class EasyEsService extends AbstractService {
* @param consumer
* @throws IOException
*/
public void searchForResponse(String index, SearchSourceBuilder builder, Consumer<SearchResponse> consumer) throws IOException {
public void searchForResponse(String index, SearchSourceBuilder builder, Consumer<SearchResponse> consumer) {
SearchRequest searchRequest = new SearchRequest(index);
searchRequest.source(builder);
SearchSourceBuilder sourceBuilder = searchRequest.source();
sourceBuilder.from(0);
sourceBuilder.size(1000); //在构建查询条件时，即可设置大小
//设置排序字段， 先判断有没有id字段， 如果没有就用_uid
if (sourceBuilder.size() < 1000) {
//在构建查询条件时，即可设置大小
sourceBuilder.size(1000);
}
try {
//设置排序字段， 先判断有没有id字段， 如果没有就用_uid
// sourceBuilder.sort(Const.ID, SortOrder.ASC);
sourceBuilder.sort(Const.UID, SortOrder.ASC);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
while (searchHits.length > 0) {
consumer.accept(searchResponse);
SearchHit last = searchHits[searchHits.length - 1];
sourceBuilder.searchAfter(last.getSortValues());
searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
searchHits = searchResponse.getHits().getHits();
sourceBuilder.sort(Const.UID, SortOrder.ASC);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
while (searchHits.length > 0) {
consumer.accept(searchResponse);
SearchHit last = searchHits[searchHits.length - 1];
sourceBuilder.searchAfter(last.getSortValues());
searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
searchHits = searchResponse.getHits().getHits();
}
} catch (Exception e) {
logger.error(">>> easy-es 请求出错，错误信息：{}", e.getMessage());
}
}
public void searchForResponse(String index, QueryBuilder query, Consumer<SearchResponse> consumer) throws IOException {
public void searchForResponse(String index, QueryBuilder query, Consumer<SearchResponse> consumer) {
SearchSourceBuilder searchBuilder = new SearchSourceBuilder();
searchBuilder.query(query);
......@@ -345,15 +406,6 @@ public class EasyEsService extends AbstractService {
searchForResponse(index, query, searchResponse -> forEachHits(searchResponse, consumer));
}
// public static void searchForHit(SearchRequestBuilder requestBuilder, Consumer<SearchHit> consumer) {
// searchForResponse(requestBuilder, searchResponse -> {
// SearchHit[] searchHits = searchResponse.getHits().getHits();
// for (SearchHit searchHit : searchHits) {
// consumer.accept(searchHit);
// }
// });
// }
public void forEachHits(SearchResponse searchResponse, Consumer<SearchHit> consumer) {
if (searchResponse == null) {
return;
......
easy-es-core/src/main/java/indi/felix/easy/core/elastic/function/SearchHitsFunction.java 已删除 100644 → 0
浏览文件 @ f81dc7a3
package indi.felix.easy.core.elastic.function;
import org.elasticsearch.search.SearchHit;
import java.io.Serializable;
import java.util.function.Function;
@FunctionalInterface
public interface SearchHitsFunction<T, R> extends Function<T, R>, Serializable {
default String execute(SearchHit hits) {
// SerializedLambda serializedLambda = this.getSerializedLambda();
// String methodName = serializedLambda.getImplMethodName();
// return resolveFieldName(methodName);
return "";
}
}
easy-es-core/src/test/java/indi/felix/easy/core/elastic/client/EasyEsServiceTest.java
浏览文件 @ ec4a8f9c
......@@ -4,6 +4,8 @@ import indi.felix.easy.core.elastic.EasyEs;
import javafx.util.Pair;
import junit.framework.TestCase;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
......@@ -61,6 +63,10 @@ public class EasyEsServiceTest extends TestCase {
easyEs.search().scrollForResponse("csmsearch", searchSourceBuilder, searchHit -> {
System.out.println("查询" + searchHit.getHits().getHits().length);
});
// easyEs.search().scrollForHit("csmsearch", searchSourceBuilder, searchHit -> {
// System.out.println("查询" + searchHit.getSourceAsMap());
// });
}
public void testSearchAfter() throws IOException {
......@@ -75,7 +81,15 @@ public class EasyEsServiceTest extends TestCase {
}
public void testSearchResponsePage() {
public void testSearchResponsePage() throws IOException {
EasyEs easyEs = new EasyEs("localhost:9200");
TermQueryBuilder termQueryBuilder = new TermQueryBuilder("pa_sex", "女");
// 注意， 第一页要传1，
SearchHits hits = easyEs.search().searchHisPage("csmsearch", 1, 10);
for (SearchHit hit : hits.getHits()){
System.out.println(hit.getId());
}
}
public void testSearchForResponse() {
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或