解决ES每次只能查询一万条数据的问题
@Overridepublic List<ESHandleDto> getVisitorsNum(String startTime, String endTime, String schoolName, String typeFunction) throws IOException {List<ESHandleDto> esHandleDtos = new ArrayList<>();SearchRequest searchRequest = new SearchRequest();searchRequest.indices(ElasticEnum.FUNCTIONLOG_INDEX.getValue());SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();if (StringUtils.hasText(schoolName)) {boolQueryBuilder.must(QueryBuilders.termQuery("schoolName.keyword", schoolName));}if (StringUtils.hasText(typeFunction)) {boolQueryBuilder.must(QueryBuilders.termQuery("typeFunction.keyword", typeFunction));}if (StringUtils.hasText(startTime) && StringUtils.hasText(endTime)) {boolQueryBuilder.must(QueryBuilders.rangeQuery("createDate").gte(startTime + "T00:00:00.000Z").lte(endTime + "T23:59:59.000Z"));}BoolQueryBuilder shouldQuery = QueryBuilders.boolQuery();shouldQuery.should().add(QueryBuilders.termQuery("description.keyword", "查询学生信息表"));shouldQuery.should().add(QueryBuilders.termQuery("description.keyword", "获取学校访客数据"));boolQueryBuilder.must(shouldQuery);sourceBuilder.sort("_doc", SortOrder.DESC);sourceBuilder.size(10000);sourceBuilder.query(boolQueryBuilder);searchRequest.scroll(TimeValue.timeValueMinutes(1));searchRequest.source(sourceBuilder);
// System.out.println(sourceBuilder.toString());SearchResponse searchResponse = null;try {searchResponse = esConfig.restHighLevelClient().search(searchRequest, RequestOptions.DEFAULT);} catch (Throwable e) {throw new RuntimeException(e);}scrollHandle(searchResponse, esConfig, esHandleDtos, startTime, endTime);return esHandleDtos;}public static void scrollHandle(SearchResponse searchResponse, ESConfig esConfig, List<ESHandleDto> esHandleDtos, String startTime, String endTime) {SearchHits hits = searchResponse.getHits();String scrollId = searchResponse.getScrollId();SearchHit[] searchHits = hits.getHits();//对结果集处理List<FunctionLogElasticEntity> functionLogs = ESUtil.convertToFunctionLog(searchHits);visitorsResultHandle(functionLogs, esHandleDtos, ESUtil.isMoreThanMonth(startTime, endTime));//滚动查询部分,将从第10001笔数据开始while (searchHits != null && searchHits.length > 0) {SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId);searchScrollRequest.scroll(TimeValue.timeValueMinutes(1));try {searchResponse = esConfig.restHighLevelClient().scroll(searchScrollRequest, RequestOptions.DEFAULT);} catch (Throwable e) {throw new RuntimeException(e);}scrollId = searchResponse.getScrollId();hits = searchResponse.getHits();searchHits = hits.getHits();//对结果集处理functionLogs = ESUtil.convertToFunctionLog(searchHits);visitorsResultHandle(functionLogs, esHandleDtos, ESUtil.isMoreThanMonth(startTime, endTime));}//清除滚动,否则影响下次查询ClearScrollRequest clearScrollRequest = new ClearScrollRequest();clearScrollRequest.addScrollId(scrollId);ClearScrollResponse clearScrollResponse = null;try {clearScrollResponse = esConfig.restHighLevelClient().clearScroll(clearScrollRequest, RequestOptions.DEFAULT);} catch (IOException e) {throw new RuntimeException(e);}boolean succeeded = clearScrollResponse.isSucceeded();System.out.println(succeeded);}