java查詢es超過10000條數據
背景:需要每天零點導出es中日志數據到數據庫中給數據分析人員做清洗,然后展示給業務人員。但在es中默認一次最多只能查詢10000條數據。
在這里我就只貼一下關鍵代碼
SearchRequest searchRequest = new SearchRequest("索引名");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();//時間字段
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("logdate").from("2025-03-20 00:00") // 設置開始時間.to("2025-03-20 23:00") // 設置結束時間.format("yyyy-MM-dd HH:mm") //設置時間格式.includeLower(true) // 包含起始時間點.includeUpper(false); // 不包含結束時間點
searchSourceBuilder.query(rangeQuery);
searchSourceBuilder.size(1000); //每批次大小
SortBuilder sortBuilder = SortBuilders.fieldSort("logDate").order(SortOrder.ASC);
searchSourceBuilder.sort(sortBuilder); searchSourceBuilder.sort(SortBuilders.fieldSort("_id").order(SortOrder.ASC));
Object[] searchAfter = null;
do {if (searchAfter != null){searchSourceBuilder.searchAfter(searchAfter);}searchRequest.source(searchSourceBuilder);SearchResponse search = null;try {search = client.search(searchRequest, RequestOptions.DEFAULT);} catch (IOException e) {throw new RuntimeException(e);}SearchHit[] hits = search.getHits().getHits();if (hits.length==0){break;}for (SearchHit hit : hits) {System.out.println(hit.getSourceAsString());}//獲取最后一個文檔的排序值searchAfter = hits[hits.length-1].getSortValues();}while (searchAfter != null);System.out.println("總共查詢到 "+i+" 條數據");}
上面這段代碼的本質就是用的分頁查詢
所用es相關依賴為
<dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-data-elasticsearch</artifactId><version>2.5.14</version></dependency>
es客戶端為RestHighLevelClient