ES多种分页方式

es有两种分页方式

• from+size 浅分页
• scroll 深分页

在这里就不展开介绍了，网络上有很多相关介绍。 我们还可以通过维护自增主键来实现分页 下面结合Springboot，通过代码展现实现es分页的三种实现方式

需要引入依赖：

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
            <version>2.2.12.RELEASE</version>
        </dependency>

代码：

ESBase.java

@Service
@Slf4j
public class ESBase<T, U extends UserIdxESReq> {

//    @Autowired
//    ElasticsearchRepository<T, String> elasticsearchRepository;

    @Autowired
    ElasticsearchTemplate elasticsearchTemplate;

    @Value("${scroll.timeout.ms:60000}")
    private long scrollTimeoutMs;

    public NativeSearchQueryBuilder getNativeSearchQueryBuilder(U esReq) {
        /*NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        if (StrUtil.isNotBlank(taskItemAddReq.getSettleNo())) {
            TermsQueryBuilder deptNoTermsQb = QueryBuilders.termsQuery(CommonConstant.SETTLE_NO, portBillDetailFactoringFreightReq.getSettleNo());
            boolQueryBuilder.must(deptNoTermsQb);
        }
        searchQueryBuilder.withQuery(boolQueryBuilder);*/
        return new NativeSearchQueryBuilder();
    }

    protected NativeSearchQuery getNativeSearchQuery(U esReq) {
        String reqStr = JSONUtil.toJsonStr(esReq);
        String reqMd5 = SecureUtil.md5(reqStr);
        NativeSearchQuery searchQuery = getNativeSearchQueryBuilder(esReq)
//                .withSort(SortBuilders.fieldSort("pin").order(SortOrder.ASC))
                .withPageable(PageRequest.of(esReq.getPageNo(),
                        esReq.getPageSize())).build();
        searchQuery.setPreference(reqMd5);
        return searchQuery;
    }

    /**
     * 通过自增主键来实现分页
     * @param start
     * @param end
     * @param clazz
     * @return
     */
    public List<T> autoIncrementKeyPage(long start, long end, Class<T> clazz) {
        NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("key").gte(start).lte(end);
        boolQueryBuilder.must(rangeQueryBuilder);

        searchQueryBuilder.withQuery(boolQueryBuilder);
        NativeSearchQuery searchQuery = searchQueryBuilder
                .withSort(SortBuilders.fieldSort("key").order(SortOrder.ASC))
                .build();
        List<T> list = elasticsearchTemplate.queryForList(searchQuery, clazz);
        return list;
    }

    /**
     * from+size 浅分页
     * @param esReq
     * @return
     */
    public EsQueryPojo<List<T>> fromSizePage(U esReq, Class<T> clazz) {
        Page<T> page = elasticsearchTemplate.queryForPage(getNativeSearchQuery(esReq), clazz);
//        Page<T> page = elasticsearchRepository.search(getNativeSearchQuery(esReq));
        EsQueryPojo<List<T>> listEsQueryPojo = new EsQueryPojo<>();
        listEsQueryPojo.setRows(page.getContent());//当前分页返回的结果
        listEsQueryPojo.setPage(page.getNumber());//当前页码
        listEsQueryPojo.setTotal(page.getTotalPages());//分页总数
        listEsQueryPojo.setRecords(page.getTotalElements());//元素总数
        listEsQueryPojo.setSize(page.getSize());//当前页返回的数量
        return listEsQueryPojo;
    }

    /**
     * scroll 深分页,另一种方式是通过维护自增主键
     * @param esReq
     * @return
     */
    public EsScrollQueryPojo<List<T>> scrollPage(U esReq, Class<T> clazz) {
        EsScrollQueryPojo<List<T>> listEsQueryPojo = new EsScrollQueryPojo<>();
        ScrolledPage<T> scrolledPage;
        try {
            if (StringUtils.isEmpty(esReq.getScrollId())) {
                //第一次查询
                scrolledPage = elasticsearchTemplate.startScroll(scrollTimeoutMs, getNativeSearchQuery(esReq), clazz);//游标的过期时间是指两个查询之间的间隔，而不是整个查询过程
            } else {
                //带scrollId查询
                scrolledPage = elasticsearchTemplate.continueScroll(esReq.getScrollId(), scrollTimeoutMs, clazz);
            }

            listEsQueryPojo.setRows(scrolledPage.getContent());//当前分页返回的结果
            listEsQueryPojo.setPage(scrolledPage.getNumber());//当前页码，在深度分页不生效
            listEsQueryPojo.setTotal(scrolledPage.getTotalPages());//分页总数
            listEsQueryPojo.setRecords(scrolledPage.getTotalElements());//元素总数
            listEsQueryPojo.setSize(scrolledPage.getSize());//当前返回的数量
            listEsQueryPojo.setScrollId(scrolledPage.getScrollId());
        } catch (Exception e) {
            log.error(ExceptionEnum.ES_SCROLL_PAGE_EXCEPTION.getMsg() + e.getMessage());
            throw e;
        }
        return listEsQueryPojo;
    }

    public void clearScroll(String scrollId) {
        if (scrollId != null) {
            elasticsearchTemplate.clearScroll(scrollId);
        }
    }

    public void deleteIndexData(Class<T> clazz) {
        if (elasticsearchTemplate.indexExists(clazz)) {
            DeleteQuery deleteQuery = new DeleteQuery();
            deleteQuery.setQuery(QueryBuilders.matchAllQuery());
            elasticsearchTemplate.delete(deleteQuery, clazz);//删除语句
            log.info("clearing index data");
        } else {
            log.warn("index not exsit");
        }
    }

    public long countIndex(Class<T> clazz) {
        if (elasticsearchTemplate.indexExists(clazz)) {
            NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();
            searchQueryBuilder.withQuery(QueryBuilders.matchAllQuery());
            return elasticsearchTemplate.count(searchQueryBuilder.build(), clazz);
        } else {
            log.warn("index not exsit");
            return 0L;
        }
    }
}

UserIdxES.java

@Service
@Slf4j
public class UserIdxES extends ESBase<UserIdx, UserIdxESReq> {

    @Override
    protected NativeSearchQuery getNativeSearchQuery(UserIdxESReq esReq) {
        String reqStr = JSONUtil.toJsonStr(esReq);
        String reqMd5 = SecureUtil.md5(reqStr);
        NativeSearchQuery searchQuery = getNativeSearchQueryBuilder(esReq)
//                .withSort(SortBuilders.fieldSort("pin").order(SortOrder.ASC))
                .withPageable(PageRequest.of(esReq.getPageNo(),
                        esReq.getPageSize())).build();
        searchQuery.setPreference(reqMd5);
        return searchQuery;
    }
}

UserIdx.java

@AllArgsConstructor
@NoArgsConstructor
@Data
@Document(indexName = "user_idx", type = "user_type")
public class UserIdx {

    @Id
    private String id;

    @Field(name = "pin", type = FieldType.Keyword)
    private String pin;

    @Field(name = "name", type = FieldType.Keyword)
    private String name;

    @Field(name = "sex", type = FieldType.Keyword)
    private String sex;

    @Field(name = "age", type = FieldType.Keyword)
    private String age;

    @Field(name = "score", type = FieldType.Keyword)
    private String score;
}

UserIdxESReq.java

@NoArgsConstructor
@AllArgsConstructor
@Data
@ApiModel(value = "UserIdxESReq", description = "UserIdxESReq request bean")
public class UserIdxESReq {

    @ApiModelProperty(example = "0")
    private int pageNo;

    @ApiModelProperty(example = "10")
    private int pageSize;

    @ApiModelProperty(value = "分页ID", name = "scrollId")
    private String scrollId;
}

测试 ESBaseTest.java

@SpringBootTest
@RunWith(SpringRunner.class)
//TODO
public class ESBaseTest {
    @Autowired
    ESBase<UserIdx, UserIdxESReq> esBase;

    @Autowired
    ESBase<UserWithKeyIdx, UserIdxESReq> esBase1;
    @Test
    public void fromSizePageTest() {
        UserIdxESReq UserIdxEsReq = new UserIdxESReq();
        UserIdxEsReq.setPageNo(0);
        UserIdxEsReq.setPageSize(10);
        esBase.fromSizePage(UserIdxEsReq, UserIdx.class);
    }

    @Test
    public void scrollPageTest() {
        UserIdxESReq UserIdxEsReq = new UserIdxESReq();
        UserIdxEsReq.setPageNo(0);
        UserIdxEsReq.setPageSize(10);
        esBase.scrollPage(UserIdxEsReq, UserIdx.class);
    }

    @Test
    public void autoIncrementIdPageTest() {
        List<UserIdx> list = esBase.autoIncrementKeyPage(0, 1000, UserIdx.class);
        Assert.assertNotNull(list);
    }

    @Test
    public void deleteIndexDataTest() {
        esBase.deleteIndexData(UserIdx.class);
    }

    @Test
    public void countIndexTest() {
        long cnt = esBase1.countIndex(UserWithKeyIdx.class);
        Assert.assertEquals(cnt,100);
    }
}

UserIdxESBaseTest.java

@SpringBootTest
@RunWith(SpringRunner.class)
public class UserIdxESBaseTest {

    @Autowired
    UserIdxES esService;

    @Test
    public void fromSizePageTest() {
        UserIdxESReq esReq = new UserIdxESReq();
//        esReq.setAge(123);
//        esReq.setPageNo(0);
        esReq.setPageSize(40);
        int pages = 0;
        int currentPage = 0;
        do {
            esReq.setPageNo(currentPage);
            EsQueryPojo<List<UserIdx>> esScrollQueryPojo = esService.fromSizePage(esReq,UserIdx.class);
            pages = esScrollQueryPojo.getTotal();
            currentPage = esScrollQueryPojo.getPage();
            List<UserIdx> list = esScrollQueryPojo.getRows();
//            currentPage++;
        } while (currentPage++ < pages);
    }

    @Test
    public void scrollPageTest() {
        UserIdxESReq esReq = new UserIdxESReq();
        esReq.setPageNo(0);
        esReq.setPageSize(10);
//        esService.scrollPage(esReq, TaskItemAddIndex.class);
        EsScrollQueryPojo<List<UserIdx>> esScrollQueryPojo = esService.scrollPage(esReq, UserIdx.class);
        List<UserIdx> taskItemAddDtoList = (List<UserIdx>) esScrollQueryPojo.getRows();
        Assert.assertNotNull(taskItemAddDtoList);
    }
}

附录： 2. http 测试笔记

测试es:
spring.data.elasticsearch.cluster-name=docker-cluster
spring.data.elasticsearch.cluster-nodes=x.x.x.x:9300

查看：
    curl -X GET http://x.x.x.x:9200/_cat
创建索引：
    curl -X PUT http://x.x.x.x:9200/user_with_key_idx
删除索引：
    curl -XDELETE http://x.x.x.x:9200/user_with_key_idx
查看mapping
    curl -XGET http://x.x.x.x:9200/user_with_key_idx?pretty
    curl -XGET http://x.x.x.x:9200/user_with_key_idx?pretty
查询数据：
    curl -XGET http://x.x.x.x:9200/user_with_key_idx/user_with_key_type/_search?pretty


深度分页scroll游标使用
    curl -XGET http://x.x.x.x:9200/user_with_key_idx/_search?scroll=1m \
    -H "Content-Type: application/json" \
    -d '
    {
        "query": { "match_all": {}},
        "sort" : ["_doc"],
        "size":  1
    }'

curl -XGET http://x.x.x.x:9200/_search/scroll?pretty \
-H "Content-Type: application/json" \
-d '{
    "scroll": "1m",
    "scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAAJFFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAACRxZTbElxVHEtalMxZWFhRVhqVlJBOUZ3AAAAAAAAAkYWU2xJcVRxLWpTMWVhYUVYalZSQTlGdwAAAAAAAAJIFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAACSRZTbElxVHEtalMxZWFhRVhqVlJBOUZ3"
}'

创建mapping
curl -XPOST 'http://x.x.x.x:9200/user_with_key_idx/user_with_key_type/_mapping?pretty' -H "Content-Type: application/json"  \
-d '
{
    "user_with_key_type": {
        "properties": {
            "pin": {
                "type": "keyword"
            },
            "name": {
                "type": "keyword"
            },
            "sex": {
                "type": "keyword"
            },
            "age": {
                "type": "keyword"
            },
            "score": {
                "type": "keyword"
            },
            "key": {
                "type": "long"
            }
        }
    }
}'

user_with_key_idx/user_with_key_type/_mapping

PUT user_with_key_idx
{
    "mappings": {
        "user_with_key_type": {
                "properties": {
                    "pin": {
                        "type": "keyword"
                    },
                    "name": {
                        "type": "keyword"
                    },
                    "sex": {
                        "type": "keyword"
                    },
                    "age": {
                        "type": "keyword"
                    },
                    "score": {
                        "type": "keyword"
                    },
                     "key": {
                         "type": "long"
                     }
                }
        }
    }
}

插入数据：
    curl -XPOST http://x.x.x.x:9200/user_with_key_idx/user_with_key_type?pretty -H "Content-Type: application/json" \
    -d '{
        "pin": "pin_001",
        "phone": "1234567890",
        "name": "lisi",
        "bizData": "",
        "sex": 1,
        "age": 18
        }'

    curl -XPOST http://x.x.x.x:9200/user_with_key_idx/user_with_key_type?pretty -H "Content-Type: application/json" \
    -d '{
        "pin": "pin_004",
        "phone": "1234567890",
        "name": "lisi02",
        "bizData": "",
        "sex": 1,
        "age": 18
        }'


查询数据：
    curl -XGET http://x.x.x.x:9200/user_with_key_idx/user_with_key_type/_search?pretty

4. 深分页和浅分页的使用区别

• scroll 深分页

curl -XGET http://11.51.197.4:9200/user_idx/user_type/_search?pretty&scroll=3m \
-d '{
  "query":{
    "match_all": {}
  },
  "from": 0,
  "size": 3,
  "sort": [
    {
      "id": {
        "order": "asc"
      }
    }
  ]
}'

curl -XGET http://11.51.197.4:9200/_search/scroll \
-d '{
  "scroll_id":"DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAABvFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAAAcRZTbElxVHEtalMxZWFhRVhqVlJBOUZ3AAAAAAAAAHAWU2xJcVRxLWpTMWVhYUVYalZSQTlGdwAAAAAAAAByFlNsSXFUcS1qUzFlYWFFWGpWUkE5RncAAAAAAAAAcxZTbElxVHEtalMxZWFhRVhqVlJBOUZ3",
  "scroll": "3m"
}'

-浅分页，调整索引的配置项index.max_result_window

PUT index_name/_settings
{
"index.max_result_window":10000
}