自定义封装es实现多集群多索引切换

一、前言

在spring框架下操作elasticsearch可采用：

• spring-data-es
• restclient

两种方式去操作，而restclient为es低级客户端，RestHighLevelClient为es高级客户端，在低级基础上进行了封装，提供了更方便、更面向对象的 API。

而项目中为了es有更好的扩展，能更好的进行集群，索引管理，整体基于 es 的原生的 client 来去做，采用 RestHighLevelClient。

二、整体架构设计

首先引入相关依赖：

<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>7.5.2</version>
</dependency>
<dependency> <!--es原生client，es低级客户端-->
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-client</artifactId>
    <version>7.5.2</version>
</dependency>
<dependency> <!--es高级客户端，在低级基础上封装了更多api-->
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.5.2</version>
</dependency>

<dependency> <!--集合工具类-->
    <groupId>org.apache.commons</groupId>
    <artifactId>commons-collections4</artifactId>
    <version>4.4</version>
</dependency>

2.1 Es集群类

@Data
public class EsClusterConfig {

    /**
     * 集群名称
     */
    private String name;

    /**
     * 集群节点, 一个集群下可能会有多个节点
     */
    private String nodes;
}

2.2 Es配置类，专门读取配置文件的配置信息

@Component
@ConfigurationProperties(prefix = "es.cluster")
public class EsConfigProperties {
    /**
     * 读取yml文件的集群信息，用于跨集群处理数据
     */
    private List<EsClusterConfig> esConfigs = new ArrayList<>();

    public List<EsClusterConfig> getEsConfigs() {
        return esConfigs;
    }

    public void setEsConfigs(List<EsClusterConfig> esConfigs) {
        this.esConfigs = esConfigs;
    }
}

yml文件：

es:
  cluster: #多个集群就配置多个
    esConfigs[0]:
      name: cf441bd98016
      nodes: 117.72.118.73:9200 #多节点可用,分隔

2.3 Es索引类

获取索引的前提要知道当前索引位于哪个集群下

@Data
public class EsIndexInfo implements Serializable {
    /**
     * 集群名称
     */
    private String clusterName;
    /**
     * 索引名称
     */
    private String indexName;

}

2.4 Es请求类

@Data
public class EsSearchRequest {
    /**
     * 搜索条件
     */
    private BoolQueryBuilder bq;

    /**
     * 搜索结果中包含哪些字段信息
     */
    private String[] fields;

    /**
     * 起始页数
     */
    private int from;

    /**
     * 页容量
     */
    private int size;

    /**
     * 是否进行快照
     */
    private Boolean needScroll;

    /**
     * 快照缓存时间
     */
    private Long minutes;

    /**
     * 排序字段
     */
    private String sortName;

    /**
     * 排序类型
     */
    private SortOrder sortOrder;

    /**
     * 高亮条件
     */
    private HighlightBuilder highlightBuilder;
}

2.5 Es数据返回类

@Data
public class EsSourceData implements Serializable {
    /**
     * 文档唯一id
     */
    private String docId;

    /**
     * 文档元数据
     */
    private Map<String, Object> data;
}

三、Es集群连接统一管理

@Component
@Log4j2
public class EsRestClient {
    //配置es集群统一连接管理
    /**
     * es集群集合  key：集群名称 value：多节点信息(ip+端口)
     */
    private static Map<String, RestHighLevelClient> clientMap = new HashMap<>();

    @Resource
    private EsConfigProperties esConfigProperties;

    /**
     * 初始化集群连接
     * @PostConstruct: spring初始化bean后执行
     */
    @PostConstruct
    public void initialize() {
        List<EsClusterConfig> esConfigs = esConfigProperties.getEsConfigs();
        for(EsClusterConfig esClusterConfigfig : esConfigs) {
            log.info("initialize.config.name:{}, node:{}", esClusterConfigfig.getName(), esClusterConfigfig.getNodes());
            RestHighLevelClient restHighLevelClient = initRestClient(esClusterConfigfig);
            if(restHighLevelClient != null) { //集群连接成功
                clientMap.put(esClusterConfigfig.getName(), restHighLevelClient);
            } else {
                log.error("config.name:{}, node:{}.initError", esClusterConfigfig.getName(), esClusterConfigfig.getNodes());
            }
        }
    }

    /**
     * 初始化es Map集群集合的value RestHighLevelClient信息
     */
    public RestHighLevelClient initRestClient(EsClusterConfig esClusterConfig) {
        //分割出当前集群下的多个节点
        String[] ipPortArr = esClusterConfig.getNodes().split(",");
        //HttpHost集合可存放ip，端口信息等
        List<HttpHost> httpHostList = new ArrayList<>();
        //遍历把每个节点拆为ip和端口，存入HttpHost集合
        for(String ipPort : ipPortArr) {
            String[] ipPortInfo = ipPort.split(":");
            if(ipPortInfo.length == 2) {
                HttpHost httpHost = new HttpHost(ipPortInfo[0], NumberUtils.toInt(ipPortInfo[1]));
                httpHostList.add(httpHost);
            }
        }
        //把HttpHost集合转为数组
        HttpHost httpHosts[] = new HttpHost[httpHostList.size()];
        httpHostList.toArray(httpHosts);

        RestClientBuilder builder = RestClient.builder(httpHosts); //构造RestClient对象
        RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder); //将RestClient转为高级的RestHighLevelClient

        return restHighLevelClient;
    }
}

四、封装Es常用操作

在EsRestClient类下封装常用操作

/**
 * 获取集群连接
 */
private static RestHighLevelClient getClient(String clusterName) {
    return clientMap.get(clusterName);
}


/**
 * 新增文档
 * @EsIndexInfo： 索引信息(索引名称，集群名称)
 * @EsSourceData： 数据信息(id，元数据)
 */
public static boolean insertDoc(EsIndexInfo esIndexInfo, EsSourceData esSourceData) {
    IndexRequest indexRequest = new IndexRequest(esIndexInfo.getIndexName()); //根据索引名称创建IndexRequest对象
    indexRequest.source(esSourceData.getData()); //传入新增的map数据
    indexRequest.id(esSourceData.getDocId()); //指定id，如未指定会自动生成
    try {
        getClient(esIndexInfo.getClusterName()).index(indexRequest, COMMON_OPTIONS); //获取集群连接，发送请求
        return true;
    } catch (Exception e) {
        log.error("insertDoc.exception:{}", e.getMessage(), e);
    }
    return false;
}

/**
 * 更新文档
 * @EsIndexInfo： 索引信息(索引名称，集群名称)
 * @EsSourceData： 数据信息(id，元数据)
 */
public static boolean updateDoc(EsIndexInfo esIndexInfo, EsSourceData esSourceData) {
    UpdateRequest updateRequest = new UpdateRequest();
    updateRequest.index(esIndexInfo.getIndexName());
    updateRequest.id(esSourceData.getDocId());
    updateRequest.doc(esSourceData.getData());
    updateRequest.docAsUpsert(true); // 如果文档不存在，则插入新文档
    try {
        getClient(esIndexInfo.getClusterName()).update(updateRequest, COMMON_OPTIONS);
        return true;
    } catch (Exception e) {
        log.error("updateDoc.exception:{}", e.getMessage(), e);
    }
    return false;
}

/**
 * 批量更新文档
 * BulkRequest批量处理，其本质就是将多个普通的CRUD请求组合在一起发送。
 */
public static boolean batchUpdateDoc(EsIndexInfo esIndexInfo, List<EsSourceData> esSourceDataList) {
    try {
        boolean flag = false; //标志位，检验是否有文档能够修改
        BulkRequest bulkRequest = new BulkRequest();
        for(EsSourceData esSourceData : esSourceDataList) {
            //当文档id不为空时，则更新
            if(StringUtils.isNotBlank(esSourceData.getDocId())) {
                UpdateRequest updateRequest = new UpdateRequest();
                updateRequest.index(esIndexInfo.getIndexName());
                updateRequest.id(esSourceData.getDocId());
                updateRequest.doc(esSourceData.getData());
                bulkRequest.add(updateRequest);
                flag = true;
            }
        }

        //只要flag为true，就代表有文档能够修改
        if(flag) {
            BulkResponse bulk = getClient(esIndexInfo.getClusterName()).bulk(bulkRequest, COMMON_OPTIONS);
            //判断是否有失败的
            if(bulk.hasFailures()) {
                return false;
            }
        }
        return true;
    } catch (Exception e) {
        log.error("batchUpdateDoc.exception:{}", e.getMessage(), e);
    }
    return false;
}


/**
 * 删除文档
 * DeleteByQueryRequest：根据条件批量删除文档
 */
public static boolean delete(EsIndexInfo esIndexInfo) {
    DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(esIndexInfo.getIndexName());
    deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); //删除所有
    try {
        BulkByScrollResponse response = getClient(esIndexInfo.getClusterName())
                .deleteByQuery(deleteByQueryRequest, COMMON_OPTIONS);

        long deleted = response.getDeleted(); //删除的文档数量
        log.info("deleted.size:{}", deleted);
        return true;
    } catch (Exception e) {
        log.error("delete.exception:{}", e.getMessage(), e);
    }
    return false;
}

/**
 * 根据id删除文档
 */
public static boolean deleteDoc(EsIndexInfo esIndexInfo, String docId) {
    DeleteRequest deleteRequest = new DeleteRequest(esIndexInfo.getIndexName());
    deleteRequest.id(docId);
    try {
        DeleteResponse response = getClient(esIndexInfo.getClusterName()).delete(deleteRequest, COMMON_OPTIONS);
        log.info("deleteDoc.size:{}", JSON.toJSONString(response));
        return true;
    } catch (Exception e) {
        log.error("deleteDoc.exception:{}", e.getMessage(), e);
    }
    return false;
}

/**
 * 检查文档是否存在
 */
public static boolean isExistDocById(EsIndexInfo esIndexInfo, String docId) {
    GetRequest getRequest = new GetRequest(esIndexInfo.getIndexName());
    getRequest.id(docId);
    try {
        return getClient(esIndexInfo.getClusterName()).exists(getRequest, COMMON_OPTIONS);
    } catch (IOException e) {
        log.error("isExistDocById.exception:{}", e.getMessage(), e);
    }
    return false;
}

/**
 * 根据id查询文档指定字段的值(搜索结果只包含指定的字段)
 *
 * @fields：想要获取的字段name FetchSourceContext 构造函数：
 * fetchSource：是否要从搜索结果中获取文档的源数据, false 时，Elasticsearch 不会返回任何源数据
 * includes：搜索结果中只包含哪些字段，null或空时，全部字段
 * excludes：搜索结果中排除那些字段，null或空时，不排除任何字段
 */
public static Map<String, Object> getDocById(EsIndexInfo esIndexInfo, String docId, String[] fields) {
    GetRequest getRequest = new GetRequest(esIndexInfo.getIndexName());
    getRequest.id(docId);
    FetchSourceContext fetchSourceContext = new FetchSourceContext(true, fields, null);
    //GetRequest设置fetchSourceContext属性
    getRequest.fetchSourceContext(fetchSourceContext);
    try {
        GetResponse response = getClient(esIndexInfo.getClusterName()).get(getRequest, COMMON_OPTIONS);
        Map<String, Object> source = response.getSource();
        return source;
    } catch (IOException e) {
        log.error("getDocById.exception:{}", e.getMessage(), e);
    }
    return null;
}

/**
 * 精确搜索
 * @param esSearchRequest es查询条件
 */
public static SearchResponse searchWithTermQuery(EsIndexInfo esIndexInfo, EsSearchRequest esSearchRequest) {
    try {
        BoolQueryBuilder bq = esSearchRequest.getBq(); //搜索条件
        String[] fields = esSearchRequest.getFields(); //搜索字段
        int from = esSearchRequest.getFrom(); //分页起始位置
        int size = esSearchRequest.getSize(); //页容量
        Long minutes = esSearchRequest.getMinutes(); //快照缓存时间
        Boolean needScroll = esSearchRequest.getNeedScroll(); //是否需要快照
        String sortName = esSearchRequest.getSortName(); //排序字段
        SortOrder sortOrder = esSearchRequest.getSortOrder(); //排序方式

        //构建搜索条件（fetchSource搜索结果中包含哪些字段信息）
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(bq);
        searchSourceBuilder.fetchSource(fields, null).from(from).size(size);

        //设置高亮显示
        if (Objects.nonNull(esSearchRequest.getHighlightBuilder())) {
            searchSourceBuilder.highlighter(esSearchRequest.getHighlightBuilder());
        }

        //设置排序字段
        if (StringUtils.isNotBlank(sortName)) {
            searchSourceBuilder.sort(sortName);
        }

        //设置排序字段
        if(Objects.nonNull(sortOrder)) {
            searchSourceBuilder.sort(new ScoreSortBuilder().order(sortOrder));
        } else {
            searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
        }


        //构建搜索请求
        SearchRequest searchRequest = new SearchRequest();
        searchRequest.searchType(SearchType.DEFAULT);
        searchRequest.indices(esIndexInfo.getIndexName());
        searchRequest.source(searchSourceBuilder);
        //设置快照
        if (needScroll) {
            Scroll scroll = new Scroll(TimeValue.timeValueMinutes(minutes));
            searchRequest.scroll(scroll);
        }
        SearchResponse search = getClient(esIndexInfo.getClusterName()).search(searchRequest, COMMON_OPTIONS);
        return search;
    } catch (Exception e) {
        log.error("searchWithTermQuery.exception:{}", e.getMessage(), e);
    }
    return null;
}

searchSourceBuilder有两个方法：

• query：参数为BoolQueryBuilder用于构建搜索的条件，筛选满足条件的文档。
• fetchSource：
  • 第一个参数为：String[] includes，指定搜索结果中只包含 includes 数组中列出的字段
  • 第二个参数为：String[] excludes，指定搜索结果中排除 excludes 数组中列出的字段