es学习

es 原生操作

一、es介绍

1、es和mysql的关系

2、es和mysql的应用场景

3、es和mysql的总结

es分词器

POST /_analyze
{
  "text": "黑马程序员",
  "analyzer": "ik_max_word"
}

POST /_analyze
{
  "text": "黑马程序员",
  "analyzer": "ik_smart"
}

4、es 拓展

二、索引库的操作

PUT student
{
  "mappings": {
    "properties": {
      "info":{
        "type": "text",
        "analyzer": "ik_smart"
      },
      "email":{
        "type": "keyword",
        "index": false
      },
      "name":{
        "type": "object",
        "properties": {
          "firstName": {
            "type": "keyword"
          },
          "lastName": {
            "type": "keyword"
          }
        }
      }
    }
  }
}

1、查询和删除

2、索引库的修改

3、新增文档

POST /student/_doc/1
{
  "info":"info",
  "email": "info@qq.com",
  "name": {
    "firstName":"first",
    "lastName":"last"
  }  
}

查询文档

GET /student/_doc/1

删除文档

DELETE /student/_doc/1

4、修改文档

全量修改文档【不存在新增，存在删除再新增】

PUT /student/_doc/1
{
  "info":"info",
  "email": "info@qq.com",
  "name": {
    "firstName":"first",
    "lastName":"last"
  }  
}

修改

POST /student/_update/1
{
  "doc": {
    "email": "2@qq.com"
  }
}

三、java操作es索引库---esclient

public class MyIndexTest {

    private RestHighLevelClient client;

    @BeforeEach
    void setUp() {
        this.client = new RestHighLevelClient(RestClient.builder(
                HttpHost.create("http://localhost:9200")
        ));
    }

    @Test
    void testCreateHotelIndex() throws IOException {
        // 1、创建 Request对象
        CreateIndexRequest request = new CreateIndexRequest("hotel");
        // 2、请求参数，MAPPING_TEMPLATE 是静态常量字符串，内容是创建索引库的DSL语句
        request.source(HotelConstant.MAPPING_TEMPLATE, XContentType.JSON);
        client.indices().create(request, RequestOptions.DEFAULT);
    }


    @Test
    void testDeleteHotelIndex() throws IOException {
        // 1、创建 Request对象
        DeleteIndexRequest request = new DeleteIndexRequest("hotel");
        // 2、请求参数，MAPPING_TEMPLATE 是静态常量字符串，内容是创建索引库的DSL语句
        client.indices().delete(request, RequestOptions.DEFAULT);
    }

    @Test
    void testExistHotelIndex() throws IOException {
        // 1、创建 Request对象
        GetIndexRequest request = new GetIndexRequest("hotel");
        // 2、请求参数，MAPPING_TEMPLATE 是静态常量字符串，内容是创建索引库的DSL语句
        boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
        System.out.println("索引库是否存在?" + exists);
    }


    @Test
    void testGetHotelIndex() throws IOException {
        // 1、创建 Request对象
        GetIndexRequest request = new GetIndexRequest("hotel");
        // 2、请求参数，MAPPING_TEMPLATE 是静态常量字符串，内容是创建索引库的DSL语句
        GetIndexResponse getIndexResponse = client.indices().get(request, RequestOptions.DEFAULT);
    }

    @AfterEach
    void setDown() throws IOException {
        this.client.close();
    }
}

public class MyDocTest {

    private RestHighLevelClient client;

    @Resource
    private TbHotelService hotelService;

    @BeforeEach
    void setUp() {
        this.client = new RestHighLevelClient(RestClient.builder(
                HttpHost.create("http://localhost:9200")
        ));
    }

    @Test
    void testAddDocument() throws IOException {
        TbHotel tbHotel = hotelService.getById(1);
        HotelDoc hotelDoc = BeanUtil.copyProperties(tbHotel, HotelDoc.class);
        // 准备 Request对象
        IndexRequest request = new IndexRequest("hotel").id(tbHotel.getId().toString());
        // 准备文档
        request.source(JSONUtil.toJsonStr(hotelDoc), XContentType.JSON);
        // 发送请求
        client.index(request, RequestOptions.DEFAULT);
    }


    @Test
    void testDelDocument() throws IOException {
        DeleteRequest request = new DeleteRequest("hotel", "1");
        // 发送请求
        client.delete(request, RequestOptions.DEFAULT);
    }

    @Test
    void testGetDocument() throws IOException {
        GetRequest request = new GetRequest("hotel").id("1");
        // 发送请求
        client.get(request, RequestOptions.DEFAULT);
    }

    @Test
    void testUpdateDocument() throws IOException {
        UpdateRequest request = new UpdateRequest("hotel", "1");
        // 全量
        // TbHotel tbHotel = hotelService.getById(1);
        // HotelDoc hotelDoc = BeanUtil.copyProperties(tbHotel, HotelDoc.class);
        // request.doc(JSONUtil.toJsonStr(hotelDoc), XContentType.JSON);
        // 局部更新
        request.doc("name", "erik666", "address", "daye");
        // 发送请求
        client.update(request, RequestOptions.DEFAULT);
    }


    @Test
    void testBatchDocument() throws IOException {
        BulkRequest request = new BulkRequest();
        List<TbHotel> list = hotelService.list();
        for (TbHotel tbHotel : list) {
            request.add(new IndexRequest("hotel")
                    .id(tbHotel.getId().toString())
                    .source(JSONUtil.toJsonStr(tbHotel), XContentType.JSON));
        }
        // 发送请求
        client.bulk(request, RequestOptions.DEFAULT);
    }

    @AfterEach
    void setDown() throws IOException {
        this.client.close();
    }
}

四、DSL Query

1、查询所有

GET hotel/_search
{
  "query": {
    "match_all": {}
  }
}

2、全文检索

GET /hotel/_search
{
  "query": {
    "match": {
      "all": "大冶"
    }
  }
}

GET /hotel/_search
{
  "query": {
    "multi_match": {
      "query": "大冶",
      "fields": ["address", "city", "name"]
    }
  }
}

GET /hotel/_search
{
  "query": {
    "multi_match": {
      "query": "大武冶",
      "fields": ["address", "city", "name"]
    }
  }
}

3、精确查询 【keyword查询】

# 仅仅支持 类型为keyword类型的字段
GET /hotel/_search
{
  "query": {
    "term": {
      "brand": {
        "value": "大冶"
      }
    }
  }
}

# 范围查询
GET /hotel/_search
{
  "query": {
    "range": {
      "price": {
        "gte": 10,
        "lte": 100000
      }
    }
  }
}

附近的人

4、复合查询

5、Function Score Query【权重打分】

# 打分
GET /hotel/_search
{
  "query": {
    "function_score": {
      "query": {
        "match": {
          "all": "大冶"
        }
      },
      "functions": [
        {
          "filter": {
            "term": {
              "brand": "大冶"
            }
          },
          "weight": 10
        }
      ]
    }
  }
}

GET /hotel/_search
{
  "query": {
    "function_score": {
      "query": {
        "match": {
          "all": "大冶"
        }
      },
      "functions": [
        {
          "filter": {
            "term": {
              "brand": "大冶"
            }
          },
          "weight": 10
        },
        {
          "filter": {
            "term": {
              "score": "5"
            }
          },
          "weight": 2
        }
      ],
      "boost_mode": "sum" # 指定运算模式，默认是相乘，可以指定相加
    }
  }
}

6、复合查询 Boolean

案例

7、排序

8、分页

分页的问题

三种分页【迁移数据可以用2、3】

9、高亮问题

GET product_search_en/_search
{
  "query": {
    "match": {
      "productsName": "Module"
    }
  },
  "highlight": {
    "fields": {
      "productsName": {
    "require_field_match": "false"
      }
    }
  }
}

评分 + 高亮

GET /product_search_en/_search
{
  "query": {
    "function_score": {
      "query": {
        "bool": {
          "must": [
            {
              "multi_match": {
                "query": "10GBASE",
                "fields": ["productsName", "productsCommonName"]
              }
              }
          ],
          "filter": [
            {
              "term": {
                "warehouseStatus.usStatus": 1
              }
            },
            {
              "term": {
                "productsStatus": 1
              }
            }
          ]
        }
      },
      "functions": [
        {
          "filter": {
            "term": {
              "label": "Hot"
            }
          },
          "weight": 100
        },
        {
          "filter": {
            "term": {
              "attributes.relateProducts.enAttrName": "Cisco"
            }
          },
          "weight": 1
        }
      ],
      "boost_mode": "sum"
    }
  },
  "highlight": {
    "fields": {
      "productsName": {
        "require_field_match": "false"
      }
    }
  }
}

五、java 操作查询

@Test
void testMatchAll() throws IOException {
    // 准备request
    SearchRequest request = new SearchRequest("hotel");
    // 准备DSL
    request.source().query(QueryBuilders.matchAllQuery());
    // 发送请求
    SearchResponse search = client.search(request, RequestOptions.DEFAULT);
    // 解析响应
    SearchHits searchHits = search.getHits();
    long total = searchHits.getTotalHits().value;
    System.out.println("一共有：" + total + "数据");
    SearchHit[] hits = searchHits.getHits();
    for (SearchHit hit : hits) {
        System.out.println(hit.getSourceAsString());
    }
    System.out.println(search);
}

1、复合查询

2、排序和分页

3、高亮

4、function score打分

5、聚合

6、查询聚合

GET /product_search_en/_search
{
  "query": {
    "multi_match": {
      "query": "Module",
      "fields": []
    }
  }, 
  "size": 100,
  "aggs": {
    "mainIdAgg": {
      "terms": {
        "field": "mainId",
        "size": 20
      }
    }
  }
}

GET /hotel/_search
{
    "size": 0,
    "aggs": {
        "brandAgg": {
            "terms": {
                "field": "brand",
                "size": 20,
                "order": {
                    "scoreAgg.avg": "desc"
                }
            },
            "aggs":{
                "scoreAgg": {
					"stats":{
                        "field": "score"
                    }
                }
            }
        }
    }
}

7、自动补全

PUT /test
{
  "settings": {
    "analysis": {
      "analyzer": { 
        "my_analyzer": { 
          "tokenizer": "ik_max_word",
          "filter": "py"
        }
      },
      "filter": {
        "py": { 
          "type": "pinyin",
          "keep_full_pinyin": false,
          "keep_joined_full_pinyin": true,
          "keep_original": true,
          "limit_first_letter_length": 16,
          "remove_duplicated_term": true,
          "none_chinese_pinyin_tokenize": false
        }
      }
    }
  },
  "mappings": {
      "properties": {
          "name": {
              "type": "text",
              "analyzer": "my_analyzer"
          }
      }
  }
}

完整的设置

PUT /hotel
{
  "settings": {
    "analysis": {
      "analyzer": {
        "text_anlyzer": {
          "tokenizer": "ik_max_word",
          "filter": "py"
        },
        "completion_analyzer": {
          "tokenizer": "keyword",
          "filter": "py"
        }
      },
      "filter": {
        "py": {
          "type": "pinyin",
          "keep_full_pinyin": false,
          "keep_joined_full_pinyin": true,
          "keep_original": true,
          "limit_first_letter_length": 16,
          "remove_duplicated_term": true,
          "none_chinese_pinyin_tokenize": false
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "id":{
        "type": "keyword"
      },
      "name":{
        "type": "text",
        "analyzer": "text_anlyzer",
        "search_analyzer": "ik_smart",
        "copy_to": "all"
      },
      "address":{
        "type": "keyword",
        "index": false
      },
      "price":{
        "type": "integer"
      },
      "score":{
        "type": "integer"
      },
      "brand":{
        "type": "keyword",
        "copy_to": "all"
      },
      "city":{
        "type": "keyword"
      },
      "starName":{
        "type": "keyword"
      },
      "business":{
        "type": "keyword",
        "copy_to": "all"
      },
      "location":{
        "type": "geo_point"
      },
      "pic":{
        "type": "keyword",
        "index": false
      },
      "all":{
        "type": "text",
        "analyzer": "text_anlyzer",
        "search_analyzer": "ik_smart"
      },
      "suggestion":{
          "type": "completion",
          "analyzer": "completion_analyzer"
      }
    }
  }
}

六、数据同步

七、ES集群