一、Elasticsearch 搜索和数据分析引擎
1. docker 部署 elasticsearch
使用docker 运行单节点 es, 并安装对应版本的 ik_analyzer 分词器
# 运行es
docker run -p 9200:9200 -p 9300:9300 -e discovery.type=single-node --name es elasticsearch:6.8.5
# 进入es的容器内部
docker exec -it containerId /bin/bash
# 安装es的ik分词插件
./bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v6.8.5/elasticsearch-analysis-ik-6.8.5.zip
2. Spring 项目集成 elasticsearch
2.1. pom 中加入 elasticsearch 依赖
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
2.2. 配置文件加入es相关连接数据
springboot 2.2x之前的系列配置文件
spring:
data:
elasticsearch:
# 此处的 cluster-name 需要查看 es默认的配置参数, 可能部署方式不一样导致这个不一样, 可以访问es的9200端口查看基础参数
cluster-name: "docker-cluster"
cluster-nodes: 192.168.128.128:9300
springboot 2.2x之后的版本, 上述配置被标记Deprecated不建议使用, 新版本配置为
spring:
elasticsearch:
rest:
uris: 192.168.128.128:9200
2.3. 使用代码示例
2.3.1 . es 实体数据模型类, index结构可以通过注解的形式, 或者是@mapping知道配置的方式生成
@Data
@Document(indexName = Employee.index, type = "_doc")
//@Mapping(mappingPath = "/mapping/es-employee.json")
public class Employee {
public static final String index = "employee-index";
@Id
@Field(type = FieldType.Long)
private Long id;
@Field(type = FieldType.Keyword)
private String no;
@Field(type = FieldType.Keyword)
private String name;
@Field(type = FieldType.Integer)
private Integer age;
@Field(analyzer = "ik_max_word", type = FieldType.Text, searchAnalyzer = "ik_smart")
private String desc;
@Field(type = FieldType.Date, value = "entry_time")
private Date entryTime;
@Field(type = FieldType.Boolean)
private Boolean deleted;
@Field(type = FieldType.Date, value = "create")
private Date create;
}
elasticsearch 数据结构mapping
{
"properties": {
"id": {
"type": "long"
},
"no": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"age": {
"type": "integer"
},
"desc": {
"type": "text"
},
"entry_time": {
"type": "date"
},
"deleted": {
"type": "boolean"
}
}
}
2.3.2. Elasticsearch 数据操作仓储类, 继承至ElasticsearchRepository, 和 Jpa都是 PagingAndSortingRepository 的子派生类(具有基础的增删改成+分页排序等功能)
public interface EmployeeRepository extends ElasticsearchRepository<Employee, Long> {
Stream<Employee> findByNameLike(String name);
}
2.3.3. es respository查询功能, 具有基础的方法命名式查询和使用queryBuilder来构建查询操作
@Service
@AllArgsConstructor
public class EmployeeQueryServiceImpl implements EmployeeQueryService {
private EmployeeAssembler employeeAssembler;
private EmployeeRepository employeeRepository;
@Override
public Page<EmployeeDTO> employeePage(EmployeeQry qry, Pageable pageable) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (qry.getAge() != null) {
boolQueryBuilder.must(QueryBuilders.termQuery("age", qry.getAge()));
}
if (!StringUtils.isEmpty(qry.getDesc())) {
boolQueryBuilder.must(QueryBuilders.matchQuery("desc", qry.getDesc()));
}
if (!StringUtils.isEmpty(qry.getName())) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery("name", "*" + qry.getName() + "*"));
}
if (qry.getStart() != null || qry.getEnd() != null) {
RangeQueryBuilder entryTime = QueryBuilders.rangeQuery("entryTime");
boolQueryBuilder.must(entryTime);
if (qry.getStart() != null) {
entryTime.gt(qry.getStart().getTime());
}
if (qry.getEnd() != null) {
entryTime.lte(qry.getEnd().getTime());
}
}
if (!StringUtils.isEmpty(qry.getNo())) {
boolQueryBuilder.must(QueryBuilders.fuzzyQuery("no", qry.getNo()));
}
Page<Employee> result = employeeRepository.search(boolQueryBuilder, pageable);
return employeeAssembler.convertDtoPage(result);
}
@Override
public List<EmployeeDTO> findByNameLike(String name) {
return employeeAssembler.convertDTOs(employeeRepository.findByNameLike("*" + name.trim() + "*").filter(x -> x.getDeleted() != null && !x.getDeleted()).collect(Collectors.toList()));
}
}
二、Logstash 应用程序日志、事件的传输、处理、管理和搜索的平台
1. Docker 安装 Logstash 服务
使用 link 来连接 es 容器
docker run --name logstash -d -p 5044:5044 --link es:es docker.elastic.co/logstash/logstash:6.5.0
修改 logstash 的配置文件
docker exec -it containerId /bin/bash
cd /usr/share/logstash/config
vi logstash-sample.conf
logstash 配置日志输出到 es
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.
input {
beats {
port => 5044
}
}
output {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
#user => "elastic"
#password => "changeme"
}
}
2. SpringBoot 集成使用 Logstash
2.1. pom 引入 logstash 的相关依赖
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>5.3</version>
</dependency>
2.2. resources目录下面增加logback.xml配置文件
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<property name="LOG_HOME" value="./logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%red(%d{yyyy-MM-dd HH:mm:ss.SSS}) [%thread] %highlight(%-5level) %logger{50} - %cyan(%msg %n)</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<!-- logstash 服务器 -->
<destination>10.20.22.51:5044</destination>
<!-- 日志输出编码 -->
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
<!-- 自定义fileds, 会输出到es中做个性化查询区分 -->
<customFields>{
"server_name":"application_name",
"environment":"${spring.profiles.active}"
}
</customFields>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
<appender-ref ref="LOGSTASH"/>
</root>
</configuration>
三、Kibana Elasticsearch的数据可视化平台
1. Docker 安装部署 Kibana 服务
使用 ELASTICSEARCH_URL 绑定对应的 es 服务, 也可用通过 --link, 需要注意镜像版本最好和es的保持一致
docker run --name kibana6.8.5 -e ELASTICSEARCH_URL=http://192.168.172.20:9200 -p 5601:5601 -d kibana:6.8.5
通过 http://ip:5601 访问 kibana, 选择 management 查看index信息
创建 Index Patterns 来查询日志数据
