1. 安装blackbox-exporter
[root@xy-5-server14 prometheus]# helm pull prometheus-community/prometheus-blackbox-exporter --version 7.7.0
[root@xy-5-server14 prometheus]# ls
kube-prometheus-stack kube-prometheus-stack-45.7.1.tgz prometheus prometheus-19.7.2.tgz prometheus-blackbox-exporter-7.7.0.tgz test
[root@xy-5-server14 prometheus]# tar -zxvf prometheus-blackbox-exporter-7.7.0.tgz
[root@xy-5-server14 prometheus]# cd prometheus-blackbox-exporter/
[root@xy-5-server14 prometheus-blackbox-exporter]# ls
Chart.yaml ci README.md templates values.yaml
[root@xy-5-server14 prometheus-blackbox-exporter]# helm upgrade --install blackbox-exporter -n kube-monitor . --create-namespace -f values.yaml
Release "blackbox-exporter" does not exist. Installing it now.
W0404 10:55:48.205557 41359 warnings.go:70] policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
W0404 10:55:48.299265 41359 warnings.go:70] policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
NAME: blackbox-exporter
LAST DEPLOYED: Tue Apr 4 10:55:47 2023
NAMESPACE: kube-monitor
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
See https://github.com/prometheus/blackbox_exporter/ for how to configure Prometheus and the Blackbox Exporter.
1. Get the application URL by running these commands:
export POD_NAME=$(kubectl get pods --namespace kube-monitor -l "app.kubernetes.io/name=prometheus-blackbox-exporter,app.kubernetes.io/instance=blackbox-exporter" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace kube-monitor $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace kube-monitor port-forward $POD_NAME 8080:$CONTAINER_PORT
2. 添加监控module
使用helm包自带的即可，如果有需要，可以添加。
3. 添加测试服务
FROM registry.knowdee.com/library/ubuntu-git:1.0.0 as gitLayer
ARG GIT_REPO
ARG GIT_TAG
ARG GIT_USER
ARG GIT_TOKEN
WORKDIR /data
RUN git-clone -u $GIT_USER -p $GIT_TOKEN -a $GIT_REPO && git checkout $GIT_TAG
FROM registry.knowdee.com/library/maven:3.8.6-jdk-11 as mavenLayer
ARG PROJECT_NAME
COPY --from=gitLayer /data /data
WORKDIR /data
RUN mvn clean package && ls /data/$PROJECT_NAME/target/*.jar
FROM registry.knowdee.com/library/jmx-exporter:0.18.0-jdk11
ARG PROJECT_NAME
WORKDIR /data
VOLUME /workspace
COPY --from=mavenLayer /data/$PROJECT_NAME/target/*.jar /data/app.jar
EXPOSE 80
#CMD echo "caoyong echo JAVA_OPTS:"$JAVA_OPTS
#docker build --no-cache -t registry.knowdee.com/tests/demo-jmx-exporter:1.0.0 . -f ./dockerfiles/java11.dockerfile \
#--build-arg GIT_REPO=http://gitlab.knowdee.com/test-k8s/demo-with-jmx-mysql.git \
#--build-arg GIT_TAG=dev \
#--build-arg GIT_TOKEN={{token}} \
#--build-arg GIT_USER=caoyong \
#--build-arg PROJECT_NAME=start --progress=plain
#
# docker run --rm --name app -e MX_MEM=128m -p 18080:8080 -p 17070:7070 registry.knowdee.com/tests/app:v1.0.0
编排文件：
apiVersion: v1
kind: Namespace
metadata:
name: tools
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: demo-with-jmx-mysql
type: deploy
name: test-jmx
namespace: tools
spec:
replicas: 1
selector:
matchLabels:
app: demo-with-jmx-mysql
template:
metadata:
labels:
app: demo-with-jmx-mysql
spec:
tolerations:
- effect: NoExecute
key: taint.knowdee.io/apps
operator: Exists
containers:
- image: registry.knowdee.com/tests/demo-jmx-exporter:2.0.0
imagePullPolicy: Always
name: demo-with-jmx-mysql
ports:
- containerPort: 80
name: http
- containerPort: 7070
name: prom-http
readinessProbe:
initialDelaySeconds: 10
periodSeconds: 5
tcpSocket:
port: 80
timeoutSeconds: 3
resources:
limits:
cpu: "2"
memory: 4G
requests:
cpu: 250m
memory: 100Mi
envFrom:
- configMapRef:
name: db-config
---
apiVersion: v1
data:
SERVER_PORT: "80" # 你的aws账号id
DB_HOST: "mysql-headless.mysql123" # 你的ecr所在region
DB_PORT: "3306" # namespace
DB_USER: "knowdee"
DB_PASSWORD: "knowdee666"
DB_NAME: "prometheus_example"
kind: ConfigMap
metadata:
name: db-config
namespace: tools
---
apiVersion: v1
kind: Service
metadata:
name: test-jmx
namespace: tools
labels:
blackbox-exporter: "true"
name: demo-with-jmx-mysql
spec:
ports:
- name: http
port: 80
targetPort: 80
- name: http-metrics
port: 7070
targetPort: 7070
selector:
app: demo-with-jmx-mysql
4. 添加serviceMonitor
添加serviceMonitor，编排文件如下：
[root@xy-5-server14 caoyong]# cat monitoring.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
annotations:
meta.helm.sh/release-name: prometheus-stack
meta.helm.sh/release-namespace: kube-monitor
labels:
release: prometheus
name: blackbox-exporter-service-monitor
namespace: kube-monitor
spec:
endpoints:
- interval: 30s
params:
module:
- http_2xx
path: /probe #默认是: /metrics
port: http
scheme: http
relabelings:
- sourceLabels: [__address__]
targetLabel: __param_target
regex: (.*)
replacement: $1
action: replace
- targetLabel: __address__
replacement: blackbox-prometheus-blackbox-exporter.kube-monitor:9115
- sourceLabels: [__param_target]
targetLabel: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- sourceLabels: [__meta_kubernetes_namespace]
targetLabel: kubernetes_namespace
- sourceLabels: [__meta_kubernetes_service_name]
targetLabel: kubernetes_service_name
namespaceSelector:
any: true #被blackbox-exporter作为target的服务所在的namespace
selector:
matchLabels:
blackbox-exporter: "true" #被blackbox-exporter作为target的服务的label
执行编排文件：
[root@xy-5-server14 caoyong]# kubectl apply -f monitoring.yaml
servicemonitor.monitoring.coreos.com/jmx-exporter-test-blackbox configured
注：要想此servicemonitor生效，一是prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues设为false或servicemonitor的lable中添加release: [releasename]。
templates/prometheus/prometheus.yaml中有相关label选择的定义：
从模版定义中，可以看到serviceMonitorSelectorNilUsesHelmValues设置为true,serviceMonitor必须有release:$relaseName的label才能被Prometheus加载
比如 helm update --install prometheus . -n kube-monitor,则此时的relaseName就是prometheus,serviceMonitor必须添加release: prometheus标签才能被加载
加载prometheus配置：
curl -X POST "http://192.168.5.17:31111/-/reload"
ts=2023-10-24T10:01:17.261Z caller=main.go:1231 level=info msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
ts=2023-10-24T10:01:17.271Z caller=kubernetes.go:329 level=info component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/kube-monitor/prometheus-kube-prometheus-kubelet/1 msg="Using pod service account via in-cluster config"
ts=2023-10-24T10:01:17.271Z caller=kubernetes.go:329 level=info component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/kube-monitor/prometheus-kube-prometheus-prometheus/0 msg="Using pod service account via in-cluster config"
ts=2023-10-24T10:01:17.272Z caller=kubernetes.go:329 level=info component="discovery manager scrape" discovery=kubernetes config=node-exporter-v2 msg="Using pod service account via in-cluster config"
ts=2023-10-24T10:01:17.272Z caller=kubernetes.go:329 level=info component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/kube-monitor/blackbox-exporter-service-monitor/0 msg="Using pod service account via in-cluster config"
ts=2023-10-24T10:01:17.272Z caller=kubernetes.go:329 level=info component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/kube-monitor/prometheus-kube-prometheus-apiserver/0 msg="Using pod service account via in-cluster config"
ts=2023-10-24T10:01:17.273Z caller=kubernetes.go:329 level=info component="discovery manager notify" discovery=kubernetes config=config-0 msg="Using pod service account via in-cluster config"
ts=2023-10-24T10:01:17.344Z caller=main.go:1268 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=83.760598ms db_storage=1.948µs remote_storage=2.095µs web_handler=655ns query_engine=28.735µs scrape=3.845215ms scrape_sd=1.505847ms notify=19.381µs notify_sd=260.99µs rules=71.746011ms tracing=7.235µs
可以在test pod中使用命令curl http://blackbox-exporter-prometheus-blackbox-exporter.kube-monitor:9115/probe?target=10.244.116.225:7070/metrics&&module=http_2xx进行测试，会得到如下请求结果：
bash-4.2# curl http://blackbox-exporter-prometheus-blackbox-exporter.kube-monitor:9115/probe?target=10.244.116.225:7070/metrics&&module=http_2xx
# HELP probe_dns_lookup_time_seconds Returns the time taken for probe dns lookup in seconds
# TYPE probe_dns_lookup_time_seconds gauge
probe_dns_lookup_time_seconds 3.5525e-05
# HELP probe_duration_seconds Returns how long the probe took to complete in seconds
# TYPE probe_duration_seconds gauge
probe_duration_seconds 0.006325804
# HELP probe_failed_due_to_regex Indicates if probe failed due to regex
# TYPE probe_failed_due_to_regex gauge
probe_failed_due_to_regex 0
# HELP probe_http_content_length Length of http content response
# TYPE probe_http_content_length gauge
probe_http_content_length 13036
# HELP probe_http_duration_seconds Duration of http request by phase, summed over all redirects
# TYPE probe_http_duration_seconds gauge
probe_http_duration_seconds{phase="connect"} 0.000505426
probe_http_duration_seconds{phase="processing"} 0.00474002
probe_http_duration_seconds{phase="resolve"} 3.5525e-05
probe_http_duration_seconds{phase="tls"} 0
probe_http_duration_seconds{phase="transfer"} 0.000290313
# HELP probe_http_redirects The number of redirects
# TYPE probe_http_redirects gauge
probe_http_redirects 0
# HELP probe_http_ssl Indicates if SSL was used for the final redirect
# TYPE probe_http_ssl gauge
probe_http_ssl 0
# HELP probe_http_status_code Response HTTP status code
# TYPE probe_http_status_code gauge
probe_http_status_code 200
# HELP probe_http_uncompressed_body_length Length of uncompressed response body
# TYPE probe_http_uncompressed_body_length gauge
probe_http_uncompressed_body_length 13036
# HELP probe_http_version Returns the version of HTTP of the probe response
# TYPE probe_http_version gauge
probe_http_version 1.1
# HELP probe_ip_addr_hash Specifies the hash of IP address. It's useful to detect if the IP address changes.
# TYPE probe_ip_addr_hash gauge
probe_ip_addr_hash 2.33583498e+09
# HELP probe_ip_protocol Specifies whether probe ip protocol is IP4 or IP6
# TYPE probe_ip_protocol gauge
probe_ip_protocol 4
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1
4. 添加告警
监控https服务
证书到期时间
grafana的blackbox监控模板9965
