TIM项目介绍
TIM是一个人开发的基于springboot的即时通信项目，目前支持单聊，群聊和获取个人用户列表功能，用来作为部署k8s的实践项目比较合适,下面以tim-server为例子说明如何部署springboot项目到k8s.tim-gateway类似
项目组成
TIM由tim-gateway 和 tim-server组成，tim-gateway支持客户注册，获取tim-server连接的功能，tim-server负责消息转发功能，并与tim-client(模拟im客户端)建立netty tcp长连接
k8s节点
三个虚拟机组成k8s集群1master-2slave.
root@master:/home/guanwu/k8s/configmap/tim# k get nodes -owide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master Ready control-plane 18d v1.28.2 192.168.193.131 <none> Ubuntu 22.04.2 LTS 6.5.0-18-generic containerd://1.6.28
worker1 Ready <none> 18d v1.28.2 192.168.193.130 <none> Ubuntu 22.04.2 LTS 6.5.0-18-generic containerd://1.6.24
worker2 Ready <none> 18d v1.28.2 192.168.193.132 <none> Ubuntu 22.04.2 LTS 6.5.0-15-generic containerd://1.6.24
root@master:/home/guanwu/k8s/configmap/tim#
项目镜像准备和导入
- 上传jar包到服务器
root@master:/home/guanwu/k8s/tim/tim-server# ls
Dockerfile tim-server-1.0.0-SNAPSHOT.jar tim-server.tar
root@master:/home/guanwu/k8s/tim/tim-server#
- 创建tim-server镜像
root@master:/home/guanwu/k8s/tim/tim-server# cat Dockerfile
from java:8
ADD tim-server-1.0.0-SNAPSHOT.jar /tim-server.jar
EXPOSE 8083
EXPOSE 9003
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/tim-server.jar"]
命令
root@master:/home/guanwu/k8s/tim/tim-server# docker build . -t tim-server:1.1
[+] Building 0.2s (7/7) FINISHED docker:default
=> [internal] load build definition from Dockerfile 0.0s
=> => transferring dockerfile: 213B 0.0s
=> [internal] load metadata for docker.io/library/java:8 0.0s
=> [internal] load .dockerignore 0.0s
=> => transferring context: 2B 0.0s
=> [internal] load build context 0.0s
=> => transferring context: 53B 0.0s
=> [1/2] FROM docker.io/library/java:8 0.0s
=> CACHED [2/2] ADD tim-server-1.0.0-SNAPSHOT.jar /tim-server.jar 0.0s
=> exporting to image 0.0s
=> => exporting layers 0.0s
=> => writing image sha256:f6dcd8c2882dce58c2764b6462fe286851232b4fd516eb14a73611532d807f4e 0.0s
=> => naming to docker.io/library/tim-server:1.1 0.0s
root@master:/home/guanwu/k8s/tim/tim-server#
可以看到如下效果:
root@master:/home/guanwu/k8s/tim/tim-server# docker images tim-server
REPOSITORY TAG IMAGE ID CREATED SIZE
tim-server 1.0 f6dcd8c2882d 25 hours ago 692MB
tim-server 1.1 f6dcd8c2882d 25 hours ago 692MB
root@master:/home/guanwu/k8s/tim/tim-server#
导出镜像
root@master:/home/guanwu/k8s/tim/tim-server# docker image save tim-server:1.1 -o tim-server-1.1.tar
Containerd导入镜像
root@master:/home/guanwu/k8s/tim/tim-server# ctr -n k8s.io image import tim-server-1.1.tar
unpacking docker.io/library/tim-server:1.1 (sha256:6edab08570a23484d68a4501129dbd37955f48fb4181d422d9ba5f17f59ae693)...done
root@master:/home/guanwu/k8s/tim/tim-server# ctr -n k8s.io images list | grep tim-server:1.1
docker.io/library/tim-server:1.1 application/vnd.oci.image.manifest.v1+json sha256:6edab08570a23484d68a4501129dbd37955f48fb4181d422d9ba5f17f59ae693 674.9 MiB linux/amd64 io.cri-containerd.image=managed
root@master:/home/guanwu/k8s/tim/tim-server#
请注意，导入镜像时，请保证各个节点已经存在tim-server镜像了，可以在各个node节点使用crictl命令查看
root@master:/home/guanwu/k8s/configmap/tim# crictl images | grep tim-server
docker.io/library/tim-server 1.0 f6dcd8c2882dc 708MB
docker.io/library/tim-server 1.1 f6dcd8c2882dc 708MB
root@master:/home/guanwu/k8s/configmap/tim#
部署到K8s
configmap
创建项目的configmap,主要是zookeeper，gateway和mysql以及redis的host
root@master:/home/guanwu/k8s/configmap/tim# cat tim-app-config.yaml
apiVersion: v1
data:
zookeeper_server: "192.168.201.129:2181"
time_gateway_host: "tim-gateway"
mysql_host: "192.168.0.132"
rocketmq_nameserver: "192.168.201.129:9876"
app_log_dir: "/var/log"
redis_host: "192.168.201.129"
spring_profiles_active: "sit"
kind: ConfigMap
metadata:
name: tim-app-config
namespace: tim-sit
root@master:/home/guanwu/k8s/configmap/tim#
使用命令"k get cm tim-app-config -n tim-sit -o yaml" 效果如下:
root@master:/home/guanwu/k8s/configmap/tim# k get cm tim-app-config -n tim-sit -o yaml
apiVersion: v1
data:
app_log_dir: /var/log
mysql_host: 192.168.0.132
redis_host: 192.168.201.129
rocketmq_nameserver: 192.168.201.129:9876
spring_profiles_active: sit
time_gateway_host: tim-gateway
zookeeper_server: 192.168.201.129:2181
kind: ConfigMap
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","data":{"app_log_dir":"/var/log","mysql_host":"192.168.0.132","redis_host":"192.168.201.129","rocketmq_nameserver":"192.168.201.129:9876","spring_profiles_active":"sit","time_gateway_host":"tim-gateway","zookeeper_server":"192.168.201.129:2181"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"tim-app-config","namespace":"tim-sit"}}
creationTimestamp: "2024-02-20T13:06:06Z"
name: tim-app-config
namespace: tim-sit
resourceVersion: "70933"
uid: 06efa733-5b51-4165-b9b1-33af9a7468fe
root@master:/home/guanwu/k8s/configmap/tim#
secret
创建秘钥，避免明文显示Mysql密码
apiVersion: v1
kind: Secret
metadata:
name: tim-app-secret
namespace: tim-sit
type: Opaque
data:
mysql.username: cm9vdA==
mysql.password: MTIzNDU2
deployment
deployment需要引用configmap和secret
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: tim-server
name: tim-server
namespace: tim-sit
spec:
replicas: 3
selector:
matchLabels:
app: tim-server
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: tim-server
spec:
containers:
- image: tim-server:1.0
name: tim-server
resources: {}
env:
- name: ZOOKEEPER_SERVER
valueFrom:
configMapKeyRef:
name: tim-app-config
key: zookeeper_server
- name: TIM_GATEWAY_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: time_gateway_host
- name: MYSQL_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: mysql_host
- name: ROCKETMQ_NAMESERVER
valueFrom:
configMapKeyRef:
name: tim-app-config
key: rocketmq_nameserver
- name: REDIS_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: redis_host
- name: APP_LOG_HOME
valueFrom:
configMapKeyRef:
name: tim-app-config
key: app_log_dir
- name: APP_ACTIVE_PROFILE
valueFrom:
configMapKeyRef:
name: tim-app-config
key: spring_profiles_active
- name: MYSQL_USER_NAME
valueFrom:
secretKeyRef:
name: tim-app-secret
key: mysql.username
- name: MYSQL_USER_PASSWD
valueFrom:
secretKeyRef:
name: tim-app-secret
key: mysql.password
volumeMounts:
- name: config
mountPath: "/config"
readOnly: true
- name: secret-volume
mountPath: "/secret-volume"
volumes:
- name: config
configMap:
name: tim-app-config
- name: secret-volume
secret:
secretName: tim-app-secret
status: {}
svc
创建tim-server的svc
root@master:/home/guanwu/k8s/configmap/tim# cat tim-server-service.yaml
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
app: tim-server
name: tim-server
namespace: tim-sit
spec:
ports:
- port: 9003
protocol: TCP
targetPort: 9003
name: tcp
- port: 8083
protocol: TCP
targetPort: 8083
name: http
selector:
app: tim-server
status:
loadBalancer: {}
同时apply tim-gateway和tim-server的service,可以看到效果如下:
root@master:/home/guanwu/k8s/configmap/tim# k get pods -n tim-sit
NAME READY STATUS RESTARTS AGE
ngx19-5c9c699cf8-wfxjg 1/1 Running 0 148m
tim-gateway-868dfc668f-bqlc4 1/1 Running 0 148m
tim-gateway-868dfc668f-tzhdw 1/1 Running 0 148m
tim-gateway-868dfc668f-zfjrg 1/1 Running 0 148m
tim-server-c4fd88d7d-fntwn 1/1 Running 0 148m
tim-server-c4fd88d7d-qg85w 1/1 Running 0 148m
tim-server-c4fd88d7d-t5tsh 1/1 Running 0 148m
root@master:/home/guanwu/k8s/configmap/tim#
root@master:/home/guanwu/k8s/configmap/tim# k get svc -n tim-sit
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
tim-gateway ClusterIP 10.20.190.155 <none> 8090/TCP 24h
tim-server ClusterIP 10.20.141.94 <none> 9003/TCP,8083/TCP 24h
root@master:/home/guanwu/k8s/configmap/tim#
效果
访问tim-gateway集群api,请注意，使用clusterIP访问，即10.20.190.155,可以看到访问成功。
root@master:/home/guanwu/k8s/configmap/tim# curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{
"reqNo": "13766668891",
"timeStamp": 0,
"userName": "wangguanwu_2024_02_21"
}' 'http://10.20.190.155:8090/registerAccount' && echo
{"code":"9000","message":"成功","reqNo":null,"dataBody":{"userId":1708523210743,"userName":"wangguanwu_2024_02_21"}}
root@master:/home/guanwu/k8s/configmap/tim#
本地启动tim-client客户端，并使用sit配置
启动效果如下:
查看k8s gateway日志,可以看到gateway转发的消息日志已经打印出来了
root@master:/home/guanwu/rocketmq# k get pods -n tim-sit
NAME READY STATUS RESTARTS AGE
ngx19-5c9c699cf8-wfxjg 1/1 Running 1 (28m ago) 5d
tim-gateway-868dfc668f-lhrjv 1/1 Running 2 (9m44s ago) 10m
tim-gateway-868dfc668f-nzktv 1/1 Running 2 (9m55s ago) 10m
tim-gateway-868dfc668f-skgfs 1/1 Running 2 (9m44s ago) 10m
tim-server-c4fd88d7d-d85w4 1/1 Running 4 (8m37s ago) 10m
tim-server-c4fd88d7d-lr7x8 1/1 Running 4 (8m51s ago) 10m
tim-server-c4fd88d7d-r4xq9 1/1 Running 4 (8m33s ago) 10m
root@master:/home/guanwu/rocketmq# k logs tim-gateway-868dfc668f-nzktv -n tim-sit | tail -n 10
2024-02-26 11:34:38 [ZkClient-EventThread-15-192.168.201.129:2181] INFO [38] com.gw.tim.gateway.kit.ZkUtils - Clear and update local cache parentPath=[/route],currentChildren=[[ip-10.10.2.85:9003:8083, ip-10.10.1.63:9003:8083]]
2024-02-26 11:34:45 [ZkClient-EventThread-15-192.168.201.129:2181] INFO [38] com.gw.tim.gateway.kit.ZkUtils - Clear and update local cache parentPath=[/route],currentChildren=[[ip-10.10.2.85:9003:8083, ip-10.10.1.63:9003:8083, ip-10.10.1.64:9003:8083]]
2024-02-26 11:38:59 [http-nio-8090-exec-1] INFO [173] o.a.c.core.ContainerBase.[Tomcat].[localhost].[/] - Initializing Spring DispatcherServlet 'dispatcherServlet'
2024-02-26 11:38:59 [http-nio-8090-exec-1] INFO [525] org.springframework.web.servlet.DispatcherServlet - Initializing Servlet 'dispatcherServlet'
2024-02-26 11:38:59 [http-nio-8090-exec-1] INFO [547] org.springframework.web.servlet.DispatcherServlet - Completed initialization in 18 ms
2024-02-26 11:38:59 [http-nio-8090-exec-1] INFO [140] com.gw.tim.gateway.controller.RouteController - user start login: {"reqNo":null,"timeStamp":1708947536,"userId":1701357523670,"userName":"zhangsan"}
2024-02-26 11:39:00 [http-nio-8090-exec-1] INFO [147] com.gw.tim.gateway.controller.RouteController - userName=[zhangsan] route server info=[10.10.2.85:9003:8083]
2024-02-26 11:39:37 [http-nio-8090-exec-4] INFO [74] com.gw.tim.gateway.controller.RouteController - msg={"reqNo":"20590573966589952","timeStamp":1708947575,"userId":1701357523670,"msg":"群聊消息","groupId":0,"msgId":20590573966589952}
2024-02-26 11:39:37 [http-nio-8090-exec-4] INFO [39] c.g.t.g.service.impl.MqPushMessageServiceImpl - start push group message to mq:{"reqNo":"20590573966589952","timeStamp":1708947575,"userId":1701357523670,"msg":"群聊消息","groupId":0,"msgId":20590573966589952}
2024-02-26 11:39:37 [http-nio-8090-exec-4] INFO [45] c.g.t.g.service.impl.MqPushMessageServiceImpl - end push group message to mq <====
root@master:/home/guanwu/rocketmq#
完整的文件
项目地址
tim-gateway完整yaml
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
app: tim-gateway
name: tim-gateway
namespace: tim-sit
spec:
ports:
- port: 8090
protocol: TCP
targetPort: 8090
selector:
app: tim-gateway
status:
loadBalancer: {}
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: tim-gateway
name: tim-gateway
namespace: tim-sit
spec:
replicas: 3
selector:
matchLabels:
app: tim-gateway
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: tim-gateway
spec:
containers:
- image: tim-gateway:1.0
name: tim-gateway
resources: {}
env:
- name: ZOOKEEPER_SERVER
valueFrom:
configMapKeyRef:
name: tim-app-config
key: zookeeper_server
- name: TIM_GATEWAY_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: time_gateway_host
- name: MYSQL_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: mysql_host
- name: ROCKETMQ_NAMESERVER
valueFrom:
configMapKeyRef:
name: tim-app-config
key: rocketmq_nameserver
- name: REDIS_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: redis_host
- name: APP_LOG_HOME
valueFrom:
configMapKeyRef:
name: tim-app-config
key: app_log_dir
- name: APP_ACTIVE_PROFILE
valueFrom:
configMapKeyRef:
name: tim-app-config
key: spring_profiles_active
- name: MYSQL_USER_NAME
valueFrom:
secretKeyRef:
name: tim-app-secret
key: mysql.username
- name: MYSQL_USER_PASSWD
valueFrom:
secretKeyRef:
name: tim-app-secret
key: mysql.password
volumeMounts:
- name: config
mountPath: "/config"
readOnly: true
- name: secret-volume
mountPath: "/secret-volume"
volumes:
- name: config
configMap:
name: tim-app-config
- name: secret-volume
secret:
secretName: tim-app-secret
status: {}
---
apiVersion: v1
data:
zookeeper_server: "192.168.201.129:2181"
time_gateway_host: "tim-gateway"
mysql_host: "192.168.0.132"
rocketmq_nameserver: "192.168.201.129:9876"
app_log_dir: "/var/log"
redis_host: "192.168.201.129"
spring_profiles_active: "sit"
kind: ConfigMap
metadata:
name: tim-app-config
namespace: tim-sit
---
apiVersion: v1
kind: Secret
metadata:
name: tim-app-secret
namespace: tim-sit
type: Opaque
data:
mysql.username: cm9vdA==
mysql.password: MTIzNDU2
tim-server完整yaml
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
app: tim-server
name: tim-server
namespace: tim-sit
spec:
ports:
- port: 9003
protocol: TCP
targetPort: 9003
name: tcp
- port: 8083
protocol: TCP
targetPort: 8083
name: http
selector:
app: tim-server
status:
loadBalancer: {}
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: tim-server
name: tim-server
namespace: tim-sit
spec:
replicas: 3
selector:
matchLabels:
app: tim-server
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: tim-server
spec:
containers:
- image: tim-server:1.0
name: tim-server
resources: {}
env:
- name: ZOOKEEPER_SERVER
valueFrom:
configMapKeyRef:
name: tim-app-config
key: zookeeper_server
- name: TIM_GATEWAY_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: time_gateway_host
- name: MYSQL_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: mysql_host
- name: ROCKETMQ_NAMESERVER
valueFrom:
configMapKeyRef:
name: tim-app-config
key: rocketmq_nameserver
- name: REDIS_HOST
valueFrom:
configMapKeyRef:
name: tim-app-config
key: redis_host
- name: APP_LOG_HOME
valueFrom:
configMapKeyRef:
name: tim-app-config
key: app_log_dir
- name: APP_ACTIVE_PROFILE
valueFrom:
configMapKeyRef:
name: tim-app-config
key: spring_profiles_active
- name: MYSQL_USER_NAME
valueFrom:
secretKeyRef:
name: tim-app-secret
key: mysql.username
- name: MYSQL_USER_PASSWD
valueFrom:
secretKeyRef:
name: tim-app-secret
key: mysql.password
volumeMounts:
- name: config
mountPath: "/config"
readOnly: true
- name: secret-volume
mountPath: "/secret-volume"
volumes:
- name: config
configMap:
name: tim-app-config
- name: secret-volume
secret:
secretName: tim-app-secret
status: {}
---
apiVersion: v1
data:
zookeeper_server: "192.168.201.129:2181"
time_gateway_host: "tim-gateway"
mysql_host: "192.168.0.132"
rocketmq_nameserver: "192.168.201.129:9876"
app_log_dir: "/var/log"
redis_host: "192.168.201.129"
spring_profiles_active: "sit"
kind: ConfigMap
metadata:
name: tim-app-config
namespace: tim-sit
---
apiVersion: v1
kind: Secret
metadata:
name: tim-app-secret
namespace: tim-sit
type: Opaque
data:
mysql.username: cm9vdA==
mysql.password: MTIzNDU2
