1.11.1 版本 出现健康检查失败，导致 kubectl kill掉 该 pod 并触发重启，修改 liveness 时间变长无效。 "Killing container with a grace period override"
Jul 18 10:32:04 eis069 kubelet[362845]: I0718 10:32:04.870605 362845 kuberuntime_manager.go:683] "Message for Container of pod" containerName="ovn-central" containerStatusID={Type:containerd ID:c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2} pod="kube-system/ovn-central-8d7c86969-tw6t4" containerMessage="Container ovn-central failed liveness probe, will be restarted"
Jul 18 10:32:04 eis069 kubelet[362845]: I0718 10:32:04.870739 362845 kuberuntime_container.go:661] "Killing container with a grace period override" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" containerID="containerd://c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" gracePeriod=30
Jul 18 10:32:05 eis069 kubelet[362845]: I0718 10:32:05.283102 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:28:06 eis069 kubelet[362845]: I0718 10:28:06.868850 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:28:06 eis069 kubelet[362845]: I0718 10:28:06.876974 362845 prober.go:116] "Probe failed" probeType="Liveness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:28:52 eis069 kubelet[362845]: I0718 10:28:52.350109 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
Jul 18 10:29:05 eis069 kubelet[362845]: I0718 10:29:05.589487 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:29:05 eis069 kubelet[362845]: I0718 10:29:05.595218 362845 prober.go:116] "Probe failed" probeType="Liveness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:29:58 eis069 kubelet[362845]: I0718 10:29:58.898416 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
Jul 18 10:30:07 eis069 kubelet[362845]: I0718 10:30:07.159015 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:30:07 eis069 kubelet[362845]: I0718 10:30:07.159032 362845 prober.go:116] "Probe failed" probeType="Liveness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:30:07 eis069 kubelet[362845]: I0718 10:30:07.160730 362845 kubelet.go:2026] "SyncLoop (probe)" probe="readiness" status="" pod="kube-system/ovn-central-8d7c86969-tw6t4"
Jul 18 10:30:22 eis069 kubelet[362845]: I0718 10:30:22.292887 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
Jul 18 10:31:04 eis069 kubelet[362845]: I0718 10:31:04.778662 362845 prober.go:116] "Probe failed" probeType="Liveness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:31:05 eis069 kubelet[362845]: I0718 10:31:05.134806 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:31:19 eis069 kubelet[362845]: I0718 10:31:19.489549 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
Jul 18 10:31:48 eis069 kubelet[362845]: I0718 10:31:48.004876 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
Jul 18 10:32:04 eis069 kubelet[362845]: I0718 10:32:04.844482 362845 prober.go:116] "Probe failed" probeType="Liveness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:32:04 eis069 kubelet[362845]: I0718 10:32:04.854604 362845 kubelet.go:2026] "SyncLoop (probe)" probe="liveness" status="unhealthy" pod="kube-system/ovn-central-8d7c86969-tw6t4"
Jul 18 10:32:04 eis069 kubelet[362845]: I0718 10:32:04.870605 362845 kuberuntime_manager.go:683] "Message for Container of pod" containerName="ovn-central" containerStatusID={Type:containerd ID:c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2} pod="kube-system/ovn-central-8d7c86969-tw6t4" containerMessage="Container ovn-central failed liveness probe, will be restarted"
Jul 18 10:32:04 eis069 kubelet[362845]: I0718 10:32:04.870739 362845 kuberuntime_container.go:661] "Killing container with a grace period override" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" containerID="containerd://c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" gracePeriod=30
Jul 18 10:32:05 eis069 kubelet[362845]: I0718 10:32:05.283102 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:32:12 eis069 kubelet[362845]: I0718 10:32:12.356526 362845 kubelet.go:1970] "SyncLoop (PLEG): event for pod" pod="kube-system/ovn-central-8d7c86969-tw6t4" event=&{ID:3821daa9-69fd-435a-a45a-d49bafd5091b Type:ContainerDied Data:c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2}
Jul 18 10:32:12 eis069 kubelet[362845]: E0718 10:32:12.359438 362845 remote_runtime.go:394] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" cmd=[bash /kube-ovn/ovn-healthcheck.sh]
Jul 18 10:32:12 eis069 kubelet[362845]: E0718 10:32:12.361466 362845 remote_runtime.go:394] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" cmd=[bash /kube-ovn/ovn-healthcheck.sh]
Jul 18 10:32:12 eis069 kubelet[362845]: E0718 10:32:12.363616 362845 remote_runtime.go:394] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" cmd=[bash /kube-ovn/ovn-healthcheck.sh]
Jul 18 10:32:12 eis069 kubelet[362845]: E0718 10:32:12.364470 362845 prober.go:113] "Probe errored" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central"
Jul 18 10:32:13 eis069 kubelet[362845]: E0718 10:32:13.407393 362845 remote_runtime.go:394] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" cmd=[bash /kube-ovn/ovn-healthcheck.sh]
Jul 18 10:32:13 eis069 kubelet[362845]: E0718 10:32:13.409423 362845 remote_runtime.go:394] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" cmd=[bash /kube-ovn/ovn-healthcheck.sh]
Jul 18 10:32:13 eis069 kubelet[362845]: E0718 10:32:13.411483 362845 remote_runtime.go:394] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="c656817b10a9b210650e84742672bc47db824b33e640c6e687aca6c7838781b2" cmd=[bash /kube-ovn/ovn-healthcheck.sh]
Jul 18 10:32:13 eis069 kubelet[362845]: E0718 10:32:13.411551 362845 prober.go:113] "Probe errored" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central"
Jul 18 10:32:13 eis069 kubelet[362845]: I0718 10:32:13.446380 362845 kubelet.go:1970] "SyncLoop (PLEG): event for pod" pod="kube-system/ovn-central-8d7c86969-tw6t4" event=&{ID:3821daa9-69fd-435a-a45a-d49bafd5091b Type:ContainerStarted Data:98292217a5750381c6072eeb0ef79bb99f48e878ac951387db5efee13ec3efc1}
Jul 18 10:32:14 eis069 kubelet[362845]: I0718 10:32:14.441877 362845 kubelet.go:2026] "SyncLoop (probe)" probe="readiness" status="" pod="kube-system/ovn-central-8d7c86969-tw6t4"
Jul 18 10:32:14 eis069 kubelet[362845]: I0718 10:32:14.920449 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is not running\n"
Jul 18 10:32:42 eis069 kubelet[362845]: I0718 10:32:42.993201 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
Jul 18 10:33:04 eis069 kubelet[362845]: I0718 10:33:04.755074 362845 prober.go:116] "Probe failed" probeType="Readiness" pod="kube-system/ovn-central-8d7c86969-tw6t4" podUID=3821daa9-69fd-435a-a45a-d49bafd5091b containerName="ovn-central" probeResult=failure output="ovn-northd is running with pid 225\nStatus: cluster member\nStatus: disconnected from the cluster (election timeout)\nStatus: disconnected from the cluster (election timeout)\nRole: candidate\nsb health check failed\n"
Jul 18 10:33:05 eis069 kubelet[362845]: I0718 10:33:05.800810 362845 kubelet.go:1939] "SyncLoop UPDATE" source="api" pods=[kube-system/ovn-central-8d7c86969-tw6t4]
[deployer@eis069 ovn]$
[deployer@eis069 ovn]$
[deployer@eis069 ovn]$ sudo journalctl -lu kubelet | grep ovn
