源码仓库地址
https://github.com/kubernetes/kubernetes/tree/master/cmd/kubeadm
kubeadm reset命令
帮助提示
我们可以根据帮助提示来对应地查找代码中相应的位置
[root@node1 ~]# kubeadm reset -h
Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'
The "reset" command executes the following phases:
preflight Run reset pre-flight checks remove-etcd-member Remove a local etcd member. cleanup-node Run cleanup node.
Usage:
kubeadm reset [flags]
kubeadm reset [command]
Available Commands:
phase Use this command to invoke single phase of the reset workflow
Flags:
--cert-dir string The path to the directory where the certificates are stored. If specified, clean this directory. (default "/etc/kubernetes/pki")
--cleanup-tmp-dir Cleanup the "/etc/kubernetes/tmp" directory
--config string Path to a kubeadm configuration file.
--cri-socket string Path to the CRI socket to connect. If empty kubeadm will try to auto-detect this value; use this option only if you have more than one CRI installed or if you have non-standard CRI socket.
--dry-run Don't apply any changes; just output what would be done.
-f, --force Reset the node without prompting for confirmation.
-h, --help help for reset
--ignore-preflight-errors strings A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.
--kubeconfig string The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations can be searched for an existing kubeconfig file. (default "/etc/kubernetes/admin.conf")
--skip-phases strings List of phases to be skipped
Global Flags:
--add-dir-header If true, adds the file directory to the header of the log messages
--log-file string If non-empty, use this log file (no effect when -logtostderr=true)
--log-file-max-size uint Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
--one-output If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)
--rootfs string [EXPERIMENTAL] The path to the 'real' host root filesystem.
--skip-headers If true, avoid header prefixes in the log messages
--skip-log-headers If true, avoid headers when opening log files (no effect when -logtostderr=true)
-v, --v Level number for the log level verbosity
Use "kubeadm reset [command] --help" for more information about a command.
reset源代码
// newCmdReset returns the "kubeadm reset" command
func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command {
if resetOptions == nil {
resetOptions = newResetOptions()
}
resetRunner := workflow.NewRunner()
cmd := &cobra.Command{
Use: "reset", // 代表传人的命令，这里也就是reset, 传入卸载集群的命令
Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'", // 命令的简短描述
RunE: func(cmd *cobra.Command, args []string) error {
data, err := resetRunner.InitData(args)
if err != nil {
return err
}
if _, ok := data.(*resetData); !ok {
return errors.New("invalid data struct")
}
if err := resetRunner.Run(args); err != nil {
return err
}
// output help text instructing user how to remove cni folders
fmt.Print(cniCleanupInstructions)
// Output help text instructing user how to remove iptables rules
fmt.Print(iptablesCleanupInstructions)
return nil
},
}
AddResetFlags(cmd.Flags(), resetOptions)
// initialize the workflow runner with the list of phases
resetRunner.AppendPhase(phases.NewPreflightPhase()) // 卸载时候检查
resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase())
resetRunner.AppendPhase(phases.NewCleanupNodePhase())
// sets the data builder function, that will be used by the runner
// both when running the entire workflow or single phases
resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
if cmd.Flags().Lookup(options.NodeCRISocket) == nil {
// skip CRI detection
// assume that the command execution does not depend on CRISocket when --cri-socket flag is not set
resetOptions.skipCRIDetect = true
}
data, err := newResetData(cmd, resetOptions, in, out, true)
if err != nil {
return nil, err
}
// If the flag for skipping phases was empty, use the values from config
if len(resetRunner.Options.SkipPhases) == 0 {
resetRunner.Options.SkipPhases = data.resetCfg.SkipPhases
}
return data, nil
})
// binds the Runner to kubeadm reset command by altering
// command help, adding --skip-phases flag and by adding phases subcommands
resetRunner.BindToCommand(cmd)
return cmd
}
phases.NewPreflightPhase()
该阶段只是做了交互式的判断行为，NewPreflightPhase创建一个Kubeadm工作流阶段，实现预检查以进行重置，当执行kubeadm reset时候先执行该阶段，对应的源码如下
// NewPreflightPhase creates a kubeadm workflow phase implements preflight checks for reset
func NewPreflightPhase() workflow.Phase {
return workflow.Phase{
Name: "preflight",
Aliases: []string{"pre-flight"},
Short: "Run reset pre-flight checks",
Long: "Run pre-flight checks for kubeadm reset.",
Run: runPreflight,
InheritFlags: []string{
options.IgnorePreflightErrors,
options.ForceReset,
options.DryRun,
},
}
}
// runPreflight executes preflight checks logic.
func runPreflight(c workflow.RunData) error {
r, ok := c.(resetData)
if !ok {
return errors.New("preflight phase invoked with an invalid data struct")
}
if !r.ForceReset() && !r.DryRun() {
klog.Warning("[reset] WARNING: Changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.")
if err := util.InteractivelyConfirmAction("reset", "Are you sure you want to proceed?", r.InputReader()); err != nil {
return err
}
}
fmt.Println("[preflight] Running pre-flight checks")
return preflight.RunRootCheckOnly(r.IgnorePreflightErrors())
}
phases.NewRemoveETCDMemberPhase()
此步骤会将本地的etcd的数据目录进行删除
// NewRemoveETCDMemberPhase creates a kubeadm workflow phase for remove-etcd-member
func NewRemoveETCDMemberPhase() workflow.Phase {
return workflow.Phase{
Name: "remove-etcd-member",
Short: "Remove a local etcd member.",
Long: "Remove a local etcd member for a control plane node.",
Run: runRemoveETCDMemberPhase,
InheritFlags: []string{
options.KubeconfigPath,
options.DryRun,
},
}
}
func runRemoveETCDMemberPhase(c workflow.RunData) error {
r, ok := c.(resetData)
if !ok {
return errors.New("remove-etcd-member-phase phase invoked with an invalid data struct")
}
cfg := r.Cfg()
// Only clear etcd data when using local etcd.
// 仅在使用本地etcd时清除etcd数据。
klog.V(1).Infoln("[reset] Checking for etcd config")
etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
etcdDataDir, err := getEtcdDataDir(etcdManifestPath, cfg)
if err == nil {
if cfg != nil {
if !r.DryRun() {
err := etcdphase.RemoveStackedEtcdMemberFromCluster(r.Client(), cfg)
if err != nil {
klog.Warningf("[reset] Failed to remove etcd member: %v, please manually remove this etcd member using etcdctl", err)
} else {
if err := CleanDir(etcdDataDir); err != nil {
klog.Warningf("[reset] Failed to delete contents of the etcd directory: %q, error: %v", etcdDataDir, err)
} else {
fmt.Printf("[reset] Deleted contents of the etcd data directory: %v\n", etcdDataDir)
}
}
} else {
fmt.Println("[reset] Would remove the etcd member on this node from the etcd cluster")
fmt.Printf("[reset] Would delete contents of the etcd data directory: %v\n", etcdDataDir)
}
}
// This could happen if the phase `cleanup-node` is run before the `remove-etcd-member`.
// Cleanup the data in the etcd data dir to avoid some stale files which might cause the failure to build cluster in the next time.
empty, _ := IsDirEmpty(etcdDataDir)
if !empty && !r.DryRun() {
if err := CleanDir(etcdDataDir); err != nil {
klog.Warningf("[reset] Failed to delete contents of the etcd directory: %q, error: %v", etcdDataDir, err)
} else {
fmt.Printf("[reset] Deleted contents of the etcd data directory: %v\n", etcdDataDir)
}
}
} else {
fmt.Println("[reset] No etcd config found. Assuming external etcd")
fmt.Println("[reset] Please, manually reset etcd to prevent further issues")
}
return nil
}
func getEtcdDataDir(manifestPath string, cfg *kubeadmapi.InitConfiguration) (string, error) {
const etcdVolumeName = "etcd-data"
var dataDir string
if cfg != nil && cfg.Etcd.Local != nil {
return cfg.Etcd.Local.DataDir, nil
}
klog.Warningln("[reset] No kubeadm config, using etcd pod spec to get data directory")
if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
// Fall back to use the default cluster config if etcd.yaml doesn't exist, this could happen that
// etcd.yaml is removed by other reset phases, e.g. cleanup-node.
cfg := &v1beta3.ClusterConfiguration{}
scheme.Scheme.Default(cfg)
return cfg.Etcd.Local.DataDir, nil
}
etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath)
if err != nil {
return "", err
}
for _, volumeMount := range etcdPod.Spec.Volumes {
if volumeMount.Name == etcdVolumeName {
dataDir = volumeMount.HostPath.Path
break
}
}
if dataDir == "" {
return dataDir, errors.New("invalid etcd pod manifest")
}
return dataDir, nil
}
phases.NewCleanupNodePhase()
此步骤用来清理节点，比如停止kubelet
// NewCleanupNodePhase creates a kubeadm workflow phase that cleanup the node
func NewCleanupNodePhase() workflow.Phase {
return workflow.Phase{
Name: "cleanup-node",
Aliases: []string{"cleanupnode"},
Short: "Run cleanup node.",
Run: runCleanupNode,
InheritFlags: []string{
options.CertificatesDir,
options.NodeCRISocket,
options.CleanupTmpDir,
options.DryRun,
},
}
}
func runCleanupNode(c workflow.RunData) error {
dirsToClean := []string{filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}
r, ok := c.(resetData)
if !ok {
return errors.New("cleanup-node phase invoked with an invalid data struct")
}
certsDir := r.CertificatesDir()
// Try to stop the kubelet service
klog.V(1).Infoln("[reset] Getting init system")
initSystem, err := initsystem.GetInitSystem()
if err != nil {
klog.Warningln("[reset] The kubelet service could not be stopped by kubeadm. Unable to detect a supported init system!")
klog.Warningln("[reset] Please ensure kubelet is stopped manually")
} else {
if !r.DryRun() {
fmt.Println("[reset] Stopping the kubelet service")
// 调用了initSystem来停止kubelet
if err := initSystem.ServiceStop("kubelet"); err != nil {
klog.Warningf("[reset] The kubelet service could not be stopped by kubeadm: [%v]\n", err)
klog.Warningln("[reset] Please ensure kubelet is stopped manually")
}
} else {
fmt.Println("[reset] Would stop the kubelet service")
}
}
if !r.DryRun() {
// In case KubeletRunDirectory holds a symbolic link, evaluate it.
// This would also throw an error if the directory does not exist.
kubeletRunDirectory, err := filepath.EvalSymlinks(kubeadmconstants.KubeletRunDirectory)
if err != nil {
klog.Warningf("[reset] Skipping unmount of directories in %q: %v\n",
kubeadmconstants.KubeletRunDirectory, err)
} else {
// Unmount all mount paths under kubeletRunDirectory.
fmt.Printf("[reset] Unmounting mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
if err := unmountKubeletDirectory(kubeletRunDirectory, r.ResetCfg().UnmountFlags); err != nil {
return err
}
// Clean the kubeletRunDirectory.
dirsToClean = append(dirsToClean, kubeletRunDirectory)
}
} else {
fmt.Printf("[reset] Would unmount mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
}
if !r.DryRun() {
klog.V(1).Info("[reset] Removing Kubernetes-managed containers")
if err := removeContainers(utilsexec.New(), r.CRISocketPath()); err != nil {
klog.Warningf("[reset] Failed to remove containers: %v\n", err)
}
} else {
fmt.Println("[reset] Would remove Kubernetes-managed containers")
}
// Remove contents from the config and pki directories
if certsDir != kubeadmapiv1.DefaultCertificatesDir {
klog.Warningf("[reset] WARNING: Cleaning a non-default certificates directory: %q\n", certsDir)
}
dirsToClean = append(dirsToClean, certsDir)
if r.CleanupTmpDir() {
tempDir := path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)
dirsToClean = append(dirsToClean, tempDir)
}
resetConfigDir(kubeadmconstants.KubernetesDir, dirsToClean, r.DryRun())
if r.Cfg() != nil && features.Enabled(r.Cfg().FeatureGates, features.RootlessControlPlane) {
if !r.DryRun() {
klog.V(1).Infoln("[reset] Removing users and groups created for rootless control-plane")
if err := users.RemoveUsersAndGroups(); err != nil {
klog.Warningf("[reset] Failed to remove users and groups: %v\n", err)
}
} else {
fmt.Println("[reset] Would remove users and groups created for rootless control-plane")
}
}
return nil
}
func removeContainers(execer utilsexec.Interface, criSocketPath string) error {
containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath)
if err != nil {
return err
}
containers, err := containerRuntime.ListKubeContainers()
if err != nil {
return err
}
return containerRuntime.RemoveContainers(containers)
}
// resetConfigDir is used to cleanup the files in the folder defined in dirsToClean.
func resetConfigDir(configPathDir string, dirsToClean []string, isDryRun bool) {
if !isDryRun {
fmt.Printf("[reset] Deleting contents of directories: %v\n", dirsToClean)
for _, dir := range dirsToClean {
if err := CleanDir(dir); err != nil {
klog.Warningf("[reset] Failed to delete contents of %q directory: %v", dir, err)
}
}
} else {
fmt.Printf("[reset] Would delete contents of directories: %v\n", dirsToClean)
}
filesToClean := []string{
filepath.Join(configPathDir, kubeadmconstants.AdminKubeConfigFileName),
filepath.Join(configPathDir, kubeadmconstants.SuperAdminKubeConfigFileName),
filepath.Join(configPathDir, kubeadmconstants.KubeletKubeConfigFileName),
filepath.Join(configPathDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName),
filepath.Join(configPathDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
filepath.Join(configPathDir, kubeadmconstants.SchedulerKubeConfigFileName),
}
if !isDryRun {
fmt.Printf("[reset] Deleting files: %v\n", filesToClean)
for _, path := range filesToClean {
if err := os.RemoveAll(path); err != nil {
klog.Warningf("[reset] Failed to remove file: %q [%v]\n", path, err)
}
}
} else {
fmt.Printf("[reset] Would delete files: %v\n", filesToClean)
}
}
// CleanDir removes everything in a directory, but not the directory itself
func CleanDir(filePath string) error {
// If the directory doesn't even exist there's nothing to do, and we do
// not consider this an error
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return nil
}
d, err := os.Open(filePath)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
if err = os.RemoveAll(filepath.Join(filePath, name)); err != nil {
return err
}
}
return nil
}
// IsDirEmpty returns true if a directory is empty
func IsDirEmpty(dir string) (bool, error) {
d, err := os.Open(dir)
if err != nil {
return false, err
}
defer d.Close()
_, err = d.Readdirnames(1)
if err == io.EOF {
return true, nil
}
return false, nil
}
