kubeadm源码部分
最近想学习 k8s 生态的相关源码知识，从kubeadm入手来看看 k8s 的搭建及管理过程，在这里做个记录
这里就是kubeadm的源码位置，可以从这里看到github源码
cobra库
kubeadm源码使用的是cobra库开发的，在Golang中，我们可以使用Cobra很方便的进行命令行工具的开发。可以快速创建应用程序并添加所需的任何命令。
Cobra库文档地址：pkg.go.dev/github.com/…
程序入口
通过上面的链接进入源码库后，可以看到文件 kubeadm.go，这就是kubeadm程序的入口文件：
package main
import (
"k8s.io/kubernetes/cmd/kubeadm/app"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
func main() {
kubeadmutil.CheckErr(app.Run())
}
调用了 app 包中的 Run() 函数，打开app目录后，可以再找到一个 kubeadm.go 文件，并包含目标 Run() 函数：
func Run() error {
klog.InitFlags(nil)
pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
pflag.Set("logtostderr", "true")
// We do not want these flags to show up in --help
// These MarkHidden calls must be after the lines above
pflag.CommandLine.MarkHidden("version")
pflag.CommandLine.MarkHidden("log-flush-frequency")
pflag.CommandLine.MarkHidden("alsologtostderr")
pflag.CommandLine.MarkHidden("log-backtrace-at")
pflag.CommandLine.MarkHidden("log-dir")
pflag.CommandLine.MarkHidden("logtostderr")
pflag.CommandLine.MarkHidden("stderrthreshold")
pflag.CommandLine.MarkHidden("vmodule")
cmd := cmd.NewKubeadmCommand(os.Stdin, os.Stdout, os.Stderr)
return cmd.Execute()
}
可以看到 Run 函数通过 pflag 包定义了一系列命令行解析的事项（相较于flag包，pflag能够支持更加精细的参数类型、更多的参数类型、对标准flag的兼容、更高级的功能等）
核心功能通过对 cmd.NewKubeadmCommand(os.Stdin, os.Stdout, os.Stderr) 的调用来实现，这个函数真正详细定义了对 kubeadm 的命令行交互操作
打开 app/cmd/cmd.go 文件：
// NewKubeadmCommand returns cobra.Command to run kubeadm command
func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command {
var rootfsPath string
cmds := &cobra.Command{
Use: "kubeadm",
Short: "kubeadm: easily bootstrap a secure Kubernetes cluster",
Long: dedent.Dedent(`
┌──────────────────────────────────────────────────────────┐
│ KUBEADM │
│ Easily bootstrap a secure Kubernetes cluster │
│ │
│ Please give us feedback at: │
│ https://github.com/kubernetes/kubeadm/issues │
└──────────────────────────────────────────────────────────┘
Example usage:
Create a two-machine cluster with one control-plane node
(which controls the cluster), and one worker node
(where your workloads, like Pods and Deployments run).
┌──────────────────────────────────────────────────────────┐
│ On the first machine: │
├──────────────────────────────────────────────────────────┤
│ control-plane# kubeadm init │
└──────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────┐
│ On the second machine: │
├──────────────────────────────────────────────────────────┤
│ worker# kubeadm join <arguments-returned-from-init> │
└──────────────────────────────────────────────────────────┘
You can then repeat the second step on as many other machines as you like.
`),
SilenceErrors: true,
SilenceUsage: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if rootfsPath != "" {
if err := kubeadmutil.Chroot(rootfsPath); err != nil {
return err
}
}
return nil
},
}
cmds.ResetFlags()
cmds.AddCommand(newCmdCertsUtility(out))
cmds.AddCommand(newCmdCompletion(out, ""))
cmds.AddCommand(newCmdConfig(out))
cmds.AddCommand(newCmdInit(out, nil))
cmds.AddCommand(newCmdJoin(out, nil))
cmds.AddCommand(newCmdReset(in, out, nil))
cmds.AddCommand(newCmdVersion(out))
cmds.AddCommand(newCmdToken(out, err))
cmds.AddCommand(upgrade.NewCmdUpgrade(out))
cmds.AddCommand(alpha.NewCmdAlpha())
options.AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath)
cmds.AddCommand(newCmdKubeConfigUtility(out))
return cmds
}
如果有安装kubeadm工具，可以在屏幕输出kubeadm的命令
从代码中不难看出，kubeadm通过 AddCommand() 函数添加了 Init、Join、Reset等常用功能，将所有子命令都附加到了父命令 kubeadm 中
Reset
进入 newCmdReset() 函数，可以看到 reset 命令是如何传进去的，以及他的命令子集：
// newCmdReset returns the "kubeadm reset" command
func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command {
if resetOptions == nil {
resetOptions = newResetOptions()
}
resetRunner := workflow.NewRunner()
cmd := &cobra.Command{
Use: "reset",
Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'",
RunE: func(cmd *cobra.Command, args []string) error {
err := resetRunner.Run(args)
if err != nil {
return err
}
// output help text instructing user how to remove cni folders
fmt.Print(cniCleanupInstructions)
// Output help text instructing user how to remove iptables rules
fmt.Print(iptablesCleanupInstructions)
return nil
},
}
AddResetFlags(cmd.Flags(), resetOptions)
// initialize the workflow runner with the list of phases
resetRunner.AppendPhase(phases.NewPreflightPhase())
resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase())
resetRunner.AppendPhase(phases.NewCleanupNodePhase())
// sets the data builder function, that will be used by the runner
// both when running the entire workflow or single phases
resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
if cmd.Flags().Lookup(options.NodeCRISocket) == nil {
// avoid CRI detection
// assume that the command execution does not depend on CRISocket when --cri-socket flag is not set
resetOptions.criSocketPath = kubeadmconstants.UnknownCRISocket
}
return newResetData(cmd, resetOptions, in, out)
})
// binds the Runner to kubeadm reset command by altering
// command help, adding --skip-phases flag and by adding phases subcommands
resetRunner.BindToCommand(cmd)
return cmd
}
我们在集群内执行reset命令可以看到：
首先会展示 reset 命令的 short 短描述，并通过 resetRunner.AppendPhase() 函数绑定了三个阶段的功能函数执行：
preflight Run reset pre-flight checks （对应preflight运行reset前的检查）
remove-etcd-member Remove a local etcd member. （删除本地etcd成员）
cleanup-node Run cleanup node. （执行cleanup node命令）
// AppendPhase adds the given phase to the ordered sequence of phases managed by the runner.
func (e *Runner) AppendPhase(t Phase) {
e.Phases = append(e.Phases, t)
}
Runner.Phases 就是结构体中的一个数组，来保存 Phases
phases.NewPreflightPhase()
执行结果：
[reset] Reading configuration from the cluster...
[reset] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[reset] WARNING: Changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted
[reset] Are you sure you want to proceed? [y/N]: y
[preflight] Running pre-flight checks
通过 NewPreflightPhase() 源码可以看出，这里的代码只是做了交互式的判断行为：
// NewPreflightPhase creates a kubeadm workflow phase implements preflight checks for reset
func NewPreflightPhase() workflow.Phase {
return workflow.Phase{
Name: "preflight",
Aliases: []string{"pre-flight"},
Short: "Run reset pre-flight checks",
Long: "Run pre-flight checks for kubeadm reset.",
Run: runPreflight,
InheritFlags: []string{
options.IgnorePreflightErrors,
options.ForceReset,
options.DryRun,
},
}
}
// runPreflight executes preflight checks logic.
func runPreflight(c workflow.RunData) error {
r, ok := c.(resetData)
if !ok {
return errors.New("preflight phase invoked with an invalid data struct")
}
if !r.ForceReset() && !r.DryRun() {
klog.Warning("[reset] WARNING: Changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.")
if err := util.InteractivelyConfirmAction("reset", "Are you sure you want to proceed?", r.InputReader()); err != nil {
return err
}
}
fmt.Println("[preflight] Running pre-flight checks")
return preflight.RunRootCheckOnly(r.IgnorePreflightErrors())
}
创建一个kubeadm工作流阶段，实现预检检查以进行重置，这里没有做检查操作，只做了一个执行预检检查逻辑，如果不小心输入，提供用户交互式的操作以便确认是否进行 reset kubeadm 节点
其中 (c workflow.RunData) 断言为 resetData 类型，通过 r.InputReader() 获取到用户的输入结果，通过封装的函数 util.InteractivelyConfirmAction() 来判断用户的行为：
// InteractivelyConfirmAction asks the user whether they _really_ want to take the action.
func InteractivelyConfirmAction(action, question string, r io.Reader) error {
fmt.Printf("[%s] %s [y/N]: ", action, question)
scanner := bufio.NewScanner(r)
scanner.Scan()
if err := scanner.Err(); err != nil {
return errors.Wrap(err, "couldn't read from standard input")
}
answer := scanner.Text()
if strings.EqualFold(answer, "y") || strings.EqualFold(answer, "yes") {
return nil
}
return errors.New("won't proceed; the user didn't answer (Y|y) in order to continue")
}
最后调用 preflight.RunRootCheckOnly() 函数进行预检查
phases.NewRemoveETCDMemberPhase()
获取数据目录的位置，将本地的etcd数据目录进行删除，核心代码：
func runRemoveETCDMemberPhase(c workflow.RunData) error {
r, ok := c.(resetData)
if !ok {
return errors.New("remove-etcd-member-phase phase invoked with an invalid data struct")
}
cfg := r.Cfg()
// Only clear etcd data when using local etcd.
klog.V(1).Infoln("[reset] Checking for etcd config")
etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
etcdDataDir, err := getEtcdDataDir(etcdManifestPath, cfg)
...
CleanDir(etcdDataDir)
...
return nil
}
func getEtcdDataDir(manifestPath string, cfg *kubeadmapi.InitConfiguration) (string, error) {
const etcdVolumeName = "etcd-data"
var dataDir string
if cfg != nil && cfg.Etcd.Local != nil {
return cfg.Etcd.Local.DataDir, nil
}
klog.Warningln("[reset] No kubeadm config, using etcd pod spec to get data directory")
if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
// Fall back to use the default cluster config if etcd.yaml doesn't exist, this could happen that
// etcd.yaml is removed by other reset phases, e.g. cleanup-node.
cfg := &v1beta3.ClusterConfiguration{}
scheme.Scheme.Default(cfg)
return cfg.Etcd.Local.DataDir, nil
}
etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath)
if err != nil {
return "", err
}
for _, volumeMount := range etcdPod.Spec.Volumes {
if volumeMount.Name == etcdVolumeName {
dataDir = volumeMount.HostPath.Path
break
}
}
if dataDir == "" {
return dataDir, errors.New("invalid etcd pod manifest")
}
return dataDir, nil
}
通过代码可以看出，根据路径 /etc/kubernetes/manifests/etcd.yaml 的配置文件来删除本地数据
phases.NewCleanupNodePhase()
用来清理节点，比如 stop kubelet，应用了 GetInitSystem() 接口，进行一系列操作，包括删除一些创建的目录等，核心代码：
func runCleanupNode(c workflow.RunData) error {
dirsToClean := []string{filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}
r, ok := c.(resetData)
if !ok {
return errors.New("cleanup-node phase invoked with an invalid data struct")
}
certsDir := r.CertificatesDir()
// Try to stop the kubelet service
klog.V(1).Infoln("[reset] Getting init system")
initSystem, err := initsystem.GetInitSystem()
...
initSystem.ServiceStop("kubelet")
...
// Remove contents from the config and pki directories
if certsDir != kubeadmapiv1.DefaultCertificatesDir {
klog.Warningf("[reset] WARNING: Cleaning a non-default certificates directory: %q\n", certsDir)
}
dirsToClean = append(dirsToClean, certsDir)
if r.CleanupTmpDir() {
tempDir := path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)
dirsToClean = append(dirsToClean, tempDir)
}
resetConfigDir(kubeadmconstants.KubernetesDir, dirsToClean, r.DryRun())
if r.Cfg() != nil && features.Enabled(r.Cfg().FeatureGates, features.RootlessControlPlane) {
if !r.DryRun() {
klog.V(1).Infoln("[reset] Removing users and groups created for rootless control-plane")
if err := users.RemoveUsersAndGroups(); err != nil {
klog.Warningf("[reset] Failed to remove users and groups: %v\n", err)
}
} else {
fmt.Println("[reset] Would remove users and groups created for rootless control-plane")
}
}
return nil
}
