1. kubelet介绍

kubelet在k8s中主要承担以下责任：

负责node上pod的创建、更新、监控、删除等全生命周期管理
定时上报node的状态给api-server
kubelet是master和node之间的桥梁，负责接收master api server分配给它的command和worker，通过api-server间接的与etcd集群交互。具体工作如下：
设置容器的环境变量，给容器绑定volume，给容器绑定port、根据指定的pod运行一个单一容器，给指定的pod创建network容器。
同步pod的状态，从cadvisor获取container info、pod info、root info、 machine info。
在容器中运行命令、杀死容器、删除pod的所有容器。

2. kubelet的目录结构

其中option目录主要是包括kubelet使用到的option

kubelet.go是kubelet的main函数入口。

2.1 main函数

kubelet代码主要采用Cobra命令行框架，核心代码如下：

// 初始化命令行
command := app.NewKubeletCommand()
// 执行Execute
if err := command.Execute(); err != nil {

2.2 NewKubeletCommand

NewKubeletCommand基于参数创建一个*cobra.Command 对象，其中核心代码为参数解析部分和Run函数。用于后边的command.Execute()来运行创建的这个command。

2.2.1 初始化参数和配置

初始化参数解析，初始化cleanFlagSet、kubeletFlags、kubeletConfig。

cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)
cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
kubeletFlags := options.NewKubeletFlags()
kubeletConfig, err := options.NewKubeletConfiguration()

2.2.2 打印帮助信息和版本信息

DisableFlagParsing: true,

kubelet开启了DisableFlagParsing参数，没有使用Cobra框架中的默认参数解析，而是自定义参数解析。

if err := cleanFlagSet.Parse(args); err != nil {
   cmd.Usage()
   klog.Fatal(err)
}

// check if there are non-flag arguments in the command line
cmds := cleanFlagSet.Args()
if len(cmds) > 0 {
   cmd.Usage()
   klog.Fatalf("unknown command: %s", cmds[0])
}

进行自定义参数解析，如果输入非法参数，打印使用帮助信息。

// short-circuit on help
help, err := cleanFlagSet.GetBool("help")
if err != nil {
   klog.Fatal(`"help" flag is non-bool, programmer error, please correct`)
}
if help {
   cmd.Help()
   return
}

如果遇到help和version参数，则打印相关内容并退出。

2.2.3 kubelet config

// set feature gates from initial flags-based config
if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
   klog.Fatal(err)
}

// validate the initial KubeletFlags
if err := options.ValidateKubeletFlags(kubeletFlags); err != nil {
   klog.Fatal(err)
}

if kubeletFlags.ContainerRuntime == "remote" && cleanFlagSet.Changed("pod-infra-container-image") {
   klog.Warning("Warning: For remote container runtime, --pod-infra-container-image is ignored in kubelet, which should be set in that remote runtime instead")
}

// load kubelet config file, if provided
if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
   kubeletConfig, err = loadConfigFile(configFile)
   if err != nil {
      klog.Fatal(err)
   }
   // We must enforce flag precedence by re-parsing the command line into the new object.
   // This is necessary to preserve backwards-compatibility across binary upgrades.
   // See issue #56171 for more details.
   if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil {
      klog.Fatal(err)
   }
   // update feature gates based on new config
   if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
      klog.Fatal(err)
   }
}

// We always validate the local configuration (command line + config file).
// This is the default "last-known-good" config for dynamic config, and must always remain valid.
if err := kubeletconfigvalidation.ValidateKubeletConfiguration(kubeletConfig); err != nil {
   klog.Fatal(err)
}

加载并校验kubelet config，包括校验初始化kubeletFlags，并从kubeletFlags中读取出kubelet config的配置文件，根据配置文件获取kubelet config

2.2.4 dynamic kubelet config

// use dynamic kubelet config, if enabled
var kubeletConfigController *dynamickubeletconfig.Controller
if dynamicConfigDir := kubeletFlags.DynamicConfigDir.Value(); len(dynamicConfigDir) > 0 {
   var dynamicKubeletConfig *kubeletconfiginternal.KubeletConfiguration
   dynamicKubeletConfig, kubeletConfigController, err = BootstrapKubeletConfigController(dynamicConfigDir,
      func(kc *kubeletconfiginternal.KubeletConfiguration) error {
         // Here, we enforce flag precedence inside the controller, prior to the controller's validation sequence,
         // so that we get a complete validation at the same point where we can decide to reject dynamic config.
         // This fixes the flag-precedence component of issue #63305.
         // See issue #56171 for general details on flag precedence.
         return kubeletConfigFlagPrecedence(kc, args)
      })
   if err != nil {
      klog.Fatal(err)
   }
   // If we should just use our existing, local config, the controller will return a nil config
   if dynamicKubeletConfig != nil {
      kubeletConfig = dynamicKubeletConfig
      // Note: flag precedence was already enforced in the controller, prior to validation,
      // by our above transform function. Now we simply update feature gates from the new config.
      if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
         klog.Fatal(err)
      }
   }
}

如果开启了使用动态kubelet的配置，则由动态配置文件替换之前的kubelet config文件。

2.2.5 总结

通过对各种特定参数的解析，最终生成kubeletFlags和kubeletConfig两个参数对象，用来构造KubeletServer和其他需求

2.2 KubeletServer

kubeletServer := &options.KubeletServer{
   KubeletFlags:         *kubeletFlags,
   KubeletConfiguration: *kubeletConfig,
}

2.3 kubeletDeps

// use kubeletServer to construct the default KubeletDeps
kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate)
if err != nil {
   klog.Fatal(err)
}

// add the kubelet config controller to kubeletDeps
kubeletDeps.KubeletConfigController = kubeletConfigController

根据KubeletServer创建KubeletDeps，同时增加kubelet config controller到KubeletDeps

2.4 RunDockershim

// start the experimental docker shim, if enabled
if kubeletServer.KubeletFlags.ExperimentalDockershim {
   if err := RunDockershim(&kubeletServer.KubeletFlags, kubeletConfig, stopCh); err != nil {
      klog.Fatal(err)
   }
   return
}

如果开启了docker shim参数，则执行RunDockershim

2.5 运行kubelet

// run the kubelet
klog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration)
if err := Run(kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate, stopCh); err != nil {
   klog.Fatal(err)
}

运行kubelet并且不退出，由Run函数进入后续操作。

2.6 AddFlags

// keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags
kubeletFlags.AddFlags(cleanFlagSet)
options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig)
options.AddGlobalFlags(cleanFlagSet)
cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))

// ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
const usageFmt = "Usage:\n  %s\n\nFlags:\n%s"
cmd.SetUsageFunc(func(cmd *cobra.Command) error {
   fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
   return nil
})
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
   fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
})

将cleanFlagSet分开，这样Cobra就不会用全局标志污染它。

重新设置SetUsageFunc 和 SetHelpFunc 因为Cobra的默认UsageFunc和HelpFunc使用全局标志污染了标志集。

k8s kubelet源码分析（一）