kubernetes apiserver源码: Naming Controller

Naming Controller 负责控制CRD的命名冲突，是一个相对简单的Controller.

what's Controller ? A control loop that watches the shared state of the cluster through the apiserver and makes changes attempting to move the current state towards the desired state.

CRD样例

apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  # name must match the spec fields below, and be in the form: <plural>.<group>
  name: crontabs.stable.example.com
spec:
  # group name to use for REST API: /apis/<group>/<version>
  group: stable.example.com
  # list of versions supported by this CustomResourceDefinition
  versions:
    - name: v1
      # Each version can be enabled/disabled by Served flag.
      served: true
      # One and only one version must be marked as the storage version.
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              properties:
                cronSpec:
                  type: string
                image:
                  type: string
                replicas:
                  type: integer
  # either Namespaced or Cluster
  scope: Namespaced
  names:
    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
    plural: crontabs
    # singular name to be used as an alias on the CLI and for display
    singular: crontab
    # kind is normally the CamelCased singular type. Your resource manifests use this.
    kind: CronTab
    # shortNames allow shorter string to match your resource on the CLI
    shortNames:
    - ct

CRD结构体定义

// CustomResourceDefinition represents a resource that should be exposed on the API server.  Its name MUST be in the format
// <.spec.name>.<.spec.group>.
type CustomResourceDefinition struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// spec describes how the user wants the resources to appear
	Spec CustomResourceDefinitionSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"`
	// status indicates the actual state of the CustomResourceDefinition
	// +optional
	Status CustomResourceDefinitionStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
}

Naming Controller

Naming Controller的任务是处理命名冲突，简单实现方式就是监听CRD的事件，并检查比如：新创建的CRD是否与已声明的CRD冲突。

还是贴一张这个熟悉的图：

/ This controller is reserving names. To avoid conflicts, be sure to run only one instance of the worker at a time.
// This could eventually be lifted, but starting simple.
type NamingConditionController struct {
        // 通过Key获取CRD
	crdClient client.CustomResourceDefinitionsGetter
    
        // 获取CRD列表
	crdLister listers.CustomResourceDefinitionLister
	crdSynced cache.InformerSynced
	// crdMutationCache backs our lister and keeps track of committed updates to avoid racy
	// write/lookup cycles.  It's got 100 slots by default, so it unlikely to overrun
	// TODO to revisit this if naming conflicts are found to occur in the wild
	crdMutationCache cache.MutationCache

	// 同步方法
	syncFn func(key string) error

        // 图中7->8之间的workqueue
	queue workqueue.RateLimitingInterface
}

func NewNamingConditionController(
	crdInformer informers.CustomResourceDefinitionInformer,
	crdClient client.CustomResourceDefinitionsGetter,
) *NamingConditionController {
	c := &NamingConditionController{
		crdClient: crdClient,
		crdLister: crdInformer.Lister(),
		crdSynced: crdInformer.Informer().HasSynced,
		queue:     workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_naming_condition_controller"),
	}

	informerIndexer := crdInformer.Informer().GetIndexer()
	c.crdMutationCache = cache.NewIntegerResourceVersionMutationCache(informerIndexer, informerIndexer, 60*time.Second, false)
        
        // 添加ResourceEventHandler
	crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addCustomResourceDefinition,
		UpdateFunc: c.updateCustomResourceDefinition,
		DeleteFunc: c.deleteCustomResourceDefinition,
	})

	c.syncFn = c.sync

	return c
}

ResourceEventHandler

当发生CRD创建事件时，Informer会触发ResourceEventHandler 函数addCustomResourceDefinition执行。

它的逻辑很简单，就是检查下是不是CRD类型，如果是，把它加入workqueue.

func (c *NamingConditionController) addCustomResourceDefinition(obj interface{}) {
        // 检查下是不是CRD类型
	castObj := obj.(*apiextensionsv1.CustomResourceDefinition)
	klog.V(4).Infof("Adding %s", castObj.Name)
        // 把它加入workqueue
	c.enqueue(castObj)
}

Control loop

NamingController就是图中的Custom Controller:

Custom Controller是一个go routine方式启动的：

比如此情况下：

// 传入一个stopCh, channel关闭后停止
go namingController.Run(context.StopCh)

其执行体Run方法：

func (c *NamingConditionController) Run(stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	klog.Info("Starting NamingConditionController")
	defer klog.Info("Shutting down NamingConditionController")

        // 第一次情况下，等待从etcd整体同步全部CRD，后续动态维护减少etcd的压力
	if !cache.WaitForCacheSync(stopCh, c.crdSynced) {
		return
	}
        
	// only start one worker thread since its a slow moving API and the naming conflict resolution bits aren't thread-safe
	go wait.Until(c.runWorker, time.Second, stopCh) // 调用无限循环的runWorker()

	<-stopCh
}

func (c *NamingConditionController) runWorker() {
        // 无限循环, 等待下一个处理
	for c.processNextWorkItem() {
	}
}

// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
func (c *NamingConditionController) processNextWorkItem() bool {
        // Get blocks until it can return an item to be processed
	key, quit := c.queue.Get()
        // 队列关闭，则退出
	if quit {
		return false
	}
        // Done marks item as done processing
	defer c.queue.Done(key)

        // 调用同步方法，检测命名冲突
	err := c.syncFn(key.(string))
        // 没有冲突则忽略
	if err == nil {
		c.queue.Forget(key)
		return true
	}
        // 有命名冲突则重新加入workqueue, 进行重试
	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
	c.queue.AddRateLimited(key)

	return true
}

可见自定义控制器的Controll loop的基本逻辑，就是监听workqueue，当创建CRD事件发生时，调用sync方法进行命名冲突检测。

Sync方法

如果说前面是自定义控制器的通用逻辑的话，sync方法就是Naming Controller的独有逻辑。检测命名冲突就是在sync方法里实现的。

初始化的时候:

func NewNamingConditionController(
	crdInformer informers.CustomResourceDefinitionInformer,
	crdClient client.CustomResourceDefinitionsGetter,
) *NamingConditionController {
	c := &NamingConditionController{
		crdClient: crdClient,
                // ...
	}
        // Sync方法定义为NamingConditionController的sync方法
	c.syncFn = c.sync

	return c
}

sync方法：

• 返回err代表需要重试并重新入队(加入workqueue)，返回nil代表成功处理

func (c *NamingConditionController) sync(key string) error {
        // 通过key获取事件中的CRD对象
	inCustomResourceDefinition, err := c.crdLister.Get(key)
        
        // 如果没有找到，则把CRD所属组的所有CRD全部加入workqueue
	if apierrors.IsNotFound(err) {
		// CRD was deleted and has freed its names.
		// Reconsider all other CRDs in the same group.
		if err := c.requeueAllOtherGroupCRDs(key); err != nil {
			return err
		}
		return nil
	}
	if err != nil {
		return err
	}
        // CRD的Status字段中AcceptedNames代表实际已接受(不存在冲突)的命名
        
        
        // 如果kubectl apply的请求命名已经和已接受的命名相同，则
	// Skip checking names if Spec and Status names are same.
	if equality.Semantic.DeepEqual(inCustomResourceDefinition.Spec.Names, inCustomResourceDefinition.Status.AcceptedNames) {
		return nil
	}

// 检测CRD字段中, names字段是否存在命名冲突
// 如下：
// names:
//    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
//    plural: crontabs
//    # singular name to be used as an alias on the CLI and for display
//    singular: crontab
//    # kind is normally the CamelCased singular type. Your resource manifests use this.
//    kind: CronTab
//    # shortNames allow shorter string to match your resource on the CLI
//    shortNames:
//    - ct

// 返回的acceptedNames代表names中不存在冲突的字段
// namingCondition 包含一些冲突信息，比如 plural冲突等
// establishedCondition 包含已接受命名等信息
	acceptedNames, namingCondition, establishedCondition := c.calculateNamesAndConditions(inCustomResourceDefinition)




	// nothing to do if accepted names and NamesAccepted condition didn't change
	if reflect.DeepEqual(inCustomResourceDefinition.Status.AcceptedNames, acceptedNames) &&
		apiextensionshelpers.IsCRDConditionEquivalent(&namingCondition, apiextensionshelpers.FindCRDCondition(inCustomResourceDefinition, apiextensionsv1.NamesAccepted)) {
		return nil
	}

        // 如果存在命名状态更新，则更新CRD的Status字段信息
	crd := inCustomResourceDefinition.DeepCopy()
	crd.Status.AcceptedNames = acceptedNames
	apiextensionshelpers.SetCRDCondition(crd, namingCondition)
	apiextensionshelpers.SetCRDCondition(crd, establishedCondition)

	updatedObj, err := c.crdClient.CustomResourceDefinitions().UpdateStatus(context.TODO(), crd, metav1.UpdateOptions{})
	if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
		// deleted or changed in the meantime, we'll get called again
		return nil
	}
	if err != nil {
		return err
	}

	// if the update was successful, go ahead and add the entry to the mutation cache
	c.crdMutationCache.Mutation(updatedObj)

	// we updated our status, so we may be releasing a name.  When this happens, we need to rekick everything in our group
	// if we fail to rekick, just return as normal.  We'll get everything on a resync
	if err := c.requeueAllOtherGroupCRDs(key); err != nil {
		return err
	}

	return nil
}

• requeueAllOtherGroupCRDs

CRD所属组的全部CRD，重新加入workqueue

// name=crontabs.stable.example.com
func (c *NamingConditionController) requeueAllOtherGroupCRDs(name string) error {
	pluralGroup := strings.SplitN(name, ".", 2) // => ["crontabs","stable.example.com"]
	list, err := c.crdLister.List(labels.Everything())
	if err != nil {
		return err
	}
	for _, curr := range list {
		if curr.Spec.Group == pluralGroup[1] && curr.Name != name {
			c.queue.Add(curr.Name)
		}
	}
	return nil
}