Kubernetes 系列 - 7. apiserver（四、调用webhook）

7. apiserver（四、调用webhook）

apiserver在处理请求之后、存储进etcd之前会调用webhook进行校验、修改，我们可以利用这个机制对资源进行自定义操作，istio就是利用这一点往pod中注入了initContainer。

validatingWebook和mutatingWebhook逻辑类似，下面以validatingWebook为例进行分析。

7.1 注册webhook

注册校验webhook的资源示例如下：

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: example-webhook
webhooks:
  - name: example.example.com
    clientConfig:
      url: https://localhost:8443/validate
      caBundle: ${CaCrt_Base64}
    rules:
      - apiGroups: [""]
        apiVersions: ["v1"]
        operations: ["CREATE", "UPDATE"]
        resources: ["pods"]
    admissionReviewVersions: ["v1"]
    sideEffects: None

相关代码如下：

创建config时：

// CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them
func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
    *controlplane.Config,
    aggregatorapiserver.ServiceResolver,
    []admission.PluginInitializer,
    error,
) {
    ...
    err = opts.Admission.ApplyTo(
       genericConfig,
       versionedInformers,
       clientgoExternalClient,
       dynamicExternalClient,
       utilfeature.DefaultFeatureGate,
       pluginInitializers...)
    ...
    return config, serviceResolver, pluginInitializers, nil
}

创建apiserverConfig时调用opts.Admission.ApplyTo:

func (a *AdmissionOptions) ApplyTo(
    c *server.Config,
    informers informers.SharedInformerFactory,
    kubeClient kubernetes.Interface,
    dynamicClient dynamic.Interface,
    features featuregate.FeatureGate,
    pluginInitializers ...admission.PluginInitializer,
) error {
    if a == nil {
       return nil
    }

    // Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
    if informers == nil {
       return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
    }

    pluginNames := a.enabledPluginNames()

    pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
    if err != nil {
       return fmt.Errorf("failed to read plugin config: %v", err)
    }

    discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
    discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
    genericInitializer := initializer.New(kubeClient, dynamicClient, informers, c.Authorization.Authorizer, features,
       c.DrainedNotify(), discoveryRESTMapper)
    initializersChain := admission.PluginInitializers{genericInitializer}
    initializersChain = append(initializersChain, pluginInitializers...)

    admissionPostStartHook := func(context server.PostStartHookContext) error {
       discoveryRESTMapper.Reset()
       go utilwait.Until(discoveryRESTMapper.Reset, 30*time.Second, context.StopCh)
       return nil
    }

    err = c.AddPostStartHook("start-apiserver-admission-initializer", admissionPostStartHook)
    if err != nil {
       return fmt.Errorf("failed to add post start hook for policy admission: %w", err)
    }

    // 重点：构建webhook controller
    admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
    if err != nil {
       return err
    }

    c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain)
    return nil
}

各个plugin的初始化为：

// InitPlugin creates an instance of the named interface.
func (ps *Plugins) InitPlugin(name string, config io.Reader, pluginInitializer PluginInitializer) (Interface, error) {
    if name == "" {
       klog.Info("No admission plugin specified.")
       return nil, nil
    }

    // 1. 生成plugin
    plugin, found, err := ps.getPlugin(name, config)
    if err != nil {
       return nil, fmt.Errorf("couldn't init admission plugin %q: %v", name, err)
    }
    if !found {
       return nil, fmt.Errorf("unknown admission plugin: %s", name)
    }

    // 2. 初始化plugin
    pluginInitializer.Initialize(plugin)
    // ensure that plugins have been properly initialized
    if err := ValidateInitialization(plugin); err != nil {
       return nil, fmt.Errorf("failed to initialize admission plugin %q: %v", name, err)
    }

    return plugin, nil
}

7.1.1 生成plugin

validatingWebhookConfiguration的构建函数为：

// NewValidatingAdmissionWebhook returns a generic admission webhook plugin.
func NewValidatingAdmissionWebhook(configFile io.Reader) (*Plugin, error) {
    handler := admission.NewHandler(admission.Connect, admission.Create, admission.Delete, admission.Update)
    p := &Plugin{}
    var err error
    // 注意这里的configuration.NewValidatingWebhookConfigurationManager函数，后面需要用到
    p.Webhook, err = generic.NewWebhook(handler, configFile, configuration.NewValidatingWebhookConfigurationManager, newValidatingDispatcher(p))
    if err != nil {
       return nil, err
    }
    return p, nil
}

NewWebhook具体为：

// NewWebhook creates a new generic admission webhook.
func NewWebhook(handler *admission.Handler, configFile io.Reader, sourceFactory sourceFactory, dispatcherFactory dispatcherFactory) (*Webhook, error) {
    ...
    return &Webhook{
       Handler:          handler,
       // 这里的sourceFactory就是上述提到的configuration.NewValidatingWebhookConfigurationManager
       sourceFactory:    sourceFactory,
       clientManager:    &cm,
       namespaceMatcher: &namespace.Matcher{},
       objectMatcher:    &object.Matcher{},
       dispatcher:       dispatcherFactory(&cm),
       filterCompiler:   cel.NewFilterCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion(), utilfeature.DefaultFeatureGate.Enabled(features.StrictCostEnforcementForWebhooks))),
    }, nil
}

7.1.2 初始化plugin

// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface.
func (a *Webhook) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
    namespaceInformer := f.Core().V1().Namespaces()
    a.namespaceMatcher.NamespaceLister = namespaceInformer.Lister()
    // 重点：调用configuration.NewValidatingWebhookConfigurationManager函数
    a.hookSource = a.sourceFactory(f)
    a.SetReadyFunc(func() bool {
       return namespaceInformer.Informer().HasSynced() && a.hookSource.HasSynced()
    })
}

func NewValidatingWebhookConfigurationManager(f informers.SharedInformerFactory) generic.Source {
    informer := f.Admissionregistration().V1().ValidatingWebhookConfigurations()
    manager := &validatingWebhookConfigurationManager{
       lister:                          informer.Lister(),
       createValidatingWebhookAccessor: webhook.NewValidatingWebhookAccessor,
    }
    manager.lazy.Evaluate = manager.getConfiguration

    // 这里注意一下由AddFunc、UpdateFunc等组成的处理函数，后续将用于组成listener
    handle, _ := informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
       AddFunc: func(_ interface{}) { manager.lazy.Notify() },
       UpdateFunc: func(old, new interface{}) {
          obj := new.(*v1.ValidatingWebhookConfiguration)
          manager.configurationsCache.Delete(obj.GetName())
          manager.lazy.Notify()
       },
       DeleteFunc: func(obj interface{}) {
          vwc, ok := obj.(*v1.ValidatingWebhookConfiguration)
          if !ok {
             tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
             if !ok {
                klog.V(2).Infof("Couldn't get object from tombstone %#v", obj)
                return
             }
             vwc, ok = tombstone.Obj.(*v1.ValidatingWebhookConfiguration)
             if !ok {
                klog.V(2).Infof("Tombstone contained object that is not expected %#v", obj)
                return
             }
          }
          manager.configurationsCache.Delete(vwc.Name)
          manager.lazy.Notify()
       },
    })
    manager.hasSynced = handle.HasSynced

    return manager
}

调用informer.Informer().AddEventHandler最后会到如下代码：

func (s *sharedIndexInformer) AddEventHandlerWithResyncPeriod(handler ResourceEventHandler, resyncPeriod time.Duration) (ResourceEventHandlerRegistration, error) {
    ...

    // 1. 生成listener
    listener := newProcessListener(handler, resyncPeriod, determineResyncPeriod(resyncPeriod, s.resyncCheckPeriod), s.clock.Now(), initialBufferSize, s.HasSynced)

    // 2. 将listener放进sharedIndexInformer
    if !s.started {
       return s.processor.addListener(listener), nil
    }

    ...
    return handle, nil
}

newProcessListener主要就是将处理函数传给handler：

func newProcessListener(handler ResourceEventHandler, requestedResyncPeriod, resyncPeriod time.Duration, now time.Time, bufferSize int, hasSynced func() bool) *processorListener {
    ret := &processorListener{
       nextCh:                make(chan interface{}),
       addCh:                 make(chan interface{}),
       handler:               handler,
       syncTracker:           &synctrack.SingleFileTracker{UpstreamHasSynced: hasSynced},
       pendingNotifications:  *buffer.NewRingGrowing(bufferSize),
       requestedResyncPeriod: requestedResyncPeriod,
       resyncPeriod:          resyncPeriod,
    }

    ret.determineNextResync(now)

    return ret
}

组成listener之后放进sharedIndexInformer中：

func (p *sharedProcessor) addListener(listener *processorListener) ResourceEventHandlerRegistration {
    p.listenersLock.Lock()
    defer p.listenersLock.Unlock()

    if p.listeners == nil {
       p.listeners = make(map[*processorListener]bool)
    }

    // 主要这里将listener放进sharedIndexInformer中
    p.listeners[listener] = true

    // 注意这里p.listenersStarted为false，实际上listener未启动
    if p.listenersStarted {
       p.wg.Start(listener.run)
       p.wg.Start(listener.pop)
    }

    return listener
}

7.1.3 启动listener

在genericConfig创建的时候添加了postStartHook：

genericApiServerHookName := "generic-apiserver-start-informers"
if c.SharedInformerFactory != nil {
    if !s.isPostStartHookRegistered(genericApiServerHookName) {
       err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
          c.SharedInformerFactory.Start(context.StopCh)
          return nil
       })
       if err != nil {
          return nil, err
       }
    }
    // TODO: Once we get rid of /healthz consider changing this to post-start-hook.
    err := s.AddReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory))
    if err != nil {
       return nil, err
    }
}

启动后会启动SharedInformerFactory：

func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
    f.lock.Lock()
    defer f.lock.Unlock()

    if f.shuttingDown {
       return
    }

    for informerType, informer := range f.informers {
       if !f.startedInformers[informerType] {
          f.wg.Add(1)
          // We need a new variable in each loop iteration,
          // otherwise the goroutine would use the loop variable
          // and that keeps changing.
          informer := informer
          go func() {
             defer f.wg.Done()
             informer.Run(stopCh)
          }()
          f.startedInformers[informerType] = true
       }
    }
}

启动informer：

func (s *sharedIndexInformer) Run(stopCh <-chan struct{}) {
    defer utilruntime.HandleCrash()

    if s.HasStarted() {
       klog.Warningf("The sharedIndexInformer has started, run more than once is not allowed")
       return
    }

    func() {
       s.startedLock.Lock()
       defer s.startedLock.Unlock()

       fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{
          KnownObjects:          s.indexer,
          EmitDeltaTypeReplaced: true,
          Transformer:           s.transform,
       })

       cfg := &Config{
          Queue:             fifo,
          ListerWatcher:     s.listerWatcher,
          ObjectType:        s.objectType,
          ObjectDescription: s.objectDescription,
          FullResyncPeriod:  s.resyncCheckPeriod,
          RetryOnError:      false,
          ShouldResync:      s.processor.shouldResync,

          Process:           s.HandleDeltas,
          WatchErrorHandler: s.watchErrorHandler,
       }

       s.controller = New(cfg)
       s.controller.(*controller).clock = s.clock
       s.started = true
    }()

    // Separate stop channel because Processor should be stopped strictly after controller
    processorStopCh := make(chan struct{})
    var wg wait.Group
    defer wg.Wait()              // Wait for Processor to stop
    defer close(processorStopCh) // Tell Processor to stop
    wg.StartWithChannel(processorStopCh, s.cacheMutationDetector.Run)
    // 启动listener
    wg.StartWithChannel(processorStopCh, s.processor.run)

    defer func() {
       s.startedLock.Lock()
       defer s.startedLock.Unlock()
       s.stopped = true // Don't want any new listeners
    }()
    // 启动controller
    s.controller.Run(stopCh)
}

7.2 调用validateWebhook

func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error {
    var relevantHooks []*generic.WebhookInvocation
    // Construct all the versions we need to call our webhooks
    versionedAttrAccessor := &versionedAttributeAccessor{
       versionedAttrs:   map[schema.GroupVersionKind]*admission.VersionedAttributes{},
       attr:             attr,
       objectInterfaces: o,
    }
    for _, hook := range hooks {
       // 重点：选择webhook
       invocation, statusError := d.plugin.ShouldCallHook(ctx, hook, attr, o, versionedAttrAccessor)
       if invocation == nil {
           continue
       }

       relevantHooks = append(relevantHooks, invocation)
       // VersionedAttr result will be cached and reused later during parallel webhook calls
       _, err := versionedAttrAccessor.VersionedAttribute(invocation.Kind)
       if err != nil {
          return apierrors.NewInternalError(err)
       }
    }

    if len(relevantHooks) == 0 {
       // no matching hooks
       return nil
    }

    // Check if the request has already timed out before spawning remote calls
    select {
    case <-ctx.Done():
       // parent context is canceled or timed out, no point in continuing
       return apierrors.NewTimeoutError("request did not complete within requested timeout", 0)
    default:
    }

    wg := sync.WaitGroup{}
    errCh := make(chan error, 2*len(relevantHooks)) // double the length to handle extra errors for panics in the gofunc
    wg.Add(len(relevantHooks))
    for i := range relevantHooks {
       go func(invocation *generic.WebhookInvocation, idx int) {
          ...

          hook, ok := invocation.Webhook.GetValidatingWebhook()
          if !ok {
             utilruntime.HandleError(fmt.Errorf("validating webhook dispatch requires v1.ValidatingWebhook, but got %T", hook))
             return
          }
          hookName = hook.Name
          ignoreClientCallFailures = hook.FailurePolicy != nil && *hook.FailurePolicy == v1.Ignore
          t := time.Now()
          // 重点：调用webhook
          err := d.callHook(ctx, hook, invocation, versionedAttr)
          rejected := false
          if err != nil {
             switch err := err.(type) {
             case *webhookutil.ErrCallingWebhook:
                if !ignoreClientCallFailures {
                   rejected = true
                }
             case *webhookutil.ErrWebhookRejection:
                rejected = true
             default:
                rejected = true
             }
          } else {
             return
          }

          if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
             if ignoreClientCallFailures {
                // Ignore context cancelled from webhook metrics
                if errors.Is(callErr.Reason, context.Canceled) {
                   ...
                } else {
                   key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
                   value := hook.Name
                   if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
                      klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, value, hook.Name, err)
                   }
                }
                utilruntime.HandleError(callErr)
                return
             }

             errCh <- apierrors.NewInternalError(err)
             return
          }

          if rejectionErr, ok := err.(*webhookutil.ErrWebhookRejection); ok {
             err = rejectionErr.Status
          }
          errCh <- err
       }(relevantHooks[i], i)
    }
    wg.Wait()
    close(errCh)

    var errs []error
    for e := range errCh {
       errs = append(errs, e)
    }
    if len(errs) == 0 {
       return nil
    }
    if len(errs) > 1 {
       for i := 1; i < len(errs); i++ {
          // TODO: merge status errors; until then, just return the first one.
          utilruntime.HandleError(errs[i])
       }
    }
    return errs[0]
}