Kubernetes-apiServer源码深度分析(三)GenericServer的概念和作用

开启掘金成长之旅！这是我参与「掘金日新计划 · 12 月更文挑战」的第5天，点击查看活动详情

记得看代码中的注释哈，理解都在里面 源码基于v1.24

本篇文章讲述的ServerChain中的最基本的apiServer，是aggregation server和Master API Server的基础功能的提供server。

GenericServer是什么呢？

• 每一个server都必须有响应HTTP的功能，于是把apiServer的一些共性都抽象出来，Generic中文翻译为通用的，是每个server的基础，不同的server在GenericServer上进行配置拓展，形成独特的server
• 响应HTTP的部分使用了go-restful这个库，先看看示例了解这个库的用法
• • container和webserver对应起来，container理解为一个mux，分发请求用的

package main
import (
	"log"
	"net/http"

	"github.com/emicklei/go-restful"
)

type User struct {
	Id, Name string
}

type UserResource struct {
	// normally one would use DAO (data access object)
	users map[string]User
}

func (u UserResource) Register(container *restful.Container) {
	ws := new(restful.WebService)
	ws.
		Path("/users").
		Consumes(restful.MIME_XML, restful.MIME_JSON).
		Produces(restful.MIME_JSON, restful.MIME_XML) // you can specify this per route as well

	ws.Route(ws.GET("/{user-id}").To(u.findUser))
	ws.Route(ws.POST("").To(u.updateUser))
	ws.Route(ws.PUT("/{user-id}").To(u.createUser))
	ws.Route(ws.DELETE("/{user-id}").To(u.removeUser))

	container.Add(ws)
}

// GET http://localhost:8080/users/1
//
func (u UserResource) findUser(request *restful.Request, response *restful.Response) {
	id := request.PathParameter("user-id")
	usr, ok := u.users[id]
	if !ok {
		response.AddHeader("Content-Type", "text/plain")
		response.WriteErrorString(http.StatusNotFound, "User could not be found.")
	} else {
		response.WriteEntity(usr)
	}
}

// POST http://localhost:8080/users
// <User><Id>1</Id><Name>Melissa Raspberry</Name></User>
//
func (u *UserResource) updateUser(request *restful.Request, response *restful.Response) {
	usr := new(User)
	err := request.ReadEntity(&usr)
	if err == nil {
		u.users[usr.Id] = *usr
		response.WriteEntity(usr)
	} else {
		response.AddHeader("Content-Type", "text/plain")
		response.WriteErrorString(http.StatusInternalServerError, err.Error())
	}
}

// PUT http://localhost:8080/users/1
// <User><Id>1</Id><Name>Melissa</Name></User>
//
func (u *UserResource) createUser(request *restful.Request, response *restful.Response) {
	usr := User{Id: request.PathParameter("user-id")}
	err := request.ReadEntity(&usr)
	if err == nil {
		u.users[usr.Id] = usr
		response.WriteHeaderAndEntity(http.StatusCreated, usr)
	} else {
		response.AddHeader("Content-Type", "text/plain")
		response.WriteErrorString(http.StatusInternalServerError, err.Error())
	}
}

// DELETE http://localhost:8080/users/1
//
func (u *UserResource) removeUser(request *restful.Request, response *restful.Response) {
	id := request.PathParameter("user-id")
	delete(u.users, id)
}

func main() {
	wsContainer := restful.NewContainer()
	wsContainer.Router(restful.CurlyRouter{})
	u := UserResource{map[string]User{}}
	u.Register(wsContainer)

	log.Printf("start listening on localhost:8080")
	server := &http.Server{Addr: ":8080", Handler: wsContainer}
	log.Fatal(server.ListenAndServe())
}

GenericServer的构建

func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
...
   /*
   这个GenericConfig的New就是GenericServer的构建
    */
   s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
   if err != nil {
      return nil, err
   }
...
   /*
   在这里有个Instance，是MasterServer的代名词
    */
   m := &Instance{
      GenericAPIServer:          s,
      ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo,
   }
----------------

func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
   if c.Serializer == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
   }
   if c.LoopbackClientConfig == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
   }
   if c.EquivalentResourceRegistry == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.EquivalentResourceRegistry == nil")
   }

   handlerChainBuilder := func(handler http.Handler) http.Handler {
      return c.BuildHandlerChainFunc(handler, c.Config)
   }
/*
    生成这个Handler进行响应
*/
   apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
/*
    构建这个genericServer 根据配置来的
*/
   s := &GenericAPIServer{
      discoveryAddresses:         c.DiscoveryAddresses,
      LoopbackClientConfig:       c.LoopbackClientConfig,
      legacyAPIGroupPrefixes:     c.LegacyAPIGroupPrefixes,
      admissionControl:           c.AdmissionControl,
      Serializer:                 c.Serializer,
      AuditBackend:               c.AuditBackend,
      Authorizer:                 c.Authorization.Authorizer,
      delegationTarget:           delegationTarget,
      EquivalentResourceRegistry: c.EquivalentResourceRegistry,
      HandlerChainWaitGroup:      c.HandlerChainWaitGroup,
      Handler:                    apiServerHandler,

      listedPathProvider: apiServerHandler,

      minRequestTimeout:     time.Duration(c.MinRequestTimeout) * time.Second,
      ShutdownTimeout:       c.RequestTimeout,
      ShutdownDelayDuration: c.ShutdownDelayDuration,
      SecureServingInfo:     c.SecureServing,
      ExternalAddress:       c.ExternalAddress,

      openAPIConfig:           c.OpenAPIConfig,
      openAPIV3Config:         c.OpenAPIV3Config,
      skipOpenAPIInstallation: c.SkipOpenAPIInstallation,

      postStartHooks:         map[string]postStartHookEntry{},
      preShutdownHooks:       map[string]preShutdownHookEntry{},
      disabledPostStartHooks: c.DisabledPostStartHooks,

      healthzChecks:    c.HealthzChecks,
      livezChecks:      c.LivezChecks,
      readyzChecks:     c.ReadyzChecks,
      livezGracePeriod: c.LivezGracePeriod,

      DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),

      maxRequestBodyBytes: c.MaxRequestBodyBytes,
      livezClock:          clock.RealClock{},

      lifecycleSignals:       c.lifecycleSignals,
      ShutdownSendRetryAfter: c.ShutdownSendRetryAfter,

      APIServerID:           c.APIServerID,
      StorageVersionManager: c.StorageVersionManager,

      Version: c.Version,

      muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
   }
/*
    一些Hook的加载
*/
   for {
      if c.JSONPatchMaxCopyBytes <= 0 {
         break
      }
      existing := atomic.LoadInt64(&jsonpatch.AccumulatedCopySizeLimit)
      if existing > 0 && existing < c.JSONPatchMaxCopyBytes {
         break
      }
      if atomic.CompareAndSwapInt64(&jsonpatch.AccumulatedCopySizeLimit, existing, c.JSONPatchMaxCopyBytes) {
         break
      }
   }

   // first add poststarthooks from delegated targets
   for k, v := range delegationTarget.PostStartHooks() {
      s.postStartHooks[k] = v
   }

   for k, v := range delegationTarget.PreShutdownHooks() {
      s.preShutdownHooks[k] = v
   }

   // add poststarthooks that were preconfigured.  Using the add method will give us an error if the same name has already been registered.
   for name, preconfiguredPostStartHook := range c.PostStartHooks {
      if err := s.AddPostStartHook(name, preconfiguredPostStartHook.hook); err != nil {
         return nil, err
      }
   }

   // register mux signals from the delegated server
   for k, v := range delegationTarget.MuxAndDiscoveryCompleteSignals() {
      if err := s.RegisterMuxAndDiscoveryCompleteSignal(k, v); err != nil {
         return nil, err
      }
   }

   genericApiServerHookName := "generic-apiserver-start-informers"
   if c.SharedInformerFactory != nil {
      if !s.isPostStartHookRegistered(genericApiServerHookName) {
         err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
            c.SharedInformerFactory.Start(context.StopCh)
            return nil
         })
         if err != nil {
            return nil, err
         }
      }
      err := s.AddReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory))
      if err != nil {
         return nil, err
      }
   }

   const priorityAndFairnessConfigConsumerHookName = "priority-and-fairness-config-consumer"
   if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) {
   } else if c.FlowControl != nil {
      err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(context PostStartHookContext) error {
         go c.FlowControl.Run(context.StopCh)
         return nil
      })
      if err != nil {
         return nil, err
      }
      // TODO(yue9944882): plumb pre-shutdown-hook for request-management system?
   } else {
      klog.V(3).Infof("Not requested to run hook %s", priorityAndFairnessConfigConsumerHookName)
   }

   // Add PostStartHooks for maintaining the watermarks for the Priority-and-Fairness and the Max-in-Flight filters.
   if c.FlowControl != nil {
      const priorityAndFairnessFilterHookName = "priority-and-fairness-filter"
      if !s.isPostStartHookRegistered(priorityAndFairnessFilterHookName) {
         err := s.AddPostStartHook(priorityAndFairnessFilterHookName, func(context PostStartHookContext) error {
            genericfilters.StartPriorityAndFairnessWatermarkMaintenance(context.StopCh)
            return nil
         })
         if err != nil {
            return nil, err
         }
      }
   } else {
      const maxInFlightFilterHookName = "max-in-flight-filter"
      if !s.isPostStartHookRegistered(maxInFlightFilterHookName) {
         err := s.AddPostStartHook(maxInFlightFilterHookName, func(context PostStartHookContext) error {
            genericfilters.StartMaxInFlightWatermarkMaintenance(context.StopCh)
            return nil
         })
         if err != nil {
            return nil, err
         }
      }
   }

   // Add PostStartHook for maintenaing the object count tracker.
   if c.StorageObjectCountTracker != nil {
      const storageObjectCountTrackerHookName = "storage-object-count-tracker-hook"
      if !s.isPostStartHookRegistered(storageObjectCountTrackerHookName) {
         if err := s.AddPostStartHook(storageObjectCountTrackerHookName, func(context PostStartHookContext) error {
            go c.StorageObjectCountTracker.RunUntil(context.StopCh)
            return nil
         }); err != nil {
            return nil, err
         }
      }
   }

   for _, delegateCheck := range delegationTarget.HealthzChecks() {
      skip := false
      for _, existingCheck := range c.HealthzChecks {
         if existingCheck.Name() == delegateCheck.Name() {
            skip = true
            break
         }
      }
      if skip {
         continue
      }
      s.AddHealthChecks(delegateCheck)
   }

   s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}

   installAPI(s, c.Config)

   // use the UnprotectedHandler from the delegation target to ensure that we don't attempt to double authenticator, authorize,
   // or some other part of the filter chain in delegation cases.
   if delegationTarget.UnprotectedHandler() == nil && c.EnableIndex {
      s.Handler.NonGoRestfulMux.NotFoundHandler(routes.IndexLister{
         StatusCode:   http.StatusNotFound,
         PathProvider: s.listedPathProvider,
      })
   }

   return s, nil
}

ApiServerHandler的构建

用的是go-restful这个库去构建rest的HTTP服务

func NewAPIServerHandler(name string, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
   nonGoRestfulMux := mux.NewPathRecorderMux(name)
   if notFoundHandler != nil {
      nonGoRestfulMux.NotFoundHandler(notFoundHandler)
   }

   gorestfulContainer := restful.NewContainer()
   gorestfulContainer.ServeMux = http.NewServeMux()
   gorestfulContainer.Router(restful.CurlyRouter{}) // e.g. for proxy/{kind}/{name}/{*}
   gorestfulContainer.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
      logStackOnRecover(s, panicReason, httpWriter)
   })
   gorestfulContainer.ServiceErrorHandler(func(serviceErr restful.ServiceError, request *restful.Request, response *restful.Response) {
      serviceErrorHandler(s, serviceErr, request, response)
   })

   director := director{
      name:               name,
      goRestfulContainer: gorestfulContainer,
      nonGoRestfulMux:    nonGoRestfulMux,
   }
/*
    主要由这个director来处理请求 用handlerChainBuilder加了一层
*/
   return &APIServerHandler{
      FullHandlerChain:   handlerChainBuilder(director),
      GoRestfulContainer: gorestfulContainer,
      NonGoRestfulMux:    nonGoRestfulMux,
      Director:           director,
   }
}

BuildHandlerChain

用到了装饰器的设计模式，装饰器模式（Decorator Pattern） 也称为包装模式(Wrapper Pattern) 是指在不改变原有对象的基础之上，将功能附加到对象上，提供了比继承更有弹性的替代方案(扩展原有对象的功能)，属于结构型模式。装饰器模式的核心是功能扩展，使用装饰器模式可以透明且动态地扩展类的功能。

/*
    一些filter，要先过这一层才访问到服务
*/
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
   handler := filterlatency.TrackCompleted(apiHandler)
   handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
   handler = filterlatency.TrackStarted(handler, "authorization")

   if c.FlowControl != nil {
      workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
      requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
         c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg)
      handler = filterlatency.TrackCompleted(handler)
      handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator)
      handler = filterlatency.TrackStarted(handler, "priorityandfairness")
   } else {
      handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
   }

   handler = filterlatency.TrackCompleted(handler)
   handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
   handler = filterlatency.TrackStarted(handler, "impersonation")

   handler = filterlatency.TrackCompleted(handler)
   handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc)
   handler = filterlatency.TrackStarted(handler, "audit")

   failedHandler := genericapifilters.Unauthorized(c.Serializer)
   failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator)

   failedHandler = filterlatency.TrackCompleted(failedHandler)
   handler = filterlatency.TrackCompleted(handler)
   handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
   handler = filterlatency.TrackStarted(handler, "authentication")

   handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")

   // WithTimeoutForNonLongRunningRequests will call the rest of the request handling in a go-routine with the
   // context with deadline. The go-routine can keep running, while the timeout logic will return a timeout to the client.
   handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc)

   handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator,
      c.LongRunningFunc, c.Serializer, c.RequestTimeout)
   handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
   if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
      handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
   }
   handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
   handler = genericapifilters.WithWarningRecorder(handler)
   handler = genericapifilters.WithCacheControl(handler)
   handler = genericfilters.WithHSTS(handler, c.HSTSDirectives)
   if c.ShutdownSendRetryAfter {
      handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
   }
   handler = genericfilters.WithHTTPLogging(handler)
   if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
      handler = genericapifilters.WithTracing(handler, c.TracerProvider)
   }
   handler = genericapifilters.WithLatencyTrackers(handler)
   handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
   handler = genericapifilters.WithRequestReceivedTimestamp(handler)
   handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())
   handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
   handler = genericapifilters.WithAuditID(handler)
   return handler
}

怎么形成服务链的呢？

/*
    delegationTarget.UnprotectedHandler()在参数传入的时候就是传的genericServer
*/
apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
/*
    实际上就是notFoundHandler，在CreateServerChain的时候就构建了这个，当这个server处理不了的时候就交给下一个server，环环相扣
*/
func CreateServerChain(completedOptions completedServerRunOptions) (*aggregatorapiserver.APIAggregator, error) {
   kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions)
   if err != nil {
      return nil, err
   }
/*
   先配置在创建server，上一个的server被下一个server利用
 */
   // If additional API servers are added, they should be gated.
   apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount,
      serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIServerConfig.ExtraConfig.ProxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.GenericConfig.TracerProvider))
   if err != nil {
      return nil, err
   }

   notFoundHandler := notfoundhandler.New(kubeAPIServerConfig.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
   apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
   if err != nil {
      return nil, err
   }

   kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer)
   if err != nil {
      return nil, err
   }
/*
   aggregator相当于一个请求的转发器，把不同路径的请求转发到不同server去处理
 */
   // aggregator comes last in the chain
   aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer)
   if err != nil {
      return nil, err
   }
   aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
   if err != nil {
      // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
      return nil, err
   }

   return aggregatorServer, nil
}
/*
    最终调用的是APIServerHandler中director的ServerHTTP提供服务
*/
// ServeHTTP makes it an http.Handler
func (a *APIServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
   a.FullHandlerChain.ServeHTTP(w, r)
}

func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    ...
   d.nonGoRestfulMux.ServeHTTP(w, req)
}

服务链如下图所示

Summary

1. GenericServer是每一个server的底层通用server，提供了一些apiServer的共性，如响应HTTP的能力
2. 通过GenericServer,各个server之间形成了一条serverChain，也能方便用户通过AggregationServer来拓展apiServer的功能
3. 真正发挥HTTP作用的是GenericServer中的APIServerHandler中的director