Kubernetes-apiServer源码深度分析(十一)HttpReq的处理过程和Default Filters

开启掘金成长之旅！这是我参与「掘金日新计划 · 12 月更文挑战」的第13天，点击查看活动详情

记得看代码中的注释哈，理解都在里面 源码基于v1.24

本篇文章主要介绍，一个请求来到kube-apiServer的时候，所经历的一系列过程，其中就有default filter，默认的过滤器

RequestHandler的构建

在GenericServer的创建中

func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
   if c.Serializer == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
   }
   if c.LoopbackClientConfig == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
   }
   if c.EquivalentResourceRegistry == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.EquivalentResourceRegistry == nil")
   }
/*
    构建Handler
*/
   handlerChainBuilder := func(handler http.Handler) http.Handler {
      return c.BuildHandlerChainFunc(handler, c.Config)
   }
   ...
   }

func NewConfig(codecs serializer.CodecFactory) *Config {
   defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
   var id string
   if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
      id = "kube-apiserver-" + uuid.New().String()
   }
   lifecycleSignals := newLifecycleSignals()

   return &Config{
      Serializer:                  codecs,
      /*
          传入了默认的BuildHandlerChain
      */
      BuildHandlerChainFunc:       DefaultBuildHandlerChain,
      HandlerChainWaitGroup:       new(utilwaitgroup.SafeWaitGroup),
      LegacyAPIGroupPrefixes:      sets.NewString(DefaultLegacyAPIPrefix),
      ...

/*
    接下来就整个的处理链路，filterChain，在执行时 越晚注册的越先执行
*/
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
   handler := filterlatency.TrackCompleted(apiHandler)
   handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
   handler = filterlatency.TrackStarted(handler, "authorization")

   if c.FlowControl != nil {
      workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
      requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
         c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg)
      handler = filterlatency.TrackCompleted(handler)
      handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator)
      handler = filterlatency.TrackStarted(handler, "priorityandfairness")
   } else {
      handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
   }

   handler = filterlatency.TrackCompleted(handler)
   handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
   handler = filterlatency.TrackStarted(handler, "impersonation")

   handler = filterlatency.TrackCompleted(handler)
   handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc)
   handler = filterlatency.TrackStarted(handler, "audit")

   failedHandler := genericapifilters.Unauthorized(c.Serializer)
   failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator)

   failedHandler = filterlatency.TrackCompleted(failedHandler)
   handler = filterlatency.TrackCompleted(handler)
   handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
   handler = filterlatency.TrackStarted(handler, "authentication")

   handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")

   // WithTimeoutForNonLongRunningRequests will call the rest of the request handling in a go-routine with the
   // context with deadline. The go-routine can keep running, while the timeout logic will return a timeout to the client.
   handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc)

   handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator,
      c.LongRunningFunc, c.Serializer, c.RequestTimeout)
   handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
   if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
      handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
   }
   handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
   handler = genericapifilters.WithWarningRecorder(handler)
   handler = genericapifilters.WithCacheControl(handler)
   handler = genericfilters.WithHSTS(handler, c.HSTSDirectives)
   if c.ShutdownSendRetryAfter {
      handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
   }
   handler = genericfilters.WithHTTPLogging(handler)
   if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
      handler = genericapifilters.WithTracing(handler, c.TracerProvider)
   }
   handler = genericapifilters.WithLatencyTrackers(handler)
   handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
   handler = genericapifilters.WithRequestReceivedTimestamp(handler)
   handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())
   handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
   handler = genericapifilters.WithAuditID(handler)
   return handler
}
/*
    举一个例子  添加了Request的相关信息到Context
*/
func WithRequestInfo(handler http.Handler, resolver request.RequestInfoResolver) http.Handler {
   return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
      ctx := req.Context()
      info, err := resolver.NewRequestInfo(req)
      if err != nil {
         responsewriters.InternalError(w, req, fmt.Errorf("failed to create RequestInfo: %v", err))
         return
      }

      req = req.WithContext(request.WithRequestInfo(ctx, info))

      handler.ServeHTTP(w, req)
   })
}

流程解析如下图

编解码-HttpPayload与Go结构实例之间的转换

充当的作用如图 Serializer的加载与Admission如出一辙

func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, openAPIModels openapiproto.Models) error {
   var resourceInfos []*storageversion.ResourceInfo
   for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
      if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
         klog.Warningf("Skipping API %v because it has no resources.", groupVersion)
         continue
      }

      apiGroupVersion, err := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix)
      if err != nil {
         return err
      }
      if apiGroupInfo.OptionsExternalVersion != nil {
         apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
      }
      apiGroupVersion.OpenAPIModels = openAPIModels

      if openAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
         typeConverter, err := fieldmanager.NewTypeConverter(openAPIModels, false)
         if err != nil {
            return err
         }
         apiGroupVersion.TypeConverter = typeConverter
      }

      apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes

      r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
      if err != nil {
         return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
      }
      resourceInfos = append(resourceInfos, r...)
   }

   s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)

   if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
      utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
      // API installation happens before we start listening on the handlers,
      // therefore it is safe to register ResourceInfos here. The handler will block
      // write requests until the storage versions of the targeting resources are updated.
      s.StorageVersionManager.AddResourceInfo(resourceInfos...)
   }

   return nil
}

func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) (*genericapi.APIGroupVersion, error) {
   storage := make(map[string]rest.Storage)
   for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
      if strings.ToLower(k) != k {
         return nil, fmt.Errorf("resource names must be lowercase only, not %q", k)
      }
      storage[k] = v
   }
   version := s.newAPIGroupVersion(apiGroupInfo, groupVersion)
   version.Root = apiPrefix
   version.Storage = storage
   return version, nil
}
/*
    在这里传入的Serializer
*/
func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion {
   return &genericapi.APIGroupVersion{
      GroupVersion:     groupVersion,
      MetaGroupVersion: apiGroupInfo.MetaGroupVersion,

      ParameterCodec:        apiGroupInfo.ParameterCodec,
      Serializer:            apiGroupInfo.NegotiatedSerializer,
      Creater:               apiGroupInfo.Scheme,
      Convertor:             apiGroupInfo.Scheme,
      ConvertabilityChecker: apiGroupInfo.Scheme,
      UnsafeConvertor:       runtime.UnsafeObjectConvertor(apiGroupInfo.Scheme),
      Defaulter:             apiGroupInfo.Scheme,
      Typer:                 apiGroupInfo.Scheme,
      Namer:                 runtime.Namer(meta.NewAccessor()),

      EquivalentResourceRegistry: s.EquivalentResourceRegistry,

      Admit:             s.admissionControl,
      MinRequestTimeout: s.minRequestTimeout,
      Authorizer:        s.Authorizer,
   }
}
/*
    使用 Serializer
*/
func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, *storageversion.ResourceInfo, error) {
   admit := a.group.Admit
   
    reqScope := handlers.RequestScope{
       Serializer:      a.group.Serializer,
       ParameterCodec:  a.group.ParameterCodec,
       Creater:         a.group.Creater,
       Convertor:       a.group.Convertor,
       Defaulter:       a.group.Defaulter,
       Typer:           a.group.Typer,
       UnsafeConvertor: a.group.UnsafeConvertor,
       Authorizer:      a.group.Authorizer,

       EquivalentResourceMapper: a.group.EquivalentResourceRegistry,

       // TODO: Check for the interface on storage
       TableConvertor: tableProvider,

       // TODO: This seems wrong for cross-group subresources. It makes an assumption that a subresource and its parent are in the same group version. Revisit this.
       Resource:    a.group.GroupVersion.WithResource(resource),
       Subresource: subresource,
       Kind:        fqKindToRegister,

       AcceptsGroupVersionDelegate: gvAcceptor,

       HubGroupVersion: schema.GroupVersion{Group: fqKindToRegister.Group, Version: runtime.APIVersionInternal},

       MetaGroupVersion: metav1.SchemeGroupVersion,

       MaxRequestBodyBytes: a.group.MaxRequestBodyBytes,
  }
/*
    发挥作用 在构造handler的时候就需要reqScope做传入参数
*/
switch action.Verb {
case "GET": // Get a resource.
   var handler restful.RouteFunction
   if isGetterWithOptions {
      handler = restfulGetResourceWithOptions(getterWithOptions, reqScope, isSubresource)
   } else {
      handler = restfulGetResource(getter, reqScope)
   }

   if needOverride {
      // need change the reported verb
      handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
   } else {
      handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
   }
   handler = utilwarning.AddWarningsHandler(handler, warnings)

![image.png](https://p1-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/eabfde588b0f4e7f8f432b5d8bf88a16~tplv-k3u1fbpfcp-watermark.image?)
   doc := "read the specified " + kind
   if isSubresource {
      doc = "read " + subresource + " of the specified " + kind
   }
   route := ws.GET(action.Path).To(handler).
      Doc(doc).
      Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
      Operation("read"+namespaced+kind+strings.Title(subresource)+operationSuffix).
      Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
      Returns(http.StatusOK, "OK", producedObject).
      Writes(producedObject)
   if isGetterWithOptions {
      if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
         return nil, nil, err
      }
   }
   addParams(route, action.Params)
   routes = append(routes, route)

流程解析如下图

对Request进行响应的业务逻辑部分-Store

每一个APlObject都会有一个REST结构体，负责最终处理针对本 Object的Request;而这个REST结构体大多时候通过内嵌 genericregistry.Store结构体来直接复用其属性和方法，特别是内建 APIObject，所以这个Store结构体包含了大多数处理逻辑

上述REST结构体一般定义在APlObject相应的storage/storage.go文 件中;例如group“apps”的deployment如下图所示;该文件内还 会有NewREST方法来构造并返回REST实例，包括子Object的

流程解析如下图

type RESTCreateStrategy interface {
   runtime.ObjectTyper

   names.NameGenerator

   // NamespaceScoped returns true if the object must be within a namespace.
   NamespaceScoped() bool
    /*
    
在验证之前，在创建时调用PrepareForcreate来规范化对象。例如:删除不需要持久化的字段，对顺序不敏感的列表字段进行排序等。这不应该删除那些被认为是验证错误的字段。
    */
   PrepareForCreate(ctx context.Context, obj runtime.Object)

   Validate(ctx context.Context, obj runtime.Object) field.ErrorList

   WarningsOnCreate(ctx context.Context, obj runtime.Object) []string

   Canonicalize(obj runtime.Object)
}

响应过程

func restfulUpdateResource(r rest.Updater, scope handlers.RequestScope, admit admission.Interface) restful.RouteFunction {
   return func(req *restful.Request, res *restful.Response) {
      handlers.UpdateResource(r, &scope, admit)(res.ResponseWriter, req.Request)
   }
}

func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interface) http.HandlerFunc {
   return func(w http.ResponseWriter, req *http.Request) {
      // For performance tracking purposes.
      trace := utiltrace.New("Update", traceFields(req)...)
      defer trace.LogIfLong(500 * time.Millisecond)

      namespace, name, err := scope.Namer.Name(req)
      if err != nil {
         scope.err(err, w, req)
         return
      }

    trace.Step("About to store object in database")
    wasCreated := false
    requestFunc := func() (runtime.Object, error) {
       obj, created, err := r.Update(
          ctx,
          name,
          rest.DefaultUpdatedObjectInfo(obj, transformers...),
          withAuthorization(rest.AdmissionToValidateObjectFunc(
             admit,
             admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, updateToCreateOptions(options), dryrun.IsDryRun(options.DryRun), userInfo), scope),
             scope.Authorizer, createAuthorizerAttributes),
          rest.AdmissionToValidateObjectUpdateFunc(
             admit,
             admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Update, options, dryrun.IsDryRun(options.DryRun), userInfo), scope),
          false,
          options,
       )
       wasCreated = created
       return obj, err
    }

流程解析如下图

Summary

1. DefaultFilterChain
2. Serializer的加载和使用
3. Store存储流程