业务在使用k8s容器平台进行镜像升级时，处于升级过程中的pod时常发生集群资源不足而导致一直pending，因此无法进行顺利升级甚至在集群资源不足时不敢进行升级。只是升级又不用占用新的资源，为什么就资源不足了呢?为解决此问题，本文深入分析了kube-scheduler调度源码，以及对其进行源码修改以解决statefulset 升级过程中由于资源不足而失败的问题。

1 kube-scheduler 核心调度原理分析

1.1 执行流程

Kube-scheduler 作为整个kubernetes集群的pod调度器，与其他组件具有类似的启动流程。启动入口在cmd/kube-scheduler/scheduler.go的app.NewSchedulerCommand()
随之进入cmd/kube-scheduler/app/server.go中找到runCommand()函数，这段代码进行调度器运行的准备工作，即一些参数及方法初始化配置，我们从Run(c.Complete(), stopCh)方法开始分析主要函数。

// Run runs the Scheduler.
func Run(c schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error {
   ---
   // Build a scheduler config from the provided algorithm source.
   schedulerConfig, err := NewSchedulerConfig(c)
   if err != nil {
      return err
   }
   ---
   // Start all informers.
   go c.PodInformer.Informer().Run(stopCh)
   //go c.StatefulsetplusInformerFactory.Platform().V1alpha1().StatefulSetPluses().Informer().Run(stopCh)
   c.InformerFactory.Start(stopCh)

   // Wait for all caches to sync before scheduling.
   c.InformerFactory.WaitForCacheSync(stopCh)
   controller.WaitForCacheSync("scheduler", stopCh, c.PodInformer.Informer().HasSynced)

   // Prepare a reusable run function.
   run := func(ctx context.Context) {
      sched.Run()
      <-ctx.Done()
   }

   ctx, cancel := context.WithCancel(context.TODO()) // TODO once Run() accepts a context, it should be used here
   defer cancel()

   go func() {
      select {
      case <-stopCh:
         cancel()
      case <-ctx.Done():
      }
   }()

   // If leader election is enabled, run via LeaderElector until done and exit.
   if c.LeaderElection != nil {
      c.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{
         OnStartedLeading: run,
         OnStoppedLeading: func() {
            utilruntime.HandleError(fmt.Errorf("lost master"))
         },
      }
      leaderElector, err := leaderelection.NewLeaderElector(*c.LeaderElection)
      if err != nil {
         return fmt.Errorf("couldn't create leader elector: %v", err)
      }

      leaderElector.Run(ctx)

      return fmt.Errorf("lost lease")
   }

   // Leader election is disabled, so run inline until done.
   run(ctx)
   return fmt.Errorf("finished without leader elect")

在此重点关注生成scheduler配置的方法，schedulerConfig, err := NewSchedulerConfig() 它包含了scheduler工作所用到的所有与apiserver通信的client以及informer事件。尤其是关于scheduler队列的CRUD事件都在此配置中，进入NewSchedulerConfig()方法

// NewSchedulerConfig creates the scheduler configuration. This is exposed for use by tests.
func NewSchedulerConfig(s schedulerserverconfig.CompletedConfig) (*scheduler.Config, error) {
  ---
// Set up the configurator which can create schedulers from configs.
configurator := factory.NewConfigFactory(&factory.ConfigFactoryArgs{
   SchedulerName:                  s.ComponentConfig.SchedulerName,
   Client:                         s.Client,
   NodeInformer:                   s.InformerFactory.Core().V1().Nodes(),
   PodInformer:                    s.PodInformer,
   PvInformer:                     s.InformerFactory.Core().V1().PersistentVolumes(),
   PvcInformer:                    s.InformerFactory.Core().V1().PersistentVolumeClaims(),
   ReplicationControllerInformer:  s.InformerFactory.Core().V1().ReplicationControllers(),
   ReplicaSetInformer:             s.InformerFactory.Apps().V1().ReplicaSets(),
   StatefulSetInformer:            s.InformerFactory.Apps().V1().StatefulSets(),
   ServiceInformer:                s.InformerFactory.Core().V1().Services(),
   PdbInformer:                    s.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(),
   StorageClassInformer:           storageClassInformer,
   HardPodAffinitySymmetricWeight: s.ComponentConfig.HardPodAffinitySymmetricWeight,
   EnableEquivalenceClassCache:    utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
   DisablePreemption:              s.ComponentConfig.DisablePreemption,
   PercentageOfNodesToScore:       s.ComponentConfig.PercentageOfNodesToScore,
   BindTimeoutSeconds:             *s.ComponentConfig.BindTimeoutSeconds,
})
  ---   
return config, nil

factory.NewConfigFactory 便是注册监听能引发调度的所有事件，后面再详细分析调度过程中这些informal事件具体做了什么，这也是解决镜像升级的pod调度时资源不被抢占的关键入口点。在此需要注意的是PodInformer 单独从 "k8s.io/client-go/informers" 分离出来的，因为它只关注引起调度条件变化的特定事件, 如正处于调度中pending的，已调度成功或失败的。接下追寻到leader选举后正式执行调度的函数：pkg/scheduler/scheduler.go sched.Run()-> sched.scheduleOne()

func (sched *Scheduler) Run() {
   if !sched.config.WaitForCacheSync() {
      return
   }

   go wait.Until(sched.scheduleOne, 0, sched.config.StopEverything)
}
// scheduleOne does the entire scheduling workflow for a single pod.  It is serialized on the scheduling algorithm's host fitting.
func (sched *Scheduler) scheduleOne() {
   pod := sched.config.NextPod()
   if pod.DeletionTimestamp != nil {
      sched.config.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", "skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name)
      glog.V(3).Infof("Skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name)
      return
   }

   glog.V(3).Infof("Attempting to schedule pod: %v/%v", pod.Namespace, pod.Name)

   // Synchronously attempt to find a fit for the pod.
   start := time.Now()
   suggestedHost, err := sched.schedule(pod)
   if err != nil {
      // schedule() may have failed because the pod would not fit on any host, so we try to
      // preempt, with the expectation that the next time the pod is tried for scheduling it
      // will fit due to the preemption. It is also possible that a different pod will schedule
      // into the resources that were preempted, but this is harmless.
      if fitError, ok := err.(*core.FitError); ok {
         preemptionStartTime := time.Now()
         sched.preempt(pod, fitError)
         metrics.PreemptionAttempts.Inc()
         metrics.SchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime))
         metrics.SchedulingLatency.WithLabelValues(metrics.PreemptionEvaluation).Observe(metrics.SinceInSeconds(preemptionStartTime))
      }
      return
   }
   metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
   // Tell the cache to assume that a pod now is running on a given node, even though it hasn't been bound yet.
   // This allows us to keep scheduling without waiting on binding to occur.
   assumedPod := pod.DeepCopy()

   // Assume volumes first before assuming the pod.
   //
   // If all volumes are completely bound, then allBound is true and binding will be skipped.
   //
   // Otherwise, binding of volumes is started after the pod is assumed, but before pod binding.
   //
   // This function modifies 'assumedPod' if volume binding is required.
   allBound, err := sched.assumeVolumes(assumedPod, suggestedHost)
   if err != nil {
      return
   }

   // assume modifies `assumedPod` by setting NodeName=suggestedHost
   err = sched.assume(assumedPod, suggestedHost)
   if err != nil {
      return
   }
   // bind the pod to its host asynchronously (we can do this b/c of the assumption step above).
   go func() {
      // Bind volumes first before Pod
      if !allBound {
         err = sched.bindVolumes(assumedPod)
         if err != nil {
            return
         }
      }

      err := sched.bind(assumedPod, &v1.Binding{
         ObjectMeta: metav1.ObjectMeta{Namespace: assumedPod.Namespace, Name: assumedPod.Name, UID: assumedPod.UID},
         Target: v1.ObjectReference{
            Kind: "Node",
            Name: suggestedHost,
         },
      })
      metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
      if err != nil {
         glog.Errorf("Internal error binding pod: (%v)", err)
      }
   }()
}

scheduleOne函数是一次从调度队列取出一个pod进行调度，主要分两部分：正常打分调度和抢占调度。打分调度suggestedHost, err := sched.schedule(pod)如果没有选出合适的node，如果使能了抢占调度则进行优先级抢占调度流程sched.preempt(pod, fitError)。通过正常调度或强占调度获取最终node后，进行assume假定绑定以及实际异步bind绑定操作以提升调度效率。接下着重分析两者调度的核心算法。

1.2 打分调度算法

打分调度的核心逻辑位于pkg/scheduler/core/generic_scheduler.go


// Schedule tries to schedule the given pod to one of the nodes in the node list.
// If it succeeds, it will return the name of the node.
// If it fails, it will return a FitError error with reasons.
func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister) (string, error) {
   ---
   filteredNodes, failedPredicateMap, err := findNodesThatFit(pod, g.cachedNodeInfoMap, nodes, g.predicates, g.extenders, g.predicateMetaProducer, g.equivalenceCache, g.schedulingQueue, g.alwaysCheckAllPredicates)
   if err != nil {
      return "", err
   }

   if len(filteredNodes) == 0 {
      return "", &FitError{
         Pod:              pod,
         NumAllNodes:      len(nodes),
         FailedPredicates: failedPredicateMap,
      }
   }
   metrics.SchedulingAlgorithmPredicateEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPredicateEvalTime))

   trace.Step("Prioritizing")
   startPriorityEvalTime := time.Now()
   // When only one node after predicate, just use it.
   if len(filteredNodes) == 1 {
      metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime))
      return filteredNodes[0].Name, nil
   }

   metaPrioritiesInterface := g.priorityMetaProducer(pod, g.cachedNodeInfoMap)
   priorityList, err := PrioritizeNodes(pod, g.cachedNodeInfoMap, metaPrioritiesInterface, g.prioritizers, filteredNodes, g.extenders)
   if err != nil {
      return "", err
   }
      metrics.SchedulingAlgorithmPriorityEvaluationDuration.Observe(metrics.SinceInMicroseconds(startPriorityEvalTime))

trace.Step("Selecting host")
return g.selectHost(priorityList)

分为两步：预选findNodesThatFit和优选PrioritizeNodes。在通过预选后 1 如果没找到任何合适的node节点，则直接返回错误 2 如果只有一个nod 可以选择，则直接返回该节点名字 3 如果有多个node可以选择，则进行优选打分，选出得分最高的node进行调度

预选逻辑在findNodesThatFit函数实现。

// Filters the nodes to find the ones that fit based on the given predicate functions
// Each node is passed through the predicate functions to determine if it is a fit
func findNodesThatFit(
   pod *v1.Pod,
   nodeNameToInfo map[string]*schedulercache.NodeInfo,
   nodes []*v1.Node,
   predicateFuncs map[string]algorithm.FitPredicate,
   extenders []algorithm.SchedulerExtender,
   metadataProducer algorithm.PredicateMetadataProducer,
   ecache *EquivalenceCache,
   schedulingQueue SchedulingQueue,
   alwaysCheckAllPredicates bool,
) ([]*v1.Node, FailedPredicateMap, error) {
  ---
      checkNode := func(i int) {
         nodeName := nodes[i].Name
         fits, failedPredicates, err := podFitsOnNode(
            pod,
            meta,
            nodeNameToInfo[nodeName],
            predicateFuncs,
            ecache,
            schedulingQueue,
            alwaysCheckAllPredicates,
            equivCacheInfo,
         )
         if err != nil {
            predicateResultLock.Lock()
            errs[err.Error()]++
            predicateResultLock.Unlock()
            return
         }
         if fits {
            filtered[atomic.AddInt32(&filteredLen, 1)-1] = nodes[i]
         } else {
            predicateResultLock.Lock()
            failedPredicateMap[nodeName] = failedPredicates
            predicateResultLock.Unlock()
         }
      }
      workqueue.Parallelize(16, len(nodes), checkNode)
      filtered = filtered[:filteredLen]
      if len(errs) > 0 {
         return []*v1.Node{}, FailedPredicateMap{}, errors.CreateAggregateFromMessageCountMap(errs)
      }
   }

   if len(filtered) > 0 && len(extenders) != 0 {
      for _, extender := range extenders {
         if !extender.IsInterested(pod) {
            continue
         }
         filteredList, failedMap, err := extender.Filter(pod, filtered, nodeNameToInfo)
         if err != nil {
            return []*v1.Node{}, FailedPredicateMap{}, err
         }

         for failedNodeName, failedMsg := range failedMap {
            if _, found := failedPredicateMap[failedNodeName]; !found {
               failedPredicateMap[failedNodeName] = []algorithm.PredicateFailureReason{}
            }
            failedPredicateMap[failedNodeName] = append(failedPredicateMap[failedNodeName], predicates.NewFailureReason(failedMsg))
         }
         filtered = filteredList
         if len(filtered) == 0 {
            break
         }
      }
   }
   return filtered, failedPredicateMap, nil
}

预选是通过16个并发协程处理所有node是否符合被调度到的条件，同时分为成功匹配的和匹配失败的。成功匹配的nodes将参与PrioritizeNodes并行优选打分

func PrioritizeNodes(
   pod *v1.Pod,
   nodeNameToInfo map[string]*schedulercache.NodeInfo,
   meta interface{},
   priorityConfigs []algorithm.PriorityConfig,
   nodes []*v1.Node,
   extenders []algorithm.SchedulerExtender,
) (schedulerapi.HostPriorityList, error) {
   // If no priority configs are provided, then the EqualPriority function is applied
   // This is required to generate the priority list in the required format
   if len(priorityConfigs) == 0 && len(extenders) == 0 {
      result := make(schedulerapi.HostPriorityList, 0, len(nodes))
      for i := range nodes {
         hostPriority, err := EqualPriorityMap(pod, meta, nodeNameToInfo[nodes[i].Name])
         if err != nil {
            return nil, err
         }
         result = append(result, hostPriority)
      }
      return result, nil
   }

   var (
      mu   = sync.Mutex{}
      wg   = sync.WaitGroup{}
      errs []error
   )
   appendError := func(err error) {
      mu.Lock()
      defer mu.Unlock()
      errs = append(errs, err)
   }

   results := make([]schedulerapi.HostPriorityList, len(priorityConfigs), len(priorityConfigs))

   for i, priorityConfig := range priorityConfigs {
      if priorityConfig.Function != nil {
         // DEPRECATED
         wg.Add(1)
         go func(index int, config algorithm.PriorityConfig) {
            defer wg.Done()
            var err error
            results[index], err = config.Function(pod, nodeNameToInfo, nodes)
            if err != nil {
               appendError(err)
            }
         }(i, priorityConfig)
      } else {
         results[i] = make(schedulerapi.HostPriorityList, len(nodes))
      }
   }
   processNode := func(index int) {
      nodeInfo := nodeNameToInfo[nodes[index].Name]
      var err error
      for i := range priorityConfigs {
         if priorityConfigs[i].Function != nil {
            continue
         }
         results[i][index], err = priorityConfigs[i].Map(pod, meta, nodeInfo)
         if err != nil {
            appendError(err)
            return
         }
      }
   }
   workqueue.Parallelize(16, len(nodes), processNode)
   for i, priorityConfig := range priorityConfigs {
      if priorityConfig.Reduce == nil {
         continue
      }
      wg.Add(1)
      go func(index int, config algorithm.PriorityConfig) {
         defer wg.Done()
         if err := config.Reduce(pod, meta, nodeNameToInfo, results[index]); err != nil {
            appendError(err)
         }
         if glog.V(10) {
            for _, hostPriority := range results[index] {
               glog.Infof("%v -> %v: %v, Score: (%d)", pod.Name, hostPriority.Host, config.Name, hostPriority.Score)
            }
         }
      }(i, priorityConfig)
   }
   // Wait for all computations to be finished.
   wg.Wait()
   if len(errs) != 0 {
      return schedulerapi.HostPriorityList{}, errors.NewAggregate(errs)
   }

   // Summarize all scores.
   result := make(schedulerapi.HostPriorityList, 0, len(nodes))

   for i := range nodes {
      result = append(result, schedulerapi.HostPriority{Host: nodes[i].Name, Score: 0})
      for j := range priorityConfigs {
         result[i].Score += results[j][i].Score * priorityConfigs[j].Weight
      }
   }

   if len(extenders) != 0 && nodes != nil {
      combinedScores := make(map[string]int, len(nodeNameToInfo))
      for _, extender := range extenders {
         if !extender.IsInterested(pod) {
            continue
         }
         wg.Add(1)
         go func(ext algorithm.SchedulerExtender) {
            defer wg.Done()
            prioritizedList, weight, err := ext.Prioritize(pod, nodes)
            if err != nil {
               // Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities
               return
            }
            mu.Lock()
            for i := range *prioritizedList {
               host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score
               combinedScores[host] += score * weight
            }
            mu.Unlock()
         }(extender)
      }
      // wait for all go routines to finish
      wg.Wait()
      for i := range result {
         result[i].Score += combinedScores[result[i].Host]
      }
   }

   if glog.V(10) {
      for i := range result {
         glog.V(10).Infof("Host %s => Score %d", result[i].Host, result[i].Score)
      }
   }
   return result, nil
}

PrioritizeNodes通过并行运行各个优先级函数来对每个node节点进行优先级排序。如果没有提供优选函数和拓展函数，则将所有的节点设置为相同的优先级，即节点的score都为1，然后直接返回结果,如果提供了优选函数或拓展函数，则进行以下打分流程。 1 每个优先级函数会给节点打分，打分范围为0-10分。 2 0表示优先级最低的节点，10表示优先级最高的节点。 3 每个优先级函数也有各自的权重。 4 优先级函数返回的节点分数乘以权重以获得加权分数。 5 最后组合（添加）所有分数以获得所有节点的总加权分数如果设置了扩展函数，则增加扩展函数的打分项加权求和，这里的打分细节和权重的设置策略不是本文要解决问题的重点，不做详细分析。最后在g.selectHost(priorityList)函数中选出已打分的各node节点中分数最高的节点作为最终被选中调度的node。在此继续分析预选过程中对每个node 调用podFitsOnNode具体做了什么。

func podFitsOnNode(
   pod *v1.Pod,
   meta algorithm.PredicateMetadata,
   info *schedulercache.NodeInfo,
   predicateFuncs map[string]algorithm.FitPredicate,
   ecache *EquivalenceCache,
   queue SchedulingQueue,
   alwaysCheckAllPredicates bool,
   equivCacheInfo *equivalenceClassInfo,
) (bool, []algorithm.PredicateFailureReason, error) {
   var (
      eCacheAvailable  bool
      failedPredicates []algorithm.PredicateFailureReason
   )
   predicateResults := make(map[string]HostPredicate)

   podsAdded := false
   ---
   for i := 0; i < 2; i++ {
      metaToUse := meta
      nodeInfoToUse := info
      if i == 0 {
         podsAdded, metaToUse, nodeInfoToUse = addNominatedPods(util.GetPodPriority(pod), meta, info, queue)
      } else if !podsAdded || len(failedPredicates) != 0 {
         break
      }
      // Bypass eCache if node has any nominated pods.
      // TODO(bsalamat): consider using eCache and adding proper eCache invalidations
      // when pods are nominated or their nominations change.
      eCacheAvailable = equivCacheInfo != nil && !podsAdded
      for _, predicateKey := range predicates.Ordering() {
         var (
            fit     bool
            reasons []algorithm.PredicateFailureReason
            err     error
         )
         //TODO (yastij) : compute average predicate restrictiveness to export it as Prometheus metric
         if predicate, exist := predicateFuncs[predicateKey]; exist {
            // Use an in-line function to guarantee invocation of ecache.Unlock()
            // when the in-line function returns.
            func() {
               var invalid bool
               if eCacheAvailable {
                  // Lock ecache here to avoid a race condition against cache invalidation invoked
                  // in event handlers. This race has existed despite locks in equivClassCacheimplementation.
                  ecache.Lock()
                  defer ecache.Unlock()
                  // PredicateWithECache will return its cached predicate results.
                  fit, reasons, invalid = ecache.PredicateWithECache(
                     pod.GetName(), info.Node().GetName(),
                     predicateKey, equivCacheInfo.hash, false)
               }

               if !eCacheAvailable || invalid {
                  // we need to execute predicate functions since equivalence cache does not work
                  fit, reasons, err = predicate(pod, metaToUse, nodeInfoToUse)
                  if err != nil {
                     return
                  }

                  if eCacheAvailable {
                     // Store data to update equivClassCacheafter this loop.
                     if res, exists := predicateResults[predicateKey]; exists {
                        res.Fit = res.Fit && fit
                        res.FailReasons = append(res.FailReasons, reasons...)
                        predicateResults[predicateKey] = res
                     } else {
                        predicateResults[predicateKey] = HostPredicate{Fit: fit, FailReasons: reasons}
                     }
                     result := predicateResults[predicateKey]
                     ecache.UpdateCachedPredicateItem(
                        pod.GetName(), info.Node().GetName(),
                        predicateKey, result.Fit, result.FailReasons, equivCacheInfo.hash, false)
                  }
               }
            }()

            if err != nil {
               return false, []algorithm.PredicateFailureReason{}, err
            }

            if !fit {
               // eCache is available and valid, and predicates result is unfit, record the fail reasons
               failedPredicates = append(failedPredicates, reasons...)
               // if alwaysCheckAllPredicates is false, short circuit all predicates when one predicate fails.
               if !alwaysCheckAllPredicates {
                  glog.V(5).Infoln("since alwaysCheckAllPredicates has not been set, the predicate" +
                     "evaluation is short circuited and there are chances" +
                     "of other predicates failing as well.")
                  break
               }
            }
         }
      }
   }

   return len(failedPredicates) == 0, failedPredicates, nil
}

预选podFitsOnNode函数里循环执行了两次。第一次为考虑优先级大于或等于当前需要调度的pod且已被提名调度到所要检测的当前node的pods，即需扣除已经被提名调度到此node的所有pods的cpu 、mem，第二次为不考虑被提名调度到此node的pods情况，因为被提名调度到此node的pods最后并不一定真正调度到此node，因此有可能因为pod间的亲和性问题而不能通过调度到此节点。被提名的nominatedPods是从何而来的呢，实际是执行了强占调度算法被分到预定node的pods，接下具体分析强占调度算法。

1.3 抢占调度算法

在第一步进行正常的打分调度后若没有选出合适的node进行调度，同时开启了抢占调度功能(1.9版本后默认开启),则在scheduleOne执行pkg/scheduler/scheduler.go sched.preempt(pod, fitError)

// preempt tries to create room for a pod that has failed to schedule, by preempting lower priority pods if possible.
// If it succeeds, it adds the name of the node where preemption has happened to the pod annotations.
// It returns the node name and an error if any.
func (sched *Scheduler) preempt(preemptor *v1.Pod, scheduleErr error) (string, error) {
   if !util.PodPriorityEnabled() || sched.config.DisablePreemption {
      glog.V(3).Infof("Pod priority feature is not enabled or preemption is disabled by scheduler configuration." +
         " No preemption is performed.")
      return "", nil
   }
   preemptor, err := sched.config.PodPreemptor.GetUpdatedPod(preemptor)
   if err != nil {
      glog.Errorf("Error getting the updated preemptor pod object: %v", err)
      return "", err
   }

   node, victims, nominatedPodsToClear, err := sched.config.Algorithm.Preempt(preemptor, sched.config.NodeLister, scheduleErr)
   metrics.PreemptionVictims.Set(float64(len(victims)))
   if err != nil {
      glog.Errorf("Error preempting victims to make room for %v/%v.", preemptor.Namespace, preemptor.Name)
      return "", err
   }
   var nodeName = ""
   if node != nil {
      nodeName = node.Name
      err = sched.config.PodPreemptor.SetNominatedNodeName(preemptor, nodeName)
      if err != nil {
         glog.Errorf("Error in preemption process. Cannot update pod %v/%v annotations: %v", preemptor.Namespace, preemptor.Name, err)
         return "", err
      }
      for _, victim := range victims {
         if err := sched.config.PodPreemptor.DeletePod(victim); err != nil {
            glog.Errorf("Error preempting pod %v/%v: %v", victim.Namespace, victim.Name, err)
            return "", err
         }
         sched.config.Recorder.Eventf(victim, v1.EventTypeNormal, "Preempted", "by %v/%v on node %v", preemptor.Namespace, preemptor.Name, nodeName)
      }
   }
   // Clearing nominated pods should happen outside of "if node != nil". Node could
   // be nil when a pod with nominated node name is eligible to preempt again,
   // but preemption logic does not find any node for it. In that case Preempt()
   // function of generic_scheduler.go returns the pod itself for removal of the annotation.
   for _, p := range nominatedPodsToClear {
      rErr := sched.config.PodPreemptor.RemoveNominatedNodeName(p)
      if rErr != nil {
         glog.Errorf("Cannot remove nominated node annotation of pod: %v", rErr)
         // We do not return as this error is not critical.
      }
   }
   return nodeName, err
}

• 调用 sched.Algorithm.Preempt()执行抢占逻辑，该函数会返回抢占成功的 node、被抢占的 pods(victims) 以及需要被移除已提名的 pods • 更新 scheduler 缓存，为抢占者绑定 nodeName，即设定 pod.Status.NominatedNodeName • 将 pod info 提交到 apiserver • 删除被抢占的 pods • 删除被抢占 pods 的 NominatedNodeName 字段 1.2节中预选过程中第一次需考虑nominatedName Pods的来源就在这，其他pod调度时会通过addNominatedPods扣除被提名的node上pods所需的资源，以免占用。pod最终不一定会调度到提名的node节点，SetNominatedNodeName 方法中会执行update pod status操作，此后会触发1.1节中factory中pod update事件，重新加入调度队列进行再次一样的逻辑调度，因此有可能此时其他node已满足更优的调度条件而被调度到其他node上。接下具体分析抢占调度的逻辑，其实现也在pkg/scheduler/core/generic.go 中

func (g *genericScheduler) Preempt(pod *v1.Pod, nodeLister algorithm.NodeLister, scheduleErr error) (*v1.Node, []*v1.Pod, []*v1.Pod, error) {
  fitError, ok := scheduleErr.(*FitError)
if !ok || fitError == nil {
   return nil, nil, nil, nil
}
err := g.cache.UpdateNodeNameToInfoMap(g.cachedNodeInfoMap)
if err != nil {
   return nil, nil, nil, err
}
if !podEligibleToPreemptOthers(pod, g.cachedNodeInfoMap) {
   glog.V(5).Infof("Pod %v is not eligible for more preemption.", pod.Name)
   return nil, nil, nil, nil
}
allNodes, err := nodeLister.List()
if err != nil {
   return nil, nil, nil, err
}
if len(allNodes) == 0 {
   return nil, nil, nil, ErrNoNodesAvailable
}
potentialNodes := nodesWherePreemptionMightHelp(pod, allNodes, fitError.FailedPredicates)
if len(potentialNodes) == 0 {
   glog.V(3).Infof("Preemption will not help schedule pod %v on any node.", pod.Name)
   // In this case, we should clean-up any existing nominated node name of the pod.
   return nil, nil, []*v1.Pod{pod}, nil
}
pdbs, err := g.cache.ListPDBs(labels.Everything())
if err != nil {
   return nil, nil, nil, err
}
nodeToVictims, err := selectNodesForPreemption(pod, g.cachedNodeInfoMap, potentialNodes, g.predicates, g.predicateMetaProducer, g.schedulingQueue, pdbs)
if err != nil {
   return nil, nil, nil, err
}
for len(nodeToVictims) > 0 {
   node := pickOneNodeForPreemption(nodeToVictims)
   if node == nil {
      return nil, nil, nil, err
   }
   passes, pErr := nodePassesExtendersForPreemption(pod, node.Name, nodeToVictims[node].pods, g.cachedNodeInfoMap, g.extenders)
   if passes && pErr == nil {
      // Lower priority pods nominated to run on this node, may no longer fit on
      // this node. So, we should remove their nomination. Removing their
      // nomination updates these pods and moves them to the active queue. It
      // lets scheduler find another place for them.
      nominatedPods := g.getLowerPriorityNominatedPods(pod, node.Name)
      return node, nodeToVictims[node].pods, nominatedPods, err
   }
   if pErr != nil {
      glog.Errorf("Error occurred while checking extenders for preemption on node %v: %v", node, pErr)
   }
   // Remove the node from the map and try to pick a different node.
   delete(nodeToVictims, node)
}
return nil, nil, nil, err

• 判断 err 是否为 FitError，只有此err 才可执行抢占调度 • 调用podEligibleToPreemptOthers()确认 pod 是否有抢占其他 pod 的资格，若 pod 已经抢占了低优先级的 pod，被抢占的 pod 处于 terminating 状态中，则不会继续进行抢占 • 如果确定可以进行抢占，调度器会把自己缓存的所有节点信息复制一份，然后使用这个副本来模拟抢占过程 • nodesWherePreemptionMightHelp过滤预选失败的 node 列表，此处会检查 predicates 失败的原因，若存在 NodeSelectorNotMatch、PodNotMatchHostName 等这些 error 则不能成为抢占者，如果过滤出的候选 node 为空则返回 • 获取 PodDisruptionBudget 对象（PDB）,中断预算是为保证最小高可用的pod数 • selectNodesForPreemption()从预选失败的 node 列表中并发计算可以被抢占的 nodes，得到 nodeToVictims • 若声明了 extenders 则调用 extenders 再次过滤 nodeToVictims • 调用 pickOneNodeForPreemption() 从 nodeToVictims 中选出一个节点作为最佳候选者，选择的标准是: 1. 最少的 PDB violations,如果只有一个node符合，则直接返回 2. 最少的高优先级 victim，如果只有一个node符合，则直接返回 3. 优先级总数字最小，如果只有一个node符合，则直接返回 4. victim 总数最小，返回第一个 • 移除选出node上低优先级 pods 的 Nominated释放原被提名时占用的资源 • 给当前执行抢占调度的pod的打上提名node，移动到 activeQ 队列中，让调度器为重新再次调度bind node

1.4 优先级调度队列

前面1.2提及每次从调度队列中取出一个pod，那么这个调度队列如何进行CRUD操作及排序的呢。调度队列SchedulingQueue的操作位于pkg/scheduler/core/scheduling.go。可以看到有两种队列 FIFO和PriorityQueue.前者无优先级以先来先到方式调度，后者是具有优先级排序的调度。优先级队列PriorityQueue是通过优先级高到低的大顶堆实现的，具体逻辑分析可以参考garnettwang 的https://cloud.tencent.com/developer/article/1121557。在此提及下堆排序时用到的比较方法：优先级高的在前面，若相等则按pod的状态转换时间或创建时间先后顺序排序。

func activeQComp(pod1, pod2 interface{}) bool {
   p1 := pod1.(*v1.Pod)
   p2 := pod2.(*v1.Pod)
   prio1 := util.GetPodPriority(p1)
   prio2 := util.GetPodPriority(p2)
   return (prio1 > prio2) || prio1 == prio2 && podTimestamp(p1).Before(podTimestamp(p2))
}

2 statefulset镜像升级资源不足分析及解决方案

本文要解决的问题：当k8s集群已经存在cpu/mem资源不足且有pod处于调度过程中的pending状态，此时statefulset/statefulsetplus 进行镜像更新时的pod同样发生资源不足处于pending状态，在不新增资源的情况下能够进行正常升级。由于deployment在升级过程中会新建pod再删除旧pod以及pod的名称会变更，本文暂时解决不了在不新增资源的情况下进行Deployment升级时资源不足的问题。 Statefulset镜像更新时pod是删除重建的过程中，不占用新资源为什么也会出现资源不足的pending状态错误呢? 这就需要理清楚在镜像更新pod删除重建过程中，调度器kube-scheduler发生了什么。

2.1 Pod 事件触发的调度过程

1.1 节中已提及pkg/schduler/factory/factory.go 中factory.NewConfigFactory注册了可以引起schedulerqueue重新调度的CRUD事件，我们只重点关注pod事件会引起怎样的调度变化。 Pod事件监听分为两类：1 已经调度了并且running状态的pod，此部分pod信息存在于schedulerCache； 2 未被分配的以及非中止状态的pod

// scheduled pod cache   
 args.PodInformer.Informer().AddEventHandler(
   cache.FilteringResourceEventHandler{
      FilterFunc: func(obj interface{}) bool {
         switch t := obj.(type) {
         case *v1.Pod:
            return assignedNonTerminatedPod(t)
         case cache.DeletedFinalStateUnknown:
            if pod, ok := t.Obj.(*v1.Pod); ok {
               return assignedNonTerminatedPod(pod)
            }
            runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c))
            return false
         default:
            runtime.HandleError(fmt.Errorf("unable to handle object in %T: %T", c, obj))
            return false
         }
      },
      Handler: cache.ResourceEventHandlerFuncs{
         AddFunc:    c.addPodToCache,
         UpdateFunc: c.updatePodInCache,
         DeleteFunc: c.deletePodFromCache,
      },
   },
))

每次有新pod完成调度后会触发将其加入schedulerCache，以及将原本未能成功调度的队列且与此pod具有亲和性匹配的pods重新进行调度。

func (c *configFactory) addPodToCache(obj interface{}) {
   pod, ok := obj.(*v1.Pod)
   if !ok {
      glog.Errorf("cannot convert to *v1.Pod: %v", obj)
      return
   }

   if err := c.schedulerCache.AddPod(pod); err != nil {
      glog.Errorf("scheduler cache AddPod failed: %v", err)
   }

   c.podQueue.AssignedPodAdded(pod)

   // NOTE: Updating equivalence cache of addPodToCache has been
   // handled optimistically in: pkg/scheduler/scheduler.go#assume()
}
}

每次已调度的pod发生更新时，触发schedulerCache中删除旧pod增加新pod的方式进行更新，同样触发上次未调度成功的pod进行新一轮调度。

func (c *configFactory) updatePodInCache(oldObj, newObj interface{}) {
   oldPod, ok := oldObj.(*v1.Pod)
   if !ok {
      glog.Errorf("cannot convert oldObj to *v1.Pod: %v", oldObj)
      return
   }
   newPod, ok := newObj.(*v1.Pod)
   if !ok {
      glog.Errorf("cannot convert newObj to *v1.Pod: %v", newObj)
      return
   }

   // NOTE: Because the scheduler uses snapshots of schedulerCache and the live
   // version of equivalencePodCache, updates must be written to schedulerCache
   // before invalidating equivalencePodCache.
   if err := c.schedulerCache.UpdatePod(oldPod, newPod); err != nil {
      glog.Errorf("scheduler cache UpdatePod failed: %v", err)
   }

   c.invalidateCachedPredicatesOnUpdatePod(newPod, oldPod)
   c.podQueue.AssignedPodUpdated(newPod)

当已调度的pod发生delete事件时，会触发以下操作：

func (c *configFactory) deletePodFromCache(obj interface{}) {
   var pod *v1.Pod
   switch t := obj.(type) {
   case *v1.Pod:
      pod = t
   case cache.DeletedFinalStateUnknown:
      var ok bool
      pod, ok = t.Obj.(*v1.Pod)
      if !ok {
         glog.Errorf("cannot convert to *v1.Pod: %v", t.Obj)
         return
      }
   default:
      glog.Errorf("cannot convert to *v1.Pod: %v", t)
      return
   }
   // NOTE: Because the scheduler uses snapshots of schedulerCache and the live
   // version of equivalencePodCache, updates must be written to schedulerCache
   // before invalidating equivalencePodCache.
   if err := c.schedulerCache.RemovePod(pod); err != nil {
      glog.Errorf("scheduler cache RemovePod failed: %v", err)
   }

   c.invalidateCachedPredicatesOnDeletePod(pod)
   c.podQueue.MoveAllToActiveQueue()
}

1 将pod从schedulerCache中删除，此删除操作会将其在对应node中所占的request cpu/mem、port资源扣除释放

2 调用invalidateCachedPredicatesOnDeletePod更新Equivalence Cache。Equivalence Cache是为加速Kubernetes Scheduler预选决策设计的缓存策略，scheduler及时维护着Equivalence Cache的数据，当某些情况发生时（比如delete node、bind pod、delete pod等事件），需要立刻invalid相关的Equivalence Cache中的缓存数据。具体工作机制参考garnettwang 的https://cloud.tencent.com/developer/article/1150566。

3 然后调用MoveAllToActiveQueue将所有原先未能成功调度的pods加入activeQueue进行下一轮调度，这就是statefulset镜像升级时pod发生资源不足的关键问题所在，因为在删除重建过程中，原先pod释放的资源被正处于pending的待调度的pods得到此资源占用了，那么升级时新建的pod自然就发生资源不足的pending问题首次进行调度或未被成功调度的pod的事件是直接触发scheduler queue的增删改操作，逻辑不难就不详细赘述了。

// unscheduled pod queue
args.PodInformer.Informer().AddEventHandler(
   cache.FilteringResourceEventHandler{
      FilterFunc: func(obj interface{}) bool {
         switch t := obj.(type) {
         case *v1.Pod:
            return unassignedNonTerminatedPod(t) && responsibleForPod(t, args.SchedulerName)
         case cache.DeletedFinalStateUnknown:
            if pod, ok := t.Obj.(*v1.Pod); ok {
               return unassignedNonTerminatedPod(pod) && responsibleForPod(pod, args.SchedulerName)
            }
            runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c))
            return false
         default:
            runtime.HandleError(fmt.Errorf("unable to handle object in %T: %T", c, obj))
            return false
         }
      },
      Handler: cache.ResourceEventHandlerFuncs{
         AddFunc:    c.addPodToSchedulingQueue,
         UpdateFunc: c.updatePodInSchedulingQueue,
         DeleteFunc: c.deletePodFromSchedulingQueue,
      },
   },
)

有个疑惑就是调度失败的pod是在哪加入PriorityQueue 中的unschedulableQ呢?答案在pkg/scheduler/scheduler.go 的Config配置中 Error方法，它实际包含了很重要的pod调度失败时的处理逻辑，包括失败重试和加入unschedulableQ，具体实现在pkg/scheduler/factory/factory.go->func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue core.SchedulingQueue) func(pod *v1.Pod, err error) 中。至此我们已经分析出集群资源不足且已有pod因资源不足而处于未成功调度的pending 中，此时statefulset镜像更新时的pod也会发生资源不足的pending错误的原因，接下修改调度策略以解决此问题。

2.2 镜像升级调度过程的策略改进

2.1中已分析statefulset中处于镜像升级的pod为什么会发生资源不足的原因，另外在pod删除重建的过程中有个关键信号就是，一个pod将被删除，它先会被标记pod.DeletionTimestamp 时间戳，此时触发updatePodInCache事件，之后再是deletePodFromCache 及重新将之前调度失败的pods移到activeQueue进行调度，原先pod删除后，新pod根据新镜像的template创建，此时会触发addPodToSchedulingQueue事件进行调度。为解决此问题。拟有两种方案： 1 deletePodFromCache事件处理中如果检测到此pod删除是为了升级，释放资源后不引发未调度成功的pod进行调度，并把升级的pod的排序优先级高于非升级的； 2 deletePodFromCache事件处理中如果检测到此pod删除是为了升级，不释放资源，并且在其他pod调度预选时认为这块资源是被占用的，而对应升级的新pod预选时又认为是可用的。方案1中只能做到尽量满足升级的pod获取到资源，因为从pod删除到新pod创建有时间差，如果此时activeQueue正有pod在调度，那么此资源可能就会被占用。二者如果还有别的pod也在更新，那么可能会pod相互调度到别的node引起资源碎片导致资源不足。本文采用方案2，那怎么实现呢。其思路是借鉴1.3中抢占算法的nominatedPod方式，在deletePodFromCache方法中如果检测到此pod删除是为升级，则将此pod标记pod.Status.NominatedNodeName = pod.Spec.NodeName，并调用SchedulingQueue 中AddNominatedPodIfNeeded 方法将其添加到PriorityQueue的nominatedPods中。因此，其他pod调度时，需扣除此node上升级的pod的资源，而在对应升级新建的pod看来这部分资源是可用的，同时和抢占调度算法一样，其他node满足更优的调度条件时，可以调度到其他节点。这里需要修改generic_scheduler.go 中 addNominatedPods逻辑

for _, p := range nominatedPods {
   if util.GetPodPriority(p) >= util.GetPodPriority(pod) && p.UID != pod.UID {
      nodeInfoOut.AddPod(p)
      if metaOut != nil {
         metaOut.AddPod(p, nodeInfoOut)
      }
   }
}

更新镜像时新pod匹配原来的旧pod时，是不需要扣除node上本属于它自己的资源。

for _, p := range nominatedPods {
   if util.GetPodPriority(p) >= util.GetPodPriority(pod) && (p.UID != pod.UID) && !(p.DeletionTimestamp != nil && reflect.DeepEqual(p.OwnerReferences, pod.OwnerReferences) && p.Namespace == pod.Namespace && p.Name == pod.Name) {
      nodeInfoOut.AddPod(p)
      if metaOut != nil {
         metaOut.AddPod(p, nodeInfoOut)
      }
   }
}

什么时候delete nominatedPod呢，和抢占调度时一样的时机删除，即新pod完成调度选举了node进行assume时。

// assume signals to the cache that a pod is already in the cache, so that binding can be asynchronous.
// assume modifies `assumed`.
func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
   ---  
 // if "assumed" is a nominated pod, we should remove it from internal cache
   if sched.config.SchedulingQueue != nil {
      sched.config.SchedulingQueue.DeleteNominatedPodIfExists(assumed)
   }
   ---
   return nil
}