openkruise如何给pod自动添加sidecarSet中设置的内容，sidecarSet中设置的升级策略如何生效，本文将工作的流程进行了一个梳理，以及部分源码的解读。
源码讲解：
1，通过openkruise中podwebhook增加匹配到的sidecarset中的信息，同时给对应的pod添加注解、label 以及注入的环境变量 容器 init容器等；
openkruise/kruise/pkg/webhook/pod/mutating/pod_create_update_handler.go # Handle
// Handle handles admission requests.
func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) admission.Response {
...
// 自动注入readinessGate
if skip := injectPodReadinessGate(req, obj); !skip {
changed = true
}
if utilfeature.DefaultFeatureGate.Enabled(features.WorkloadSpread) {
if skip, err := h.workloadSpreadMutatingPod(ctx, req, obj); err != nil {
return admission.Errored(http.StatusInternalServerError, err)
} else if !skip {
changed = true
}
}
// 匹配是sidecarSet,如果有匹配上的则给pod自动注入注解 或者 label
if skip, err := h.sidecarsetMutatingPod(ctx, req, obj); err != nil {
return admission.Errored(http.StatusInternalServerError, err)
} else if !skip {
changed = true
}
}
openkruise/kruise/pkg/webhook/pod/mutating/sidecarset.go # sidecarsetMutatingPod
// mutate pod based on SidecarSet Object
func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admission.Request, pod *corev1.Pod) (skip bool, err error) {
if len(req.AdmissionRequest.SubResource) > 0 ||
(req.AdmissionRequest.Operation != admissionv1.Create && req.AdmissionRequest.Operation != admissionv1.Update) ||
req.AdmissionRequest.Resource.Resource != "pods" {
return true, nil
}
// filter out pods that don't require inject
if !sidecarcontrol.IsActivePod(pod) {
return true, nil
}
var oldPod *corev1.Pod
var isUpdated bool
//when Operation is update, decode older object
if req.AdmissionRequest.Operation == admissionv1.Update {
isUpdated = true
oldPod = new(corev1.Pod)
if err = h.Decoder.Decode(
admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Object: req.AdmissionRequest.OldObject}},
oldPod); err != nil {
return false, err
}
}
// DisableDeepCopy:true, indicates must be deep copy before update sidecarSet objection
sidecarsetList := &appsv1alpha1.SidecarSetList{}
if err = h.Client.List(ctx, sidecarsetList, utilclient.DisableDeepCopy); err != nil {
return false, err
}
matchedSidecarSets := make([]sidecarcontrol.SidecarControl, 0)
for _, sidecarSet := range sidecarsetList.Items {
if sidecarSet.Spec.InjectionStrategy.Paused {
continue
}
// 匹配可以适用在此pod上的sidecarSet
if matched, err := sidecarcontrol.PodMatchedSidecarSet(h.Client, pod, &sidecarSet); err != nil {
return false, err
} else if !matched {
continue
}
// get user-specific revision or the latest revision of SidecarSet
suitableSidecarSet, err := h.getSuitableRevisionSidecarSet(&sidecarSet, oldPod, pod, req.AdmissionRequest.Operation)
if err != nil {
return false, err
}
// check whether sidecarSet is active
// when sidecarSet is not active, it will not perform injections and upgrades process.
control := sidecarcontrol.New(suitableSidecarSet)
if !control.IsActiveSidecarSet() {
continue
}
matchedSidecarSets = append(matchedSidecarSets, control)
}
if len(matchedSidecarSets) == 0 {
return true, nil
}
// check pod
if isUpdated {
if !matchedSidecarSets[0].IsPodAvailabilityChanged(pod, oldPod) {
klog.V(3).Infof("pod(%s/%s) availability unchanged for sidecarSet, and ignore", pod.Namespace, pod.Name)
return true, nil
}
}
klog.V(3).Infof("[sidecar inject] begin to operation(%s) pod(%s/%s) resources(%s) subResources(%s)",
req.Operation, req.Namespace, req.Name, req.Resource, req.SubResource)
// patch pod metadata, annotations & labels
// When the Pod main container is upgraded in place, and the sidecarSet configuration does not change at this time,
// at this point, it can also patch pod metadata
if pod.Annotations == nil {
pod.Annotations = make(map[string]string)
}
skip = true
for _, control := range matchedSidecarSets {
sidecarSet := control.GetSidecarset()
sk, err := sidecarcontrol.PatchPodMetadata(&pod.ObjectMeta, sidecarSet.Spec.PatchPodMetadata)
if err != nil {
klog.Errorf("sidecarSet(%s) update pod(%s/%s) metadata failed: %s", sidecarSet.Name, pod.Namespace, pod.Name, err.Error())
return false, err
} else if !sk {
// skip = false
skip = false
}
}
//++++build sidecar containers, sidecar initContainers, sidecar volumes, annotations to inject into pod object++++
sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecar, injectedAnnotations, err := buildSidecars(isUpdated, pod, oldPod, matchedSidecarSets)
if err != nil {
return false, err
} else if len(sidecarContainers) == 0 && len(sidecarInitContainers) == 0 {
klog.V(3).Infof("[sidecar inject] pod(%s/%s) don't have injected containers", pod.Namespace, pod.Name)
return skip, nil
}
klog.V(3).Infof("[sidecar inject] begin inject sidecarContainers(%v) sidecarInitContainers(%v) sidecarSecrets(%v), volumes(%s)"+
"annotations(%v) into pod(%s/%s)", sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecar, injectedAnnotations,
pod.Namespace, pod.Name)
klog.V(4).Infof("[sidecar inject] before mutating: %v", util.DumpJSON(pod))
// apply sidecar set info into pod
// 1. inject init containers, sort by their name, after the original init containers
sort.SliceStable(sidecarInitContainers, func(i, j int) bool {
return sidecarInitContainers[i].Name < sidecarInitContainers[j].Name
})
for _, initContainer := range sidecarInitContainers {
pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer.Container)
}
// 2. inject containers
pod.Spec.Containers = mergeSidecarContainers(pod.Spec.Containers, sidecarContainers)
// 3. inject volumes
pod.Spec.Volumes = util.MergeVolumes(pod.Spec.Volumes, volumesInSidecar)
// 4. inject imagePullSecrets
pod.Spec.ImagePullSecrets = mergeSidecarSecrets(pod.Spec.ImagePullSecrets, sidecarSecrets)
// 5. apply annotations
for k, v := range injectedAnnotations {
pod.Annotations[k] = v
}
klog.V(4).Infof("[sidecar inject] after mutating: %v", util.DumpJSON(pod))
return false, nil
}
2，sidecarsset这个自定义的资源中增加了watch pod的事件，当有pod创建的时候，会把此pod匹配到的sidecarset放入到队列中，然
openkruise/kruise/pkg/controller/sidecarset/sidecarset_controller.go # add
func add(mgr manager.Manager, r reconcile.Reconciler) error {
// Create a new controller
c, err := controller.New("sidecarset-controller", mgr, controller.Options{
Reconciler: r, MaxConcurrentReconciles: concurrentReconciles,
RateLimiter: ratelimiter.DefaultControllerRateLimiter()})
if err != nil {
return err
}
// Watch for changes to SidecarSet
err = c.Watch(&source.Kind{Type: &appsv1alpha1.SidecarSet{}}, &handler.EnqueueRequestForObject{}, predicate.Funcs{
UpdateFunc: func(e event.UpdateEvent) bool {
oldScS := e.ObjectOld.(*appsv1alpha1.SidecarSet)
newScS := e.ObjectNew.(*appsv1alpha1.SidecarSet)
if oldScS.GetGeneration() != newScS.GetGeneration() {
klog.V(3).Infof("Observed updated Spec for SidecarSet: %s/%s", newScS.GetNamespace(), newScS.GetName())
return true
}
return false
},
})
if err != nil {
return err
}
// +++++++++++++++Watch for changes to Pod++++++++++++++++ 1
if err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &enqueueRequestForPod{reader: mgr.GetCache()}); err != nil {
return err
}
return nil
}
enqueueRequestForPod 实现了EventHandler中的Create / Update / Delete / Generic 具体在下面的方法中，以update为例：
openkruise/kruise/pkg/controller/sidecarset/sidecarset_pod_event_handler.go # update
当pod有更新事件，会执行下面的逻辑
// 通过sidecarSet增加对pod资源的watch，然后实现eventHandler的方法，通过此方法对新老pod进行对比，如果pod有对应的变化，则把pod中已经使用的sidecarSet添加到队列中
func (p *enqueueRequestForPod) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
p.updatePod(q, evt.ObjectOld, evt.ObjectNew)
}
func (p *enqueueRequestForPod) updatePod(q workqueue.RateLimitingInterface, old, cur runtime.Object) {
newPod := cur.(*corev1.Pod)
oldPod := old.(*corev1.Pod)
if newPod.ResourceVersion == oldPod.ResourceVersion {
return
}
matchedSidecarSets, err := p.getPodMatchedSidecarSets(newPod)
if err != nil {
klog.Errorf("unable to get sidecarSets of pod %s/%s, err: %v", newPod.Namespace, newPod.Name, err)
return
}
for _, sidecarSet := range matchedSidecarSets {
if sidecarSet.Spec.UpdateStrategy.Type == appsv1alpha1.NotUpdateSidecarSetStrategyType {
continue
}
var isChanged bool
var enqueueDelayTime time.Duration
//check whether pod status is changed
if isChanged = isPodStatusChanged(oldPod, newPod); !isChanged {
//check whether pod consistent is changed
isChanged, enqueueDelayTime = isPodConsistentChanged(oldPod, newPod, sidecarSet)
}
// 把sidecarSet放入队列中，对应的程序会从队列中获取，使用sidecarSet对应的Reconcile会从执行对应的操作逻辑
if isChanged {
q.AddAfter(reconcile.Request{
NamespacedName: types.NamespacedName{
Name: sidecarSet.Name,
},
}, enqueueDelayTime)
}
}
}
备注：在此项目中有很多个crd都对pod的的变更进行了监听，同时实现了eventHandler的方法，这些方法均会在pod有对应事件的时候各自执行自己的方法逻辑
3, 使用对应的reconcile进行对sidecarset的处理，包括升级的策略的逻辑 具体逻辑在GetNextUpgradePods方法中
openkruise/kruise/pkg/controller/sidecarset/sidecarset_controller.go # Reconcile
func (r *ReconcileSidecarSet) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) {
// Fetch the SidecarSet instance
sidecarSet := &appsv1alpha1.SidecarSet{}
err := r.Get(context.TODO(), request.NamespacedName, sidecarSet)
if err != nil {
if errors.IsNotFound(err) {
// Object not found, return. Created objects are automatically garbage collected.
// For additional cleanup logic use finalizers.
return reconcile.Result{}, nil
}
// Error reading the object - requeue the request.
return reconcile.Result{}, err
}
klog.V(3).Infof("begin to process sidecarset %v for reconcile", sidecarSet.Name)
return r.processor.UpdateSidecarSet(sidecarSet)
}
