Kubernetes集群中通常会有很多Event，这些Event会异步的方式发送到监听Event的多个Watcher.
Event的分发属于多播方式，语义可以分为三种：exactly-once, at-most once, at least once
下面先来实现一个简单的单播，即一个发送者和一个接收者。这个例子很简单，发送者即会顺序产生的数字发送给监听者。
单播
package main
import (
"fmt"
"time"
)
func main() {
m:=make(chan int)
count :=1
// event source
go func() {
for{
time.Sleep(1* time.Second)
m<-count
count++
}
}()
// event sink
go func() {
for i:=range m{
fmt.Println(i)
}
}()
time.Sleep(1 * time.Minute)
}
多播
多播实现则较为复杂，需要设计出一个BroadCaster将Event分发到注册的多个Watcher
注释部分，通过select如果没有channel可读写，进入阻塞特性，实现两个语义 exactly-once，at-most-once default存在代表永远有一个channel可读写
这个例子中故意将发送者的发送速率是Watcher消费速率的10倍。而Watcher的接收窗口只有一个，因此其 channel int类型的result经常满而无法写入。
package main
import (
"fmt"
"math/rand"
"time"
)
type Watcher struct {
stop chan struct{}
result chan int
idx int
}
func (w *Watcher) Run() {
for i:=range w.result{
time.Sleep(1 * time.Second)
fmt.Println("idx",w.idx,",receive result:",i)
select {
case <-w.stop:
return
default:
}
}
}
func (w *Watcher) Stop(){
close(w.stop)
}
func main() {
// broadcast group
watchers:=[]*Watcher{NewDefaultWatcher(),NewDefaultWatcher()}
for _,w:=range watchers{
go w.Run()
}
// stop on exited
defer func() {
for _,w:=range watchers{
w.Stop()
}
}()
// event source
m:=make(chan int)
count :=1
// event source go routine
go func() {
for{
time.Sleep(100* time.Microsecond)
m<-count
count++
}
}()
// the broadcaster send event to every watchers
go func() {
for i:=range m{
// broadcast to all watchers
for _,w:=range watchers{
select {
case w.result<-i:
case <-w.stop:
// default: // comment it to implement exactly-once delivery.
// uncomment it to implement at-most-once delivery.
}
}
}
}()
time.Sleep(1 * time.Minute)
}
func NewDefaultWatcher() *Watcher{
return &Watcher{
stop: make(chan struct{}),
result: make(chan int,1),
idx: rand.Intn(10),
}
}
kubernetes Events机制实现
kube-scheduler调度Pod时会产生各种事件：
比如调度失败的场景：
fwk.EventRecorder().Eventf(pod, nil, v1.EventTypeWarning, "FailedScheduling", "Scheduling", msg)
- EventRecorder
EventRecorder定义了Eventf方法，kube-scheduler属于EventSource角色。
// EventRecorder knows how to record events on behalf of an EventSource.
type EventRecorder interface {
// Eventf constructs an event from the given information and puts it in the queue for sending.
// 'regarding' is the object this event is about. Event will make a reference-- or you may also
// pass a reference to the object directly.
// 'related' is the secondary object for more complex actions. E.g. when regarding object triggers
// a creation or deletion of related object.
// 'type' of this event, and can be one of Normal, Warning. New types could be added in future
// 'reason' is the reason this event is generated. 'reason' should be short and unique; it
// should be in UpperCamelCase format (starting with a capital letter). "reason" will be used
// to automate handling of events, so imagine people writing switch statements to handle them.
// You want to make that easy.
// 'action' explains what happened with regarding/what action did the ReportingController
// (ReportingController is a type of a Controller reporting an Event, e.g. k8s.io/node-controller, k8s.io/kubelet.)
// take in regarding's name; it should be in UpperCamelCase format (starting with a capital letter).
// 'note' is intended to be human readable.
Eventf(regarding runtime.Object, related runtime.Object, eventtype, reason, action, note string, args ...interface{})
}
Eventf会初始化一个go routine将消息发送给Broadcaster.incoming channel中。
func (recorder *recorderImpl) Eventf(regarding runtime.Object, related runtime.Object, eventtype, reason, action, note string, args ...interface{}) {
// ...
event := recorder.makeEvent(refRegarding, refRelated, timestamp, eventtype, reason, message, recorder.reportingController, recorder.reportingInstance, action)
go func() {
defer utilruntime.HandleCrash()
recorder.Action(watch.Added, event) // 调用下面
}()
}
// 被上面调用
// Action distributes the given event among all watchers.
func (m *Broadcaster) Action(action EventType, obj runtime.Object) {
m.incoming <- Event{action, obj}
}
- Broadcaster
EventRecorder的实现持有*Broadcaster
Broadcaster初始化时，会创建一个定时任务Broadcaste.loop将周期性地将从channel incomming读取来的Event分发给多个broadcasterWatcher。
// Broadcaster distributes event notifications among any number of watchers. Every event
// is delivered to every watcher.
type Broadcaster struct {
watchers map[int64]*broadcasterWatcher
nextWatcher int64
distributing sync.WaitGroup
incoming chan Event
stopped chan struct{}
// How large to make watcher's channel.
watchQueueLength int
// If one of the watch channels is full, don't wait for it to become empty.
// Instead just deliver it to the watchers that do have space in their
// channels and move on to the next event.
// It's more fair to do this on a per-watcher basis than to do it on the
// "incoming" channel, which would allow one slow watcher to prevent all
// other watchers from getting new events.
fullChannelBehavior FullChannelBehavior
}
// NewBroadcaster creates a new Broadcaster. queueLength is the maximum number of events to queue per watcher.
// It is guaranteed that events will be distributed in the order in which they occur,
// but the order in which a single event is distributed among all of the watchers is unspecified.
func NewBroadcaster(queueLength int, fullChannelBehavior FullChannelBehavior) *Broadcaster {
m := &Broadcaster{
watchers: map[int64]*broadcasterWatcher{},
incoming: make(chan Event, incomingQueueLength),
stopped: make(chan struct{}),
watchQueueLength: queueLength,
fullChannelBehavior: fullChannelBehavior,
}
m.distributing.Add(1)
go m.loop()
return m
}
loop方法的是多播实现
// loop receives from m.incoming and distributes to all watchers.
func (m *Broadcaster) loop() {
// Deliberately not catching crashes here. Yes, bring down the process if there's a
// bug in watch.Broadcaster.
// 读取每个事件Event
for event := range m.incoming {
if event.Type == internalRunFunctionMarker {
event.Object.(functionFakeRuntimeObject)()
continue
}
m.distribute(event) // 将Event广播出去
}
//...
}
// distribute sends event to all watchers. Blocking.
func (m *Broadcaster) distribute(event Event) {
if m.fullChannelBehavior == DropIfChannelFull {
// 广播给多个Watcher
for _, w := range m.watchers {
select {
case w.result <- event:
case <-w.stopped:
default: // Don't block if the event can't be queued.
}
}
}
// ...
}
- EventBroadcaster
EventBroadcaster负责接收事件并将事件发送给EventSink,EventSink的实现能够将Event持久化到etcd，即相当于EventBroadcaster的作用将事件发送给EventSink做Event持久化。
kube-scheduler一启动，就会调用StartRecordingToSink。
// Run executes the scheduler based on the given configuration. It only returns on error or when context is done.
func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *scheduler.Scheduler) error {
// To help debugging, immediately log version
klog.V(1).Infof("Starting Kubernetes Scheduler version %+v", version.Get())
// ...
// Prepare the event broadcaster.
cc.EventBroadcaster.StartRecordingToSink(ctx.Done())
// ...
}
EventBroadcaster的实现也持有*Broadcaster,并且客户端可以通过NewRecorder获得往持有*Broadcaster发送事件的EventRecorder。
StartRecordingToSink方法的作用就是将事件Event从*Broadcaster中拿出来，放到EventSink中去。怎么实现？
// EventBroadcaster knows how to receive events and send them to any EventSink, watcher, or log.
type EventBroadcaster interface {
// StartRecordingToSink starts sending events received from the specified eventBroadcaster.
StartRecordingToSink(stopCh <-chan struct{})
// NewRecorder returns an EventRecorder that can be used to send events to this EventBroadcaster
// with the event source set to the given event source.
NewRecorder(scheme *runtime.Scheme, reportingController string) EventRecorder
// StartEventWatcher enables you to watch for emitted events without usage
// of StartRecordingToSink. This lets you also process events in a custom way (e.g. in tests).
// NOTE: events received on your eventHandler should be copied before being used.
// TODO: figure out if this can be removed.
StartEventWatcher(eventHandler func(event runtime.Object)) func()
// Shutdown shuts down the broadcaster
Shutdown()
}
StartRecordingToSink 会创建一个新的Broadcaster.watcher，
func (e *eventBroadcasterImpl) startRecordingEvents(stopCh <-chan struct{}) {
// Event持久化Handler
eventHandler := func(obj runtime.Object) {
event, ok := obj.(*eventsv1.Event)
if !ok {
klog.Errorf("unexpected type, expected eventsv1.Event")
return
}
e.recordToSink(event, clock.RealClock{}) // 发送到EventSink
}
stopWatcher := e.StartEventWatcher(eventHandler) // 异步启动
go func() {
<-stopCh
stopWatcher()
}()
}
// StartEventWatcher starts sending events received from this EventBroadcaster to the given event handler function.
// The return value is used to stop recording
func (e *eventBroadcasterImpl) StartEventWatcher(eventHandler func(event runtime.Object)) func() {
watcher := e.Watch() // 创建一个新的`Broadcaster.watcher`
go func() {
defer utilruntime.HandleCrash()
// 一直循环
for {
watchEvent, ok := <-watcher.ResultChan() // 从watcher的result channel中读取事件
if !ok {
return
}
eventHandler(watchEvent.Object) // Event持久化Handler
}
}()
return watcher.Stop
}
