k8s.io/apiserver@v0.31.1/pkg/storage/etcd3/watcher.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package etcd3
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"os"
    24  	"strconv"
    25  	"strings"
    26  	"sync"
    27  	"time"
    28  
    29  	clientv3 "go.etcd.io/etcd/client/v3"
    30  	grpccodes "google.golang.org/grpc/codes"
    31  	grpcstatus "google.golang.org/grpc/status"
    32  
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	"k8s.io/apimachinery/pkg/runtime"
    35  	"k8s.io/apimachinery/pkg/runtime/schema"
    36  	"k8s.io/apimachinery/pkg/util/wait"
    37  	"k8s.io/apimachinery/pkg/watch"
    38  	"k8s.io/apiserver/pkg/features"
    39  	"k8s.io/apiserver/pkg/storage"
    40  	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
    41  	"k8s.io/apiserver/pkg/storage/value"
    42  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    43  	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
    44  	"k8s.io/klog/v2"
    45  )
    46  
    47  const (
    48  	// We have set a buffer in order to reduce times of context switches.
    49  	incomingBufSize         = 100
    50  	outgoingBufSize         = 100
    51  	processEventConcurrency = 10
    52  )
    53  
    54  // defaultWatcherMaxLimit is used to facilitate construction tests
    55  var defaultWatcherMaxLimit int64 = maxLimit
    56  
    57  // fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error
    58  var fatalOnDecodeError = false
    59  
    60  func init() {
    61  	// check to see if we are running in a test environment
    62  	TestOnlySetFatalOnDecodeError(true)
    63  	fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
    64  }
    65  
    66  // TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks.
    67  func TestOnlySetFatalOnDecodeError(b bool) {
    68  	fatalOnDecodeError = b
    69  }
    70  
    71  type watcher struct {
    72  	client              *clientv3.Client
    73  	codec               runtime.Codec
    74  	newFunc             func() runtime.Object
    75  	objectType          string
    76  	groupResource       schema.GroupResource
    77  	versioner           storage.Versioner
    78  	transformer         value.Transformer
    79  	getCurrentStorageRV func(context.Context) (uint64, error)
    80  }
    81  
    82  // watchChan implements watch.Interface.
    83  type watchChan struct {
    84  	watcher           *watcher
    85  	key               string
    86  	initialRev        int64
    87  	recursive         bool
    88  	progressNotify    bool
    89  	internalPred      storage.SelectionPredicate
    90  	ctx               context.Context
    91  	cancel            context.CancelFunc
    92  	incomingEventChan chan *event
    93  	resultChan        chan watch.Event
    94  	errChan           chan error
    95  }
    96  
    97  // Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
    98  // If rev is zero, it will return the existing object(s) and then start watching from
    99  // the maximum revision+1 from returned objects.
   100  // If rev is non-zero, it will watch events happened after given revision.
   101  // If opts.Recursive is false, it watches on given key.
   102  // If opts.Recursive is true, it watches any children and directories under the key, excluding the root key itself.
   103  // pred must be non-nil. Only if opts.Predicate matches the change, it will be returned.
   104  func (w *watcher) Watch(ctx context.Context, key string, rev int64, opts storage.ListOptions) (watch.Interface, error) {
   105  	if opts.Recursive && !strings.HasSuffix(key, "/") {
   106  		key += "/"
   107  	}
   108  	if opts.ProgressNotify && w.newFunc == nil {
   109  		return nil, apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided"))
   110  	}
   111  	startWatchRV, err := w.getStartWatchResourceVersion(ctx, rev, opts)
   112  	if err != nil {
   113  		return nil, err
   114  	}
   115  	wc := w.createWatchChan(ctx, key, startWatchRV, opts.Recursive, opts.ProgressNotify, opts.Predicate)
   116  	go wc.run(isInitialEventsEndBookmarkRequired(opts), areInitialEventsRequired(rev, opts))
   117  
   118  	// For etcd watch we don't have an easy way to answer whether the watch
   119  	// has already caught up. So in the initial version (given that watchcache
   120  	// is by default enabled for all resources but Events), we just deliver
   121  	// the initialization signal immediately. Improving this will be explored
   122  	// in the future.
   123  	utilflowcontrol.WatchInitialized(ctx)
   124  
   125  	return wc, nil
   126  }
   127  
   128  func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) *watchChan {
   129  	wc := &watchChan{
   130  		watcher:           w,
   131  		key:               key,
   132  		initialRev:        rev,
   133  		recursive:         recursive,
   134  		progressNotify:    progressNotify,
   135  		internalPred:      pred,
   136  		incomingEventChan: make(chan *event, incomingBufSize),
   137  		resultChan:        make(chan watch.Event, outgoingBufSize),
   138  		errChan:           make(chan error, 1),
   139  	}
   140  	if pred.Empty() {
   141  		// The filter doesn't filter out any object.
   142  		wc.internalPred = storage.Everything
   143  	}
   144  	wc.ctx, wc.cancel = context.WithCancel(ctx)
   145  	return wc
   146  }
   147  
   148  // getStartWatchResourceVersion returns a ResourceVersion
   149  // the watch will be started from.
   150  // Depending on the input parameters the semantics of the returned ResourceVersion are:
   151  //   - start at Exact (return resourceVersion)
   152  //   - start at Most Recent (return an RV from etcd)
   153  func (w *watcher) getStartWatchResourceVersion(ctx context.Context, resourceVersion int64, opts storage.ListOptions) (int64, error) {
   154  	if resourceVersion > 0 {
   155  		return resourceVersion, nil
   156  	}
   157  	if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
   158  		return 0, nil
   159  	}
   160  	if opts.SendInitialEvents == nil || *opts.SendInitialEvents {
   161  		// note that when opts.SendInitialEvents=true
   162  		// we will be issuing a consistent LIST request
   163  		// against etcd followed by the special bookmark event
   164  		return 0, nil
   165  	}
   166  	// at this point the clients is interested
   167  	// only in getting a stream of events
   168  	// starting at the MostRecent point in time (RV)
   169  	currentStorageRV, err := w.getCurrentStorageRV(ctx)
   170  	if err != nil {
   171  		return 0, err
   172  	}
   173  	// currentStorageRV is taken from resp.Header.Revision (int64)
   174  	// and cast to uint64, so it is safe to do reverse
   175  	// at some point we should unify the interface but that
   176  	// would require changing  Versioner.UpdateList
   177  	return int64(currentStorageRV), nil
   178  }
   179  
   180  // isInitialEventsEndBookmarkRequired since there is no way to directly set
   181  // opts.ProgressNotify from the API and the etcd3 impl doesn't support
   182  // notification for external clients we simply return initialEventsEndBookmarkRequired
   183  // to only send the bookmark event after the initial list call.
   184  //
   185  // see: https://github.com/kubernetes/kubernetes/issues/120348
   186  func isInitialEventsEndBookmarkRequired(opts storage.ListOptions) bool {
   187  	if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
   188  		return false
   189  	}
   190  	return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks
   191  }
   192  
   193  // areInitialEventsRequired returns true if all events from the etcd should be returned.
   194  func areInitialEventsRequired(resourceVersion int64, opts storage.ListOptions) bool {
   195  	if opts.SendInitialEvents == nil && resourceVersion == 0 {
   196  		return true // legacy case
   197  	}
   198  	if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
   199  		return false
   200  	}
   201  	return opts.SendInitialEvents != nil && *opts.SendInitialEvents
   202  }
   203  
   204  type etcdError interface {
   205  	Code() grpccodes.Code
   206  	Error() string
   207  }
   208  
   209  type grpcError interface {
   210  	GRPCStatus() *grpcstatus.Status
   211  }
   212  
   213  func isCancelError(err error) bool {
   214  	if err == nil {
   215  		return false
   216  	}
   217  	if err == context.Canceled {
   218  		return true
   219  	}
   220  	if etcdErr, ok := err.(etcdError); ok && etcdErr.Code() == grpccodes.Canceled {
   221  		return true
   222  	}
   223  	if grpcErr, ok := err.(grpcError); ok && grpcErr.GRPCStatus().Code() == grpccodes.Canceled {
   224  		return true
   225  	}
   226  	return false
   227  }
   228  
   229  func (wc *watchChan) run(initialEventsEndBookmarkRequired, forceInitialEvents bool) {
   230  	watchClosedCh := make(chan struct{})
   231  	go wc.startWatching(watchClosedCh, initialEventsEndBookmarkRequired, forceInitialEvents)
   232  
   233  	var resultChanWG sync.WaitGroup
   234  	wc.processEvents(&resultChanWG)
   235  
   236  	select {
   237  	case err := <-wc.errChan:
   238  		if isCancelError(err) {
   239  			break
   240  		}
   241  		errResult := transformErrorToEvent(err)
   242  		if errResult != nil {
   243  			// error result is guaranteed to be received by user before closing ResultChan.
   244  			select {
   245  			case wc.resultChan <- *errResult:
   246  			case <-wc.ctx.Done(): // user has given up all results
   247  			}
   248  		}
   249  	case <-watchClosedCh:
   250  	case <-wc.ctx.Done(): // user cancel
   251  	}
   252  
   253  	// We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all.
   254  	// It's fine to double cancel.
   255  	wc.cancel()
   256  
   257  	// we need to wait until resultChan wouldn't be used anymore
   258  	resultChanWG.Wait()
   259  	close(wc.resultChan)
   260  }
   261  
   262  func (wc *watchChan) Stop() {
   263  	wc.cancel()
   264  }
   265  
   266  func (wc *watchChan) ResultChan() <-chan watch.Event {
   267  	return wc.resultChan
   268  }
   269  
   270  func (wc *watchChan) RequestWatchProgress() error {
   271  	return wc.watcher.client.RequestProgress(wc.ctx)
   272  }
   273  
   274  // sync tries to retrieve existing data and send them to process.
   275  // The revision to watch will be set to the revision in response.
   276  // All events sent will have isCreated=true
   277  func (wc *watchChan) sync() error {
   278  	opts := []clientv3.OpOption{}
   279  	if wc.recursive {
   280  		opts = append(opts, clientv3.WithLimit(defaultWatcherMaxLimit))
   281  		rangeEnd := clientv3.GetPrefixRangeEnd(wc.key)
   282  		opts = append(opts, clientv3.WithRange(rangeEnd))
   283  	}
   284  
   285  	var err error
   286  	var lastKey []byte
   287  	var withRev int64
   288  	var getResp *clientv3.GetResponse
   289  
   290  	metricsOp := "get"
   291  	if wc.recursive {
   292  		metricsOp = "list"
   293  	}
   294  
   295  	preparedKey := wc.key
   296  
   297  	for {
   298  		startTime := time.Now()
   299  		getResp, err = wc.watcher.client.KV.Get(wc.ctx, preparedKey, opts...)
   300  		metrics.RecordEtcdRequest(metricsOp, wc.watcher.groupResource.String(), err, startTime)
   301  		if err != nil {
   302  			return interpretListError(err, true, preparedKey, wc.key)
   303  		}
   304  
   305  		if len(getResp.Kvs) == 0 && getResp.More {
   306  			return fmt.Errorf("no results were found, but etcd indicated there were more values remaining")
   307  		}
   308  
   309  		// send items from the response until no more results
   310  		for i, kv := range getResp.Kvs {
   311  			lastKey = kv.Key
   312  			wc.sendEvent(parseKV(kv))
   313  			// free kv early. Long lists can take O(seconds) to decode.
   314  			getResp.Kvs[i] = nil
   315  		}
   316  
   317  		if withRev == 0 {
   318  			wc.initialRev = getResp.Header.Revision
   319  		}
   320  
   321  		// no more results remain
   322  		if !getResp.More {
   323  			return nil
   324  		}
   325  
   326  		preparedKey = string(lastKey) + "\x00"
   327  		if withRev == 0 {
   328  			withRev = getResp.Header.Revision
   329  			opts = append(opts, clientv3.WithRev(withRev))
   330  		}
   331  	}
   332  }
   333  
   334  func logWatchChannelErr(err error) {
   335  	switch {
   336  	case strings.Contains(err.Error(), "mvcc: required revision has been compacted"):
   337  		// mvcc revision compaction which is regarded as warning, not error
   338  		klog.Warningf("watch chan error: %v", err)
   339  	case isCancelError(err):
   340  		// expected when watches close, no need to log
   341  	default:
   342  		klog.Errorf("watch chan error: %v", err)
   343  	}
   344  }
   345  
   346  // startWatching does:
   347  // - get current objects if initialRev=0; set initialRev to current rev
   348  // - watch on given key and send events to process.
   349  //
   350  // initialEventsEndBookmarkSent helps us keep track
   351  // of whether we have sent an annotated bookmark event.
   352  //
   353  // it's important to note that we don't
   354  // need to track the actual RV because
   355  // we only send the bookmark event
   356  // after the initial list call.
   357  //
   358  // when this variable is set to false,
   359  // it means we don't have any specific
   360  // preferences for delivering bookmark events.
   361  func (wc *watchChan) startWatching(watchClosedCh chan struct{}, initialEventsEndBookmarkRequired, forceInitialEvents bool) {
   362  	if wc.initialRev > 0 && forceInitialEvents {
   363  		currentStorageRV, err := wc.watcher.getCurrentStorageRV(wc.ctx)
   364  		if err != nil {
   365  			wc.sendError(err)
   366  			return
   367  		}
   368  		if uint64(wc.initialRev) > currentStorageRV {
   369  			wc.sendError(storage.NewTooLargeResourceVersionError(uint64(wc.initialRev), currentStorageRV, int(wait.Jitter(1*time.Second, 3).Seconds())))
   370  			return
   371  		}
   372  	}
   373  	if forceInitialEvents {
   374  		if err := wc.sync(); err != nil {
   375  			klog.Errorf("failed to sync with latest state: %v", err)
   376  			wc.sendError(err)
   377  			return
   378  		}
   379  	}
   380  	if initialEventsEndBookmarkRequired {
   381  		wc.sendEvent(func() *event {
   382  			e := progressNotifyEvent(wc.initialRev)
   383  			e.isInitialEventsEndBookmark = true
   384  			return e
   385  		}())
   386  	}
   387  	opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()}
   388  	if wc.recursive {
   389  		opts = append(opts, clientv3.WithPrefix())
   390  	}
   391  	if wc.progressNotify {
   392  		opts = append(opts, clientv3.WithProgressNotify())
   393  	}
   394  	wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...)
   395  	for wres := range wch {
   396  		if wres.Err() != nil {
   397  			err := wres.Err()
   398  			// If there is an error on server (e.g. compaction), the channel will return it before closed.
   399  			logWatchChannelErr(err)
   400  			wc.sendError(err)
   401  			return
   402  		}
   403  		if wres.IsProgressNotify() {
   404  			wc.sendEvent(progressNotifyEvent(wres.Header.GetRevision()))
   405  			metrics.RecordEtcdBookmark(wc.watcher.groupResource.String())
   406  			continue
   407  		}
   408  
   409  		for _, e := range wres.Events {
   410  			metrics.RecordEtcdEvent(wc.watcher.groupResource.String())
   411  			parsedEvent, err := parseEvent(e)
   412  			if err != nil {
   413  				logWatchChannelErr(err)
   414  				wc.sendError(err)
   415  				return
   416  			}
   417  			wc.sendEvent(parsedEvent)
   418  		}
   419  	}
   420  	// When we come to this point, it's only possible that client side ends the watch.
   421  	// e.g. cancel the context, close the client.
   422  	// If this watch chan is broken and context isn't cancelled, other goroutines will still hang.
   423  	// We should notify the main thread that this goroutine has exited.
   424  	close(watchClosedCh)
   425  }
   426  
   427  // processEvents processes events from etcd watcher and sends results to resultChan.
   428  func (wc *watchChan) processEvents(wg *sync.WaitGroup) {
   429  	if utilfeature.DefaultFeatureGate.Enabled(features.ConcurrentWatchObjectDecode) {
   430  		wc.concurrentProcessEvents(wg)
   431  	} else {
   432  		wg.Add(1)
   433  		go wc.serialProcessEvents(wg)
   434  	}
   435  }
   436  func (wc *watchChan) serialProcessEvents(wg *sync.WaitGroup) {
   437  	defer wg.Done()
   438  	for {
   439  		select {
   440  		case e := <-wc.incomingEventChan:
   441  			res := wc.transform(e)
   442  			if res == nil {
   443  				continue
   444  			}
   445  			if len(wc.resultChan) == cap(wc.resultChan) {
   446  				klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource)
   447  			}
   448  			// If user couldn't receive results fast enough, we also block incoming events from watcher.
   449  			// Because storing events in local will cause more memory usage.
   450  			// The worst case would be closing the fast watcher.
   451  			select {
   452  			case wc.resultChan <- *res:
   453  			case <-wc.ctx.Done():
   454  				return
   455  			}
   456  		case <-wc.ctx.Done():
   457  			return
   458  		}
   459  	}
   460  }
   461  
   462  func (wc *watchChan) concurrentProcessEvents(wg *sync.WaitGroup) {
   463  	p := concurrentOrderedEventProcessing{
   464  		input:           wc.incomingEventChan,
   465  		processFunc:     wc.transform,
   466  		output:          wc.resultChan,
   467  		processingQueue: make(chan chan *watch.Event, processEventConcurrency-1),
   468  
   469  		objectType:    wc.watcher.objectType,
   470  		groupResource: wc.watcher.groupResource,
   471  	}
   472  	wg.Add(1)
   473  	go func() {
   474  		defer wg.Done()
   475  		p.scheduleEventProcessing(wc.ctx, wg)
   476  	}()
   477  	wg.Add(1)
   478  	go func() {
   479  		defer wg.Done()
   480  		p.collectEventProcessing(wc.ctx)
   481  	}()
   482  }
   483  
   484  type concurrentOrderedEventProcessing struct {
   485  	input       chan *event
   486  	processFunc func(*event) *watch.Event
   487  	output      chan watch.Event
   488  
   489  	processingQueue chan chan *watch.Event
   490  	// Metadata for logging
   491  	objectType    string
   492  	groupResource schema.GroupResource
   493  }
   494  
   495  func (p *concurrentOrderedEventProcessing) scheduleEventProcessing(ctx context.Context, wg *sync.WaitGroup) {
   496  	var e *event
   497  	for {
   498  		select {
   499  		case <-ctx.Done():
   500  			return
   501  		case e = <-p.input:
   502  		}
   503  		processingResponse := make(chan *watch.Event, 1)
   504  		select {
   505  		case <-ctx.Done():
   506  			return
   507  		case p.processingQueue <- processingResponse:
   508  		}
   509  		wg.Add(1)
   510  		go func(e *event, response chan<- *watch.Event) {
   511  			defer wg.Done()
   512  			select {
   513  			case <-ctx.Done():
   514  			case response <- p.processFunc(e):
   515  			}
   516  		}(e, processingResponse)
   517  	}
   518  }
   519  
   520  func (p *concurrentOrderedEventProcessing) collectEventProcessing(ctx context.Context) {
   521  	var processingResponse chan *watch.Event
   522  	var e *watch.Event
   523  	for {
   524  		select {
   525  		case <-ctx.Done():
   526  			return
   527  		case processingResponse = <-p.processingQueue:
   528  		}
   529  		select {
   530  		case <-ctx.Done():
   531  			return
   532  		case e = <-processingResponse:
   533  		}
   534  		if e == nil {
   535  			continue
   536  		}
   537  		if len(p.output) == cap(p.output) {
   538  			klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", p.objectType, "groupResource", p.groupResource)
   539  		}
   540  		// If user couldn't receive results fast enough, we also block incoming events from watcher.
   541  		// Because storing events in local will cause more memory usage.
   542  		// The worst case would be closing the fast watcher.
   543  		select {
   544  		case <-ctx.Done():
   545  			return
   546  		case p.output <- *e:
   547  		}
   548  	}
   549  }
   550  
   551  func (wc *watchChan) filter(obj runtime.Object) bool {
   552  	if wc.internalPred.Empty() {
   553  		return true
   554  	}
   555  	matched, err := wc.internalPred.Matches(obj)
   556  	return err == nil && matched
   557  }
   558  
   559  func (wc *watchChan) acceptAll() bool {
   560  	return wc.internalPred.Empty()
   561  }
   562  
   563  // transform transforms an event into a result for user if not filtered.
   564  func (wc *watchChan) transform(e *event) (res *watch.Event) {
   565  	curObj, oldObj, err := wc.prepareObjs(e)
   566  	if err != nil {
   567  		klog.Errorf("failed to prepare current and previous objects: %v", err)
   568  		wc.sendError(err)
   569  		return nil
   570  	}
   571  
   572  	switch {
   573  	case e.isProgressNotify:
   574  		object := wc.watcher.newFunc()
   575  		if err := wc.watcher.versioner.UpdateObject(object, uint64(e.rev)); err != nil {
   576  			klog.Errorf("failed to propagate object version: %v", err)
   577  			return nil
   578  		}
   579  		if e.isInitialEventsEndBookmark {
   580  			if err := storage.AnnotateInitialEventsEndBookmark(object); err != nil {
   581  				wc.sendError(fmt.Errorf("error while accessing object's metadata gr: %v, type: %v, obj: %#v, err: %v", wc.watcher.groupResource, wc.watcher.objectType, object, err))
   582  				return nil
   583  			}
   584  		}
   585  		res = &watch.Event{
   586  			Type:   watch.Bookmark,
   587  			Object: object,
   588  		}
   589  	case e.isDeleted:
   590  		if !wc.filter(oldObj) {
   591  			return nil
   592  		}
   593  		res = &watch.Event{
   594  			Type:   watch.Deleted,
   595  			Object: oldObj,
   596  		}
   597  	case e.isCreated:
   598  		if !wc.filter(curObj) {
   599  			return nil
   600  		}
   601  		res = &watch.Event{
   602  			Type:   watch.Added,
   603  			Object: curObj,
   604  		}
   605  	default:
   606  		if wc.acceptAll() {
   607  			res = &watch.Event{
   608  				Type:   watch.Modified,
   609  				Object: curObj,
   610  			}
   611  			return res
   612  		}
   613  		curObjPasses := wc.filter(curObj)
   614  		oldObjPasses := wc.filter(oldObj)
   615  		switch {
   616  		case curObjPasses && oldObjPasses:
   617  			res = &watch.Event{
   618  				Type:   watch.Modified,
   619  				Object: curObj,
   620  			}
   621  		case curObjPasses && !oldObjPasses:
   622  			res = &watch.Event{
   623  				Type:   watch.Added,
   624  				Object: curObj,
   625  			}
   626  		case !curObjPasses && oldObjPasses:
   627  			res = &watch.Event{
   628  				Type:   watch.Deleted,
   629  				Object: oldObj,
   630  			}
   631  		}
   632  	}
   633  	return res
   634  }
   635  
   636  func transformErrorToEvent(err error) *watch.Event {
   637  	err = interpretWatchError(err)
   638  	if _, ok := err.(apierrors.APIStatus); !ok {
   639  		err = apierrors.NewInternalError(err)
   640  	}
   641  	status := err.(apierrors.APIStatus).Status()
   642  	return &watch.Event{
   643  		Type:   watch.Error,
   644  		Object: &status,
   645  	}
   646  }
   647  
   648  func (wc *watchChan) sendError(err error) {
   649  	select {
   650  	case wc.errChan <- err:
   651  	case <-wc.ctx.Done():
   652  	}
   653  }
   654  
   655  func (wc *watchChan) sendEvent(e *event) {
   656  	if len(wc.incomingEventChan) == incomingBufSize {
   657  		klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow decoding, user not receiving fast, or other processing logic", "incomingEvents", incomingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource)
   658  	}
   659  	select {
   660  	case wc.incomingEventChan <- e:
   661  	case <-wc.ctx.Done():
   662  	}
   663  }
   664  
   665  func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) {
   666  	if e.isProgressNotify {
   667  		// progressNotify events doesn't contain neither current nor previous object version,
   668  		return nil, nil, nil
   669  	}
   670  
   671  	if !e.isDeleted {
   672  		data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.value, authenticatedDataString(e.key))
   673  		if err != nil {
   674  			return nil, nil, err
   675  		}
   676  		curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
   677  		if err != nil {
   678  			return nil, nil, err
   679  		}
   680  	}
   681  	// We need to decode prevValue, only if this is deletion event or
   682  	// the underlying filter doesn't accept all objects (otherwise we
   683  	// know that the filter for previous object will return true and
   684  	// we need the object only to compute whether it was filtered out
   685  	// before).
   686  	if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) {
   687  		data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.prevValue, authenticatedDataString(e.key))
   688  		if err != nil {
   689  			return nil, nil, err
   690  		}
   691  		// Note that this sends the *old* object with the etcd revision for the time at
   692  		// which it gets deleted.
   693  		oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
   694  		if err != nil {
   695  			return nil, nil, err
   696  		}
   697  	}
   698  	return curObj, oldObj, nil
   699  }
   700  
   701  func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) {
   702  	obj, err := runtime.Decode(codec, []byte(data))
   703  	if err != nil {
   704  		if fatalOnDecodeError {
   705  			// we are running in a test environment and thus an
   706  			// error here is due to a coder mistake if the defer
   707  			// does not catch it
   708  			panic(err)
   709  		}
   710  		return nil, err
   711  	}
   712  	// ensure resource version is set on the object we load from etcd
   713  	if err := versioner.UpdateObject(obj, uint64(rev)); err != nil {
   714  		return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err)
   715  	}
   716  	return obj, nil
   717  }