k8s.io/apiserver@v0.31.1/pkg/storage/etcd3/watcher.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package etcd3 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "os" 24 "strconv" 25 "strings" 26 "sync" 27 "time" 28 29 clientv3 "go.etcd.io/etcd/client/v3" 30 grpccodes "google.golang.org/grpc/codes" 31 grpcstatus "google.golang.org/grpc/status" 32 33 apierrors "k8s.io/apimachinery/pkg/api/errors" 34 "k8s.io/apimachinery/pkg/runtime" 35 "k8s.io/apimachinery/pkg/runtime/schema" 36 "k8s.io/apimachinery/pkg/util/wait" 37 "k8s.io/apimachinery/pkg/watch" 38 "k8s.io/apiserver/pkg/features" 39 "k8s.io/apiserver/pkg/storage" 40 "k8s.io/apiserver/pkg/storage/etcd3/metrics" 41 "k8s.io/apiserver/pkg/storage/value" 42 utilfeature "k8s.io/apiserver/pkg/util/feature" 43 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" 44 "k8s.io/klog/v2" 45 ) 46 47 const ( 48 // We have set a buffer in order to reduce times of context switches. 49 incomingBufSize = 100 50 outgoingBufSize = 100 51 processEventConcurrency = 10 52 ) 53 54 // defaultWatcherMaxLimit is used to facilitate construction tests 55 var defaultWatcherMaxLimit int64 = maxLimit 56 57 // fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error 58 var fatalOnDecodeError = false 59 60 func init() { 61 // check to see if we are running in a test environment 62 TestOnlySetFatalOnDecodeError(true) 63 fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR")) 64 } 65 66 // TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks. 67 func TestOnlySetFatalOnDecodeError(b bool) { 68 fatalOnDecodeError = b 69 } 70 71 type watcher struct { 72 client *clientv3.Client 73 codec runtime.Codec 74 newFunc func() runtime.Object 75 objectType string 76 groupResource schema.GroupResource 77 versioner storage.Versioner 78 transformer value.Transformer 79 getCurrentStorageRV func(context.Context) (uint64, error) 80 } 81 82 // watchChan implements watch.Interface. 83 type watchChan struct { 84 watcher *watcher 85 key string 86 initialRev int64 87 recursive bool 88 progressNotify bool 89 internalPred storage.SelectionPredicate 90 ctx context.Context 91 cancel context.CancelFunc 92 incomingEventChan chan *event 93 resultChan chan watch.Event 94 errChan chan error 95 } 96 97 // Watch watches on a key and returns a watch.Interface that transfers relevant notifications. 98 // If rev is zero, it will return the existing object(s) and then start watching from 99 // the maximum revision+1 from returned objects. 100 // If rev is non-zero, it will watch events happened after given revision. 101 // If opts.Recursive is false, it watches on given key. 102 // If opts.Recursive is true, it watches any children and directories under the key, excluding the root key itself. 103 // pred must be non-nil. Only if opts.Predicate matches the change, it will be returned. 104 func (w *watcher) Watch(ctx context.Context, key string, rev int64, opts storage.ListOptions) (watch.Interface, error) { 105 if opts.Recursive && !strings.HasSuffix(key, "/") { 106 key += "/" 107 } 108 if opts.ProgressNotify && w.newFunc == nil { 109 return nil, apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided")) 110 } 111 startWatchRV, err := w.getStartWatchResourceVersion(ctx, rev, opts) 112 if err != nil { 113 return nil, err 114 } 115 wc := w.createWatchChan(ctx, key, startWatchRV, opts.Recursive, opts.ProgressNotify, opts.Predicate) 116 go wc.run(isInitialEventsEndBookmarkRequired(opts), areInitialEventsRequired(rev, opts)) 117 118 // For etcd watch we don't have an easy way to answer whether the watch 119 // has already caught up. So in the initial version (given that watchcache 120 // is by default enabled for all resources but Events), we just deliver 121 // the initialization signal immediately. Improving this will be explored 122 // in the future. 123 utilflowcontrol.WatchInitialized(ctx) 124 125 return wc, nil 126 } 127 128 func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) *watchChan { 129 wc := &watchChan{ 130 watcher: w, 131 key: key, 132 initialRev: rev, 133 recursive: recursive, 134 progressNotify: progressNotify, 135 internalPred: pred, 136 incomingEventChan: make(chan *event, incomingBufSize), 137 resultChan: make(chan watch.Event, outgoingBufSize), 138 errChan: make(chan error, 1), 139 } 140 if pred.Empty() { 141 // The filter doesn't filter out any object. 142 wc.internalPred = storage.Everything 143 } 144 wc.ctx, wc.cancel = context.WithCancel(ctx) 145 return wc 146 } 147 148 // getStartWatchResourceVersion returns a ResourceVersion 149 // the watch will be started from. 150 // Depending on the input parameters the semantics of the returned ResourceVersion are: 151 // - start at Exact (return resourceVersion) 152 // - start at Most Recent (return an RV from etcd) 153 func (w *watcher) getStartWatchResourceVersion(ctx context.Context, resourceVersion int64, opts storage.ListOptions) (int64, error) { 154 if resourceVersion > 0 { 155 return resourceVersion, nil 156 } 157 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) { 158 return 0, nil 159 } 160 if opts.SendInitialEvents == nil || *opts.SendInitialEvents { 161 // note that when opts.SendInitialEvents=true 162 // we will be issuing a consistent LIST request 163 // against etcd followed by the special bookmark event 164 return 0, nil 165 } 166 // at this point the clients is interested 167 // only in getting a stream of events 168 // starting at the MostRecent point in time (RV) 169 currentStorageRV, err := w.getCurrentStorageRV(ctx) 170 if err != nil { 171 return 0, err 172 } 173 // currentStorageRV is taken from resp.Header.Revision (int64) 174 // and cast to uint64, so it is safe to do reverse 175 // at some point we should unify the interface but that 176 // would require changing Versioner.UpdateList 177 return int64(currentStorageRV), nil 178 } 179 180 // isInitialEventsEndBookmarkRequired since there is no way to directly set 181 // opts.ProgressNotify from the API and the etcd3 impl doesn't support 182 // notification for external clients we simply return initialEventsEndBookmarkRequired 183 // to only send the bookmark event after the initial list call. 184 // 185 // see: https://github.com/kubernetes/kubernetes/issues/120348 186 func isInitialEventsEndBookmarkRequired(opts storage.ListOptions) bool { 187 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) { 188 return false 189 } 190 return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks 191 } 192 193 // areInitialEventsRequired returns true if all events from the etcd should be returned. 194 func areInitialEventsRequired(resourceVersion int64, opts storage.ListOptions) bool { 195 if opts.SendInitialEvents == nil && resourceVersion == 0 { 196 return true // legacy case 197 } 198 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) { 199 return false 200 } 201 return opts.SendInitialEvents != nil && *opts.SendInitialEvents 202 } 203 204 type etcdError interface { 205 Code() grpccodes.Code 206 Error() string 207 } 208 209 type grpcError interface { 210 GRPCStatus() *grpcstatus.Status 211 } 212 213 func isCancelError(err error) bool { 214 if err == nil { 215 return false 216 } 217 if err == context.Canceled { 218 return true 219 } 220 if etcdErr, ok := err.(etcdError); ok && etcdErr.Code() == grpccodes.Canceled { 221 return true 222 } 223 if grpcErr, ok := err.(grpcError); ok && grpcErr.GRPCStatus().Code() == grpccodes.Canceled { 224 return true 225 } 226 return false 227 } 228 229 func (wc *watchChan) run(initialEventsEndBookmarkRequired, forceInitialEvents bool) { 230 watchClosedCh := make(chan struct{}) 231 go wc.startWatching(watchClosedCh, initialEventsEndBookmarkRequired, forceInitialEvents) 232 233 var resultChanWG sync.WaitGroup 234 wc.processEvents(&resultChanWG) 235 236 select { 237 case err := <-wc.errChan: 238 if isCancelError(err) { 239 break 240 } 241 errResult := transformErrorToEvent(err) 242 if errResult != nil { 243 // error result is guaranteed to be received by user before closing ResultChan. 244 select { 245 case wc.resultChan <- *errResult: 246 case <-wc.ctx.Done(): // user has given up all results 247 } 248 } 249 case <-watchClosedCh: 250 case <-wc.ctx.Done(): // user cancel 251 } 252 253 // We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all. 254 // It's fine to double cancel. 255 wc.cancel() 256 257 // we need to wait until resultChan wouldn't be used anymore 258 resultChanWG.Wait() 259 close(wc.resultChan) 260 } 261 262 func (wc *watchChan) Stop() { 263 wc.cancel() 264 } 265 266 func (wc *watchChan) ResultChan() <-chan watch.Event { 267 return wc.resultChan 268 } 269 270 func (wc *watchChan) RequestWatchProgress() error { 271 return wc.watcher.client.RequestProgress(wc.ctx) 272 } 273 274 // sync tries to retrieve existing data and send them to process. 275 // The revision to watch will be set to the revision in response. 276 // All events sent will have isCreated=true 277 func (wc *watchChan) sync() error { 278 opts := []clientv3.OpOption{} 279 if wc.recursive { 280 opts = append(opts, clientv3.WithLimit(defaultWatcherMaxLimit)) 281 rangeEnd := clientv3.GetPrefixRangeEnd(wc.key) 282 opts = append(opts, clientv3.WithRange(rangeEnd)) 283 } 284 285 var err error 286 var lastKey []byte 287 var withRev int64 288 var getResp *clientv3.GetResponse 289 290 metricsOp := "get" 291 if wc.recursive { 292 metricsOp = "list" 293 } 294 295 preparedKey := wc.key 296 297 for { 298 startTime := time.Now() 299 getResp, err = wc.watcher.client.KV.Get(wc.ctx, preparedKey, opts...) 300 metrics.RecordEtcdRequest(metricsOp, wc.watcher.groupResource.String(), err, startTime) 301 if err != nil { 302 return interpretListError(err, true, preparedKey, wc.key) 303 } 304 305 if len(getResp.Kvs) == 0 && getResp.More { 306 return fmt.Errorf("no results were found, but etcd indicated there were more values remaining") 307 } 308 309 // send items from the response until no more results 310 for i, kv := range getResp.Kvs { 311 lastKey = kv.Key 312 wc.sendEvent(parseKV(kv)) 313 // free kv early. Long lists can take O(seconds) to decode. 314 getResp.Kvs[i] = nil 315 } 316 317 if withRev == 0 { 318 wc.initialRev = getResp.Header.Revision 319 } 320 321 // no more results remain 322 if !getResp.More { 323 return nil 324 } 325 326 preparedKey = string(lastKey) + "\x00" 327 if withRev == 0 { 328 withRev = getResp.Header.Revision 329 opts = append(opts, clientv3.WithRev(withRev)) 330 } 331 } 332 } 333 334 func logWatchChannelErr(err error) { 335 switch { 336 case strings.Contains(err.Error(), "mvcc: required revision has been compacted"): 337 // mvcc revision compaction which is regarded as warning, not error 338 klog.Warningf("watch chan error: %v", err) 339 case isCancelError(err): 340 // expected when watches close, no need to log 341 default: 342 klog.Errorf("watch chan error: %v", err) 343 } 344 } 345 346 // startWatching does: 347 // - get current objects if initialRev=0; set initialRev to current rev 348 // - watch on given key and send events to process. 349 // 350 // initialEventsEndBookmarkSent helps us keep track 351 // of whether we have sent an annotated bookmark event. 352 // 353 // it's important to note that we don't 354 // need to track the actual RV because 355 // we only send the bookmark event 356 // after the initial list call. 357 // 358 // when this variable is set to false, 359 // it means we don't have any specific 360 // preferences for delivering bookmark events. 361 func (wc *watchChan) startWatching(watchClosedCh chan struct{}, initialEventsEndBookmarkRequired, forceInitialEvents bool) { 362 if wc.initialRev > 0 && forceInitialEvents { 363 currentStorageRV, err := wc.watcher.getCurrentStorageRV(wc.ctx) 364 if err != nil { 365 wc.sendError(err) 366 return 367 } 368 if uint64(wc.initialRev) > currentStorageRV { 369 wc.sendError(storage.NewTooLargeResourceVersionError(uint64(wc.initialRev), currentStorageRV, int(wait.Jitter(1*time.Second, 3).Seconds()))) 370 return 371 } 372 } 373 if forceInitialEvents { 374 if err := wc.sync(); err != nil { 375 klog.Errorf("failed to sync with latest state: %v", err) 376 wc.sendError(err) 377 return 378 } 379 } 380 if initialEventsEndBookmarkRequired { 381 wc.sendEvent(func() *event { 382 e := progressNotifyEvent(wc.initialRev) 383 e.isInitialEventsEndBookmark = true 384 return e 385 }()) 386 } 387 opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()} 388 if wc.recursive { 389 opts = append(opts, clientv3.WithPrefix()) 390 } 391 if wc.progressNotify { 392 opts = append(opts, clientv3.WithProgressNotify()) 393 } 394 wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...) 395 for wres := range wch { 396 if wres.Err() != nil { 397 err := wres.Err() 398 // If there is an error on server (e.g. compaction), the channel will return it before closed. 399 logWatchChannelErr(err) 400 wc.sendError(err) 401 return 402 } 403 if wres.IsProgressNotify() { 404 wc.sendEvent(progressNotifyEvent(wres.Header.GetRevision())) 405 metrics.RecordEtcdBookmark(wc.watcher.groupResource.String()) 406 continue 407 } 408 409 for _, e := range wres.Events { 410 metrics.RecordEtcdEvent(wc.watcher.groupResource.String()) 411 parsedEvent, err := parseEvent(e) 412 if err != nil { 413 logWatchChannelErr(err) 414 wc.sendError(err) 415 return 416 } 417 wc.sendEvent(parsedEvent) 418 } 419 } 420 // When we come to this point, it's only possible that client side ends the watch. 421 // e.g. cancel the context, close the client. 422 // If this watch chan is broken and context isn't cancelled, other goroutines will still hang. 423 // We should notify the main thread that this goroutine has exited. 424 close(watchClosedCh) 425 } 426 427 // processEvents processes events from etcd watcher and sends results to resultChan. 428 func (wc *watchChan) processEvents(wg *sync.WaitGroup) { 429 if utilfeature.DefaultFeatureGate.Enabled(features.ConcurrentWatchObjectDecode) { 430 wc.concurrentProcessEvents(wg) 431 } else { 432 wg.Add(1) 433 go wc.serialProcessEvents(wg) 434 } 435 } 436 func (wc *watchChan) serialProcessEvents(wg *sync.WaitGroup) { 437 defer wg.Done() 438 for { 439 select { 440 case e := <-wc.incomingEventChan: 441 res := wc.transform(e) 442 if res == nil { 443 continue 444 } 445 if len(wc.resultChan) == cap(wc.resultChan) { 446 klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource) 447 } 448 // If user couldn't receive results fast enough, we also block incoming events from watcher. 449 // Because storing events in local will cause more memory usage. 450 // The worst case would be closing the fast watcher. 451 select { 452 case wc.resultChan <- *res: 453 case <-wc.ctx.Done(): 454 return 455 } 456 case <-wc.ctx.Done(): 457 return 458 } 459 } 460 } 461 462 func (wc *watchChan) concurrentProcessEvents(wg *sync.WaitGroup) { 463 p := concurrentOrderedEventProcessing{ 464 input: wc.incomingEventChan, 465 processFunc: wc.transform, 466 output: wc.resultChan, 467 processingQueue: make(chan chan *watch.Event, processEventConcurrency-1), 468 469 objectType: wc.watcher.objectType, 470 groupResource: wc.watcher.groupResource, 471 } 472 wg.Add(1) 473 go func() { 474 defer wg.Done() 475 p.scheduleEventProcessing(wc.ctx, wg) 476 }() 477 wg.Add(1) 478 go func() { 479 defer wg.Done() 480 p.collectEventProcessing(wc.ctx) 481 }() 482 } 483 484 type concurrentOrderedEventProcessing struct { 485 input chan *event 486 processFunc func(*event) *watch.Event 487 output chan watch.Event 488 489 processingQueue chan chan *watch.Event 490 // Metadata for logging 491 objectType string 492 groupResource schema.GroupResource 493 } 494 495 func (p *concurrentOrderedEventProcessing) scheduleEventProcessing(ctx context.Context, wg *sync.WaitGroup) { 496 var e *event 497 for { 498 select { 499 case <-ctx.Done(): 500 return 501 case e = <-p.input: 502 } 503 processingResponse := make(chan *watch.Event, 1) 504 select { 505 case <-ctx.Done(): 506 return 507 case p.processingQueue <- processingResponse: 508 } 509 wg.Add(1) 510 go func(e *event, response chan<- *watch.Event) { 511 defer wg.Done() 512 select { 513 case <-ctx.Done(): 514 case response <- p.processFunc(e): 515 } 516 }(e, processingResponse) 517 } 518 } 519 520 func (p *concurrentOrderedEventProcessing) collectEventProcessing(ctx context.Context) { 521 var processingResponse chan *watch.Event 522 var e *watch.Event 523 for { 524 select { 525 case <-ctx.Done(): 526 return 527 case processingResponse = <-p.processingQueue: 528 } 529 select { 530 case <-ctx.Done(): 531 return 532 case e = <-processingResponse: 533 } 534 if e == nil { 535 continue 536 } 537 if len(p.output) == cap(p.output) { 538 klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", p.objectType, "groupResource", p.groupResource) 539 } 540 // If user couldn't receive results fast enough, we also block incoming events from watcher. 541 // Because storing events in local will cause more memory usage. 542 // The worst case would be closing the fast watcher. 543 select { 544 case <-ctx.Done(): 545 return 546 case p.output <- *e: 547 } 548 } 549 } 550 551 func (wc *watchChan) filter(obj runtime.Object) bool { 552 if wc.internalPred.Empty() { 553 return true 554 } 555 matched, err := wc.internalPred.Matches(obj) 556 return err == nil && matched 557 } 558 559 func (wc *watchChan) acceptAll() bool { 560 return wc.internalPred.Empty() 561 } 562 563 // transform transforms an event into a result for user if not filtered. 564 func (wc *watchChan) transform(e *event) (res *watch.Event) { 565 curObj, oldObj, err := wc.prepareObjs(e) 566 if err != nil { 567 klog.Errorf("failed to prepare current and previous objects: %v", err) 568 wc.sendError(err) 569 return nil 570 } 571 572 switch { 573 case e.isProgressNotify: 574 object := wc.watcher.newFunc() 575 if err := wc.watcher.versioner.UpdateObject(object, uint64(e.rev)); err != nil { 576 klog.Errorf("failed to propagate object version: %v", err) 577 return nil 578 } 579 if e.isInitialEventsEndBookmark { 580 if err := storage.AnnotateInitialEventsEndBookmark(object); err != nil { 581 wc.sendError(fmt.Errorf("error while accessing object's metadata gr: %v, type: %v, obj: %#v, err: %v", wc.watcher.groupResource, wc.watcher.objectType, object, err)) 582 return nil 583 } 584 } 585 res = &watch.Event{ 586 Type: watch.Bookmark, 587 Object: object, 588 } 589 case e.isDeleted: 590 if !wc.filter(oldObj) { 591 return nil 592 } 593 res = &watch.Event{ 594 Type: watch.Deleted, 595 Object: oldObj, 596 } 597 case e.isCreated: 598 if !wc.filter(curObj) { 599 return nil 600 } 601 res = &watch.Event{ 602 Type: watch.Added, 603 Object: curObj, 604 } 605 default: 606 if wc.acceptAll() { 607 res = &watch.Event{ 608 Type: watch.Modified, 609 Object: curObj, 610 } 611 return res 612 } 613 curObjPasses := wc.filter(curObj) 614 oldObjPasses := wc.filter(oldObj) 615 switch { 616 case curObjPasses && oldObjPasses: 617 res = &watch.Event{ 618 Type: watch.Modified, 619 Object: curObj, 620 } 621 case curObjPasses && !oldObjPasses: 622 res = &watch.Event{ 623 Type: watch.Added, 624 Object: curObj, 625 } 626 case !curObjPasses && oldObjPasses: 627 res = &watch.Event{ 628 Type: watch.Deleted, 629 Object: oldObj, 630 } 631 } 632 } 633 return res 634 } 635 636 func transformErrorToEvent(err error) *watch.Event { 637 err = interpretWatchError(err) 638 if _, ok := err.(apierrors.APIStatus); !ok { 639 err = apierrors.NewInternalError(err) 640 } 641 status := err.(apierrors.APIStatus).Status() 642 return &watch.Event{ 643 Type: watch.Error, 644 Object: &status, 645 } 646 } 647 648 func (wc *watchChan) sendError(err error) { 649 select { 650 case wc.errChan <- err: 651 case <-wc.ctx.Done(): 652 } 653 } 654 655 func (wc *watchChan) sendEvent(e *event) { 656 if len(wc.incomingEventChan) == incomingBufSize { 657 klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow decoding, user not receiving fast, or other processing logic", "incomingEvents", incomingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource) 658 } 659 select { 660 case wc.incomingEventChan <- e: 661 case <-wc.ctx.Done(): 662 } 663 } 664 665 func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) { 666 if e.isProgressNotify { 667 // progressNotify events doesn't contain neither current nor previous object version, 668 return nil, nil, nil 669 } 670 671 if !e.isDeleted { 672 data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.value, authenticatedDataString(e.key)) 673 if err != nil { 674 return nil, nil, err 675 } 676 curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev) 677 if err != nil { 678 return nil, nil, err 679 } 680 } 681 // We need to decode prevValue, only if this is deletion event or 682 // the underlying filter doesn't accept all objects (otherwise we 683 // know that the filter for previous object will return true and 684 // we need the object only to compute whether it was filtered out 685 // before). 686 if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) { 687 data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.prevValue, authenticatedDataString(e.key)) 688 if err != nil { 689 return nil, nil, err 690 } 691 // Note that this sends the *old* object with the etcd revision for the time at 692 // which it gets deleted. 693 oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev) 694 if err != nil { 695 return nil, nil, err 696 } 697 } 698 return curObj, oldObj, nil 699 } 700 701 func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) { 702 obj, err := runtime.Decode(codec, []byte(data)) 703 if err != nil { 704 if fatalOnDecodeError { 705 // we are running in a test environment and thus an 706 // error here is due to a coder mistake if the defer 707 // does not catch it 708 panic(err) 709 } 710 return nil, err 711 } 712 // ensure resource version is set on the object we load from etcd 713 if err := versioner.UpdateObject(obj, uint64(rev)); err != nil { 714 return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err) 715 } 716 return obj, nil 717 }