github.com/Team-Kujira/tendermint@v0.34.24-indexer/light/client.go (about) 1 package light 2 3 import ( 4 "bytes" 5 "context" 6 "errors" 7 "fmt" 8 "sort" 9 "sync" 10 "time" 11 12 "github.com/tendermint/tendermint/libs/log" 13 tmmath "github.com/tendermint/tendermint/libs/math" 14 tmsync "github.com/tendermint/tendermint/libs/sync" 15 "github.com/tendermint/tendermint/light/provider" 16 "github.com/tendermint/tendermint/light/store" 17 "github.com/tendermint/tendermint/types" 18 ) 19 20 type mode byte 21 22 const ( 23 sequential mode = iota + 1 24 skipping 25 26 defaultPruningSize = 1000 27 defaultMaxRetryAttempts = 10 28 // For verifySkipping, when using the cache of headers from the previous batch, 29 // they will always be at a height greater than 1/2 (normal verifySkipping) so to 30 // find something in between the range, 9/16 is used. 31 verifySkippingNumerator = 9 32 verifySkippingDenominator = 16 33 34 // 10s should cover most of the clients. 35 // References: 36 // - http://vancouver-webpages.com/time/web.html 37 // - https://blog.codinghorror.com/keeping-time-on-the-pc/ 38 defaultMaxClockDrift = 10 * time.Second 39 40 // 10s is sufficient for most networks. 41 defaultMaxBlockLag = 10 * time.Second 42 ) 43 44 // Option sets a parameter for the light client. 45 type Option func(*Client) 46 47 // SequentialVerification option configures the light client to sequentially 48 // check the blocks (every block, in ascending height order). Note this is 49 // much slower than SkippingVerification, albeit more secure. 50 func SequentialVerification() Option { 51 return func(c *Client) { 52 c.verificationMode = sequential 53 } 54 } 55 56 // SkippingVerification option configures the light client to skip blocks as 57 // long as {trustLevel} of the old validator set signed the new header. The 58 // verifySkipping algorithm from the specification is used for finding the minimal 59 // "trust path". 60 // 61 // trustLevel - fraction of the old validator set (in terms of voting power), 62 // which must sign the new header in order for us to trust it. NOTE this only 63 // applies to non-adjacent headers. For adjacent headers, sequential 64 // verification is used. 65 func SkippingVerification(trustLevel tmmath.Fraction) Option { 66 return func(c *Client) { 67 c.verificationMode = skipping 68 c.trustLevel = trustLevel 69 } 70 } 71 72 // PruningSize option sets the maximum amount of light blocks that the light 73 // client stores. When Prune() is run, all light blocks that are earlier than 74 // the h amount of light blocks will be removed from the store. 75 // Default: 1000. A pruning size of 0 will not prune the light client at all. 76 func PruningSize(h uint16) Option { 77 return func(c *Client) { 78 c.pruningSize = h 79 } 80 } 81 82 // ConfirmationFunction option can be used to prompt to confirm an action. For 83 // example, remove newer headers if the light client is being reset with an 84 // older header. No confirmation is required by default! 85 func ConfirmationFunction(fn func(action string) bool) Option { 86 return func(c *Client) { 87 c.confirmationFn = fn 88 } 89 } 90 91 // Logger option can be used to set a logger for the client. 92 func Logger(l log.Logger) Option { 93 return func(c *Client) { 94 c.logger = l 95 } 96 } 97 98 // MaxRetryAttempts option can be used to set max attempts before replacing 99 // primary with a witness. 100 func MaxRetryAttempts(max uint16) Option { 101 return func(c *Client) { 102 c.maxRetryAttempts = max 103 } 104 } 105 106 // MaxClockDrift defines how much new header's time can drift into 107 // the future relative to the light clients local time. Default: 10s. 108 func MaxClockDrift(d time.Duration) Option { 109 return func(c *Client) { 110 c.maxClockDrift = d 111 } 112 } 113 114 // MaxBlockLag represents the maximum time difference between the realtime 115 // that a block is received and the timestamp of that block. 116 // One can approximate it to the maximum block production time 117 // 118 // As an example, say the light client received block B at a time 119 // 12:05 (this is the real time) and the time on the block 120 // was 12:00. Then the lag here is 5 minutes. 121 // Default: 10s 122 func MaxBlockLag(d time.Duration) Option { 123 return func(c *Client) { 124 c.maxBlockLag = d 125 } 126 } 127 128 // Client represents a light client, connected to a single chain, which gets 129 // light blocks from a primary provider, verifies them either sequentially or by 130 // skipping some and stores them in a trusted store (usually, a local FS). 131 // 132 // Default verification: SkippingVerification(DefaultTrustLevel) 133 type Client struct { 134 chainID string 135 trustingPeriod time.Duration // see TrustOptions.Period 136 verificationMode mode 137 trustLevel tmmath.Fraction 138 maxRetryAttempts uint16 // see MaxRetryAttempts option 139 maxClockDrift time.Duration 140 maxBlockLag time.Duration 141 142 // Mutex for locking during changes of the light clients providers 143 providerMutex tmsync.Mutex 144 // Primary provider of new headers. 145 primary provider.Provider 146 // Providers used to "witness" new headers. 147 witnesses []provider.Provider 148 149 // Where trusted light blocks are stored. 150 trustedStore store.Store 151 // Highest trusted light block from the store (height=H). 152 latestTrustedBlock *types.LightBlock 153 154 // See RemoveNoLongerTrustedHeadersPeriod option 155 pruningSize uint16 156 // See ConfirmationFunction option 157 confirmationFn func(action string) bool 158 159 quit chan struct{} 160 161 logger log.Logger 162 } 163 164 // NewClient returns a new light client. It returns an error if it fails to 165 // obtain the light block from the primary or they are invalid (e.g. trust 166 // hash does not match with the one from the headers). 167 // 168 // Witnesses are providers, which will be used for cross-checking the primary 169 // provider. At least one witness must be given when skipping verification is 170 // used (default). A witness can become a primary iff the current primary is 171 // unavailable. 172 // 173 // See all Option(s) for the additional configuration. 174 func NewClient( 175 ctx context.Context, 176 chainID string, 177 trustOptions TrustOptions, 178 primary provider.Provider, 179 witnesses []provider.Provider, 180 trustedStore store.Store, 181 options ...Option) (*Client, error) { 182 183 if err := trustOptions.ValidateBasic(); err != nil { 184 return nil, fmt.Errorf("invalid TrustOptions: %w", err) 185 } 186 187 c, err := NewClientFromTrustedStore(chainID, trustOptions.Period, primary, witnesses, trustedStore, options...) 188 if err != nil { 189 return nil, err 190 } 191 192 if c.latestTrustedBlock != nil { 193 c.logger.Info("Checking trusted light block using options") 194 if err := c.checkTrustedHeaderUsingOptions(ctx, trustOptions); err != nil { 195 return nil, err 196 } 197 } 198 199 if c.latestTrustedBlock == nil || c.latestTrustedBlock.Height < trustOptions.Height { 200 c.logger.Info("Downloading trusted light block using options") 201 if err := c.initializeWithTrustOptions(ctx, trustOptions); err != nil { 202 return nil, err 203 } 204 } 205 206 return c, err 207 } 208 209 // NewClientFromTrustedStore initializes existing client from the trusted store. 210 // 211 // See NewClient 212 func NewClientFromTrustedStore( 213 chainID string, 214 trustingPeriod time.Duration, 215 primary provider.Provider, 216 witnesses []provider.Provider, 217 trustedStore store.Store, 218 options ...Option) (*Client, error) { 219 220 c := &Client{ 221 chainID: chainID, 222 trustingPeriod: trustingPeriod, 223 verificationMode: skipping, 224 trustLevel: DefaultTrustLevel, 225 maxRetryAttempts: defaultMaxRetryAttempts, 226 maxClockDrift: defaultMaxClockDrift, 227 maxBlockLag: defaultMaxBlockLag, 228 primary: primary, 229 witnesses: witnesses, 230 trustedStore: trustedStore, 231 pruningSize: defaultPruningSize, 232 confirmationFn: func(action string) bool { return true }, 233 quit: make(chan struct{}), 234 logger: log.NewNopLogger(), 235 } 236 237 for _, o := range options { 238 o(c) 239 } 240 241 // Validate the number of witnesses. 242 if len(c.witnesses) == 0 { 243 return nil, ErrNoWitnesses 244 } 245 246 // Verify witnesses are all on the same chain. 247 for i, w := range witnesses { 248 if w.ChainID() != chainID { 249 return nil, fmt.Errorf("witness #%d: %v is on another chain %s, expected %s", 250 i, w, w.ChainID(), chainID) 251 } 252 } 253 254 // Validate trust level. 255 if err := ValidateTrustLevel(c.trustLevel); err != nil { 256 return nil, err 257 } 258 259 if err := c.restoreTrustedLightBlock(); err != nil { 260 return nil, err 261 } 262 263 return c, nil 264 } 265 266 // restoreTrustedLightBlock loads the latest trusted light block from the store 267 func (c *Client) restoreTrustedLightBlock() error { 268 lastHeight, err := c.trustedStore.LastLightBlockHeight() 269 if err != nil { 270 return fmt.Errorf("can't get last trusted light block height: %w", err) 271 } 272 273 if lastHeight > 0 { 274 trustedBlock, err := c.trustedStore.LightBlock(lastHeight) 275 if err != nil { 276 return fmt.Errorf("can't get last trusted light block: %w", err) 277 } 278 c.latestTrustedBlock = trustedBlock 279 c.logger.Info("Restored trusted light block", "height", lastHeight) 280 } 281 282 return nil 283 } 284 285 // if options.Height: 286 // 287 // 1. ahead of trustedLightBlock.Height => fetch light blocks (same height as 288 // trustedLightBlock) from primary provider and check it's hash matches the 289 // trustedLightBlock's hash (if not, remove trustedLightBlock and all the light blocks 290 // before) 291 // 292 // 2. equals trustedLightBlock.Height => check options.Hash matches the 293 // trustedLightBlock's hash (if not, remove trustedLightBlock and all the light blocks 294 // before) 295 // 296 // 3. behind trustedLightBlock.Height => remove all the light blocks between 297 // options.Height and trustedLightBlock.Height, update trustedLightBlock, then 298 // check options.Hash matches the trustedLightBlock's hash (if not, remove 299 // trustedLightBlock and all the light blocks before) 300 // 301 // The intuition here is the user is always right. I.e. if she decides to reset 302 // the light client with an older header, there must be a reason for it. 303 func (c *Client) checkTrustedHeaderUsingOptions(ctx context.Context, options TrustOptions) error { 304 var primaryHash []byte 305 switch { 306 case options.Height > c.latestTrustedBlock.Height: 307 h, err := c.lightBlockFromPrimary(ctx, c.latestTrustedBlock.Height) 308 if err != nil { 309 return err 310 } 311 primaryHash = h.Hash() 312 case options.Height == c.latestTrustedBlock.Height: 313 primaryHash = options.Hash 314 case options.Height < c.latestTrustedBlock.Height: 315 c.logger.Info("Client initialized with old header (trusted is more recent)", 316 "old", options.Height, 317 "trustedHeight", c.latestTrustedBlock.Height, 318 "trustedHash", c.latestTrustedBlock.Hash()) 319 320 action := fmt.Sprintf( 321 "Rollback to %d (%X)? Note this will remove newer light blocks up to %d (%X)", 322 options.Height, options.Hash, 323 c.latestTrustedBlock.Height, c.latestTrustedBlock.Hash()) 324 if c.confirmationFn(action) { 325 // remove all the headers (options.Height, trustedHeader.Height] 326 err := c.cleanupAfter(options.Height) 327 if err != nil { 328 return fmt.Errorf("cleanupAfter(%d): %w", options.Height, err) 329 } 330 331 c.logger.Info("Rolled back to older header (newer headers were removed)", 332 "old", options.Height) 333 } else { 334 return nil 335 } 336 337 primaryHash = options.Hash 338 } 339 340 if !bytes.Equal(primaryHash, c.latestTrustedBlock.Hash()) { 341 c.logger.Info("Prev. trusted header's hash (h1) doesn't match hash from primary provider (h2)", 342 "h1", c.latestTrustedBlock.Hash(), "h2", primaryHash) 343 344 action := fmt.Sprintf( 345 "Prev. trusted header's hash %X doesn't match hash %X from primary provider. Remove all the stored light blocks?", 346 c.latestTrustedBlock.Hash(), primaryHash) 347 if c.confirmationFn(action) { 348 err := c.Cleanup() 349 if err != nil { 350 return fmt.Errorf("failed to cleanup: %w", err) 351 } 352 } else { 353 return errors.New("refused to remove the stored light blocks despite hashes mismatch") 354 } 355 } 356 357 return nil 358 } 359 360 // initializeWithTrustOptions fetches the weakly-trusted light block from 361 // primary provider. 362 func (c *Client) initializeWithTrustOptions(ctx context.Context, options TrustOptions) error { 363 // 1) Fetch and verify the light block. 364 l, err := c.lightBlockFromPrimary(ctx, options.Height) 365 if err != nil { 366 return err 367 } 368 369 // NOTE: - Verify func will check if it's expired or not. 370 // - h.Time is not being checked against time.Now() because we don't 371 // want to add yet another argument to NewClient* functions. 372 if err := l.ValidateBasic(c.chainID); err != nil { 373 return err 374 } 375 376 if !bytes.Equal(l.Hash(), options.Hash) { 377 return fmt.Errorf("expected header's hash %X, but got %X", options.Hash, l.Hash()) 378 } 379 380 // 2) Ensure that +2/3 of validators signed correctly. 381 err = l.ValidatorSet.VerifyCommitLight(c.chainID, l.Commit.BlockID, l.Height, l.Commit) 382 if err != nil { 383 return fmt.Errorf("invalid commit: %w", err) 384 } 385 386 // 3) Cross-verify with witnesses to ensure everybody has the same state. 387 if err := c.compareFirstHeaderWithWitnesses(ctx, l.SignedHeader); err != nil { 388 return err 389 } 390 391 // 4) Persist both of them and continue. 392 return c.updateTrustedLightBlock(l) 393 } 394 395 // TrustedLightBlock returns a trusted light block at the given height (0 - the latest). 396 // 397 // It returns an error if: 398 // - there are some issues with the trusted store, although that should not 399 // happen normally; 400 // - negative height is passed; 401 // - header has not been verified yet and is therefore not in the store 402 // 403 // Safe for concurrent use by multiple goroutines. 404 func (c *Client) TrustedLightBlock(height int64) (*types.LightBlock, error) { 405 height, err := c.compareWithLatestHeight(height) 406 if err != nil { 407 return nil, err 408 } 409 return c.trustedStore.LightBlock(height) 410 } 411 412 func (c *Client) compareWithLatestHeight(height int64) (int64, error) { 413 latestHeight, err := c.LastTrustedHeight() 414 if err != nil { 415 return 0, fmt.Errorf("can't get last trusted height: %w", err) 416 } 417 if latestHeight == -1 { 418 return 0, errors.New("no headers exist") 419 } 420 421 switch { 422 case height > latestHeight: 423 return 0, fmt.Errorf("unverified header/valset requested (latest: %d)", latestHeight) 424 case height == 0: 425 return latestHeight, nil 426 case height < 0: 427 return 0, errors.New("negative height") 428 } 429 430 return height, nil 431 } 432 433 // Update attempts to advance the state by downloading the latest light 434 // block and verifying it. It returns a new light block on a successful 435 // update. Otherwise, it returns nil (plus an error, if any). 436 func (c *Client) Update(ctx context.Context, now time.Time) (*types.LightBlock, error) { 437 lastTrustedHeight, err := c.LastTrustedHeight() 438 if err != nil { 439 return nil, fmt.Errorf("can't get last trusted height: %w", err) 440 } 441 442 if lastTrustedHeight == -1 { 443 // no light blocks yet => wait 444 return nil, nil 445 } 446 447 latestBlock, err := c.lightBlockFromPrimary(ctx, 0) 448 if err != nil { 449 return nil, err 450 } 451 452 if latestBlock.Height > lastTrustedHeight { 453 err = c.verifyLightBlock(ctx, latestBlock, now) 454 if err != nil { 455 return nil, err 456 } 457 c.logger.Info("Advanced to new state", "height", latestBlock.Height, "hash", latestBlock.Hash()) 458 return latestBlock, nil 459 } 460 461 return nil, nil 462 } 463 464 // VerifyLightBlockAtHeight fetches the light block at the given height 465 // and verifies it. It returns the block immediately if it exists in 466 // the trustedStore (no verification is needed). 467 // 468 // height must be > 0. 469 // 470 // It returns provider.ErrlightBlockNotFound if light block is not found by 471 // primary. 472 // 473 // It will replace the primary provider if an error from a request to the provider occurs 474 func (c *Client) VerifyLightBlockAtHeight(ctx context.Context, height int64, now time.Time) (*types.LightBlock, error) { 475 if height <= 0 { 476 return nil, errors.New("negative or zero height") 477 } 478 479 // Check if the light block already verified. 480 h, err := c.TrustedLightBlock(height) 481 if err == nil { 482 c.logger.Info("Header has already been verified", "height", height, "hash", h.Hash()) 483 // Return already trusted light block 484 return h, nil 485 } 486 487 // Request the light block from primary 488 l, err := c.lightBlockFromPrimary(ctx, height) 489 if err != nil { 490 return nil, err 491 } 492 493 return l, c.verifyLightBlock(ctx, l, now) 494 } 495 496 // VerifyHeader verifies a new header against the trusted state. It returns 497 // immediately if newHeader exists in trustedStore (no verification is 498 // needed). Else it performs one of the two types of verification: 499 // 500 // SequentialVerification: verifies that 2/3 of the trusted validator set has 501 // signed the new header. If the headers are not adjacent, **all** intermediate 502 // headers will be requested. Intermediate headers are not saved to database. 503 // 504 // SkippingVerification(trustLevel): verifies that {trustLevel} of the trusted 505 // validator set has signed the new header. If it's not the case and the 506 // headers are not adjacent, verifySkipping is performed and necessary (not all) 507 // intermediate headers will be requested. See the specification for details. 508 // Intermediate headers are not saved to database. 509 // https://github.com/tendermint/tendermint/blob/v0.34.x/spec/consensus/light-client.md 510 // 511 // If the header, which is older than the currently trusted header, is 512 // requested and the light client does not have it, VerifyHeader will perform: 513 // 514 // a) verifySkipping verification if nearest trusted header is found & not expired 515 // b) backwards verification in all other cases 516 // 517 // It returns ErrOldHeaderExpired if the latest trusted header expired. 518 // 519 // If the primary provides an invalid header (ErrInvalidHeader), it is rejected 520 // and replaced by another provider until all are exhausted. 521 // 522 // If, at any moment, a LightBlock is not found by the primary provider as part of 523 // verification then the provider will be replaced by another and the process will 524 // restart. 525 func (c *Client) VerifyHeader(ctx context.Context, newHeader *types.Header, now time.Time) error { 526 if newHeader == nil { 527 return errors.New("nil header") 528 } 529 if newHeader.Height <= 0 { 530 return errors.New("negative or zero height") 531 } 532 533 // Check if newHeader already verified. 534 l, err := c.TrustedLightBlock(newHeader.Height) 535 if err == nil { 536 // Make sure it's the same header. 537 if !bytes.Equal(l.Hash(), newHeader.Hash()) { 538 return fmt.Errorf("existing trusted header %X does not match newHeader %X", l.Hash(), newHeader.Hash()) 539 } 540 c.logger.Info("Header has already been verified", 541 "height", newHeader.Height, "hash", newHeader.Hash()) 542 return nil 543 } 544 545 // Request the header and the vals. 546 l, err = c.lightBlockFromPrimary(ctx, newHeader.Height) 547 if err != nil { 548 return fmt.Errorf("failed to retrieve light block from primary to verify against: %w", err) 549 } 550 551 if !bytes.Equal(l.Hash(), newHeader.Hash()) { 552 return fmt.Errorf("light block header %X does not match newHeader %X", l.Hash(), newHeader.Hash()) 553 } 554 555 return c.verifyLightBlock(ctx, l, now) 556 } 557 558 func (c *Client) verifyLightBlock(ctx context.Context, newLightBlock *types.LightBlock, now time.Time) error { 559 c.logger.Info("VerifyHeader", "height", newLightBlock.Height, "hash", newLightBlock.Hash()) 560 561 var ( 562 verifyFunc func(ctx context.Context, trusted *types.LightBlock, new *types.LightBlock, now time.Time) error 563 err error 564 ) 565 566 switch c.verificationMode { 567 case sequential: 568 verifyFunc = c.verifySequential 569 case skipping: 570 verifyFunc = c.verifySkippingAgainstPrimary 571 default: 572 panic(fmt.Sprintf("Unknown verification mode: %b", c.verificationMode)) 573 } 574 575 firstBlockHeight, err := c.FirstTrustedHeight() 576 if err != nil { 577 return fmt.Errorf("can't get first light block height: %w", err) 578 } 579 580 switch { 581 // Verifying forwards 582 case newLightBlock.Height >= c.latestTrustedBlock.Height: 583 err = verifyFunc(ctx, c.latestTrustedBlock, newLightBlock, now) 584 585 // Verifying backwards 586 case newLightBlock.Height < firstBlockHeight: 587 var firstBlock *types.LightBlock 588 firstBlock, err = c.trustedStore.LightBlock(firstBlockHeight) 589 if err != nil { 590 return fmt.Errorf("can't get first light block: %w", err) 591 } 592 err = c.backwards(ctx, firstBlock.Header, newLightBlock.Header) 593 594 // Verifying between first and last trusted light block 595 default: 596 var closestBlock *types.LightBlock 597 closestBlock, err = c.trustedStore.LightBlockBefore(newLightBlock.Height) 598 if err != nil { 599 return fmt.Errorf("can't get signed header before height %d: %w", newLightBlock.Height, err) 600 } 601 err = verifyFunc(ctx, closestBlock, newLightBlock, now) 602 } 603 if err != nil { 604 c.logger.Error("Can't verify", "err", err) 605 return err 606 } 607 608 // Once verified, save and return 609 return c.updateTrustedLightBlock(newLightBlock) 610 } 611 612 // see VerifyHeader 613 func (c *Client) verifySequential( 614 ctx context.Context, 615 trustedBlock *types.LightBlock, 616 newLightBlock *types.LightBlock, 617 now time.Time) error { 618 619 var ( 620 verifiedBlock = trustedBlock 621 interimBlock *types.LightBlock 622 err error 623 trace = []*types.LightBlock{trustedBlock} 624 ) 625 626 for height := trustedBlock.Height + 1; height <= newLightBlock.Height; height++ { 627 // 1) Fetch interim light block if needed. 628 if height == newLightBlock.Height { // last light block 629 interimBlock = newLightBlock 630 } else { // intermediate light blocks 631 interimBlock, err = c.lightBlockFromPrimary(ctx, height) 632 if err != nil { 633 return ErrVerificationFailed{From: verifiedBlock.Height, To: height, Reason: err} 634 } 635 } 636 637 // 2) Verify them 638 c.logger.Debug("Verify adjacent newLightBlock against verifiedBlock", 639 "trustedHeight", verifiedBlock.Height, 640 "trustedHash", verifiedBlock.Hash(), 641 "newHeight", interimBlock.Height, 642 "newHash", interimBlock.Hash()) 643 644 err = VerifyAdjacent(verifiedBlock.SignedHeader, interimBlock.SignedHeader, interimBlock.ValidatorSet, 645 c.trustingPeriod, now, c.maxClockDrift) 646 if err != nil { 647 err := ErrVerificationFailed{From: verifiedBlock.Height, To: interimBlock.Height, Reason: err} 648 649 switch errors.Unwrap(err).(type) { 650 case ErrInvalidHeader: 651 // If the target header is invalid, return immediately. 652 if err.To == newLightBlock.Height { 653 c.logger.Debug("Target header is invalid", "err", err) 654 return err 655 } 656 657 // If some intermediate header is invalid, replace the primary and try 658 // again. 659 c.logger.Error("primary sent invalid header -> replacing", "err", err) 660 661 replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true) 662 if removeErr != nil { 663 c.logger.Debug("failed to replace primary. Returning original error", "err", removeErr) 664 return err 665 } 666 667 if !bytes.Equal(replacementBlock.Hash(), newLightBlock.Hash()) { 668 c.logger.Error("Replacement provider has a different light block", 669 "newHash", newLightBlock.Hash(), 670 "replHash", replacementBlock.Hash()) 671 // return original error 672 return err 673 } 674 675 // attempt to verify header again 676 height-- 677 678 continue 679 default: 680 return err 681 } 682 } 683 684 // 3) Update verifiedBlock 685 verifiedBlock = interimBlock 686 687 // 4) Add verifiedBlock to trace 688 trace = append(trace, verifiedBlock) 689 } 690 691 // Compare header with the witnesses to ensure it's not a fork. 692 // More witnesses we have, more chance to notice one. 693 // 694 // CORRECTNESS ASSUMPTION: there's at least 1 correct full node 695 // (primary or one of the witnesses). 696 return c.detectDivergence(ctx, trace, now) 697 } 698 699 // see VerifyHeader 700 // 701 // verifySkipping finds the middle light block between a trusted and new light block, 702 // reiterating the action until it verifies a light block. A cache of light blocks 703 // requested from source is kept such that when a verification is made, and the 704 // light client tries again to verify the new light block in the middle, the light 705 // client does not need to ask for all the same light blocks again. 706 func (c *Client) verifySkipping( 707 ctx context.Context, 708 source provider.Provider, 709 trustedBlock *types.LightBlock, 710 newLightBlock *types.LightBlock, 711 now time.Time) ([]*types.LightBlock, error) { 712 713 var ( 714 blockCache = []*types.LightBlock{newLightBlock} 715 depth = 0 716 717 verifiedBlock = trustedBlock 718 trace = []*types.LightBlock{trustedBlock} 719 ) 720 721 for { 722 c.logger.Debug("Verify non-adjacent newHeader against verifiedBlock", 723 "trustedHeight", verifiedBlock.Height, 724 "trustedHash", verifiedBlock.Hash(), 725 "newHeight", blockCache[depth].Height, 726 "newHash", blockCache[depth].Hash()) 727 728 err := Verify(verifiedBlock.SignedHeader, verifiedBlock.ValidatorSet, blockCache[depth].SignedHeader, 729 blockCache[depth].ValidatorSet, c.trustingPeriod, now, c.maxClockDrift, c.trustLevel) 730 switch err.(type) { 731 case nil: 732 // Have we verified the last header 733 if depth == 0 { 734 trace = append(trace, newLightBlock) 735 return trace, nil 736 } 737 // If not, update the lower bound to the previous upper bound 738 verifiedBlock = blockCache[depth] 739 // Remove the light block at the lower bound in the header cache - it will no longer be needed 740 blockCache = blockCache[:depth] 741 // Reset the cache depth so that we start from the upper bound again 742 depth = 0 743 // add verifiedBlock to the trace 744 trace = append(trace, verifiedBlock) 745 746 case ErrNewValSetCantBeTrusted: 747 // do add another header to the end of the cache 748 if depth == len(blockCache)-1 { 749 pivotHeight := verifiedBlock.Height + (blockCache[depth].Height-verifiedBlock. 750 Height)*verifySkippingNumerator/verifySkippingDenominator 751 interimBlock, providerErr := source.LightBlock(ctx, pivotHeight) 752 switch providerErr { 753 case nil: 754 blockCache = append(blockCache, interimBlock) 755 756 // if the error is benign, the client does not need to replace the primary 757 case provider.ErrLightBlockNotFound, provider.ErrNoResponse, provider.ErrHeightTooHigh: 758 return nil, err 759 760 // all other errors such as ErrBadLightBlock or ErrUnreliableProvider are seen as malevolent and the 761 // provider is removed 762 default: 763 return nil, ErrVerificationFailed{From: verifiedBlock.Height, To: pivotHeight, Reason: providerErr} 764 } 765 blockCache = append(blockCache, interimBlock) 766 } 767 depth++ 768 769 default: 770 return nil, ErrVerificationFailed{From: verifiedBlock.Height, To: blockCache[depth].Height, Reason: err} 771 } 772 } 773 } 774 775 // verifySkippingAgainstPrimary does verifySkipping plus it compares new header with 776 // witnesses and replaces primary if it sends the light client an invalid header 777 func (c *Client) verifySkippingAgainstPrimary( 778 ctx context.Context, 779 trustedBlock *types.LightBlock, 780 newLightBlock *types.LightBlock, 781 now time.Time) error { 782 783 trace, err := c.verifySkipping(ctx, c.primary, trustedBlock, newLightBlock, now) 784 785 switch errors.Unwrap(err).(type) { 786 case ErrInvalidHeader: 787 // If the target header is invalid, return immediately. 788 invalidHeaderHeight := err.(ErrVerificationFailed).To 789 if invalidHeaderHeight == newLightBlock.Height { 790 c.logger.Debug("Target header is invalid", "err", err) 791 return err 792 } 793 794 // If some intermediate header is invalid, replace the primary and try 795 // again. 796 c.logger.Error("primary sent invalid header -> replacing", "err", err) 797 798 replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true) 799 if removeErr != nil { 800 c.logger.Error("failed to replace primary. Returning original error", "err", removeErr) 801 return err 802 } 803 804 if !bytes.Equal(replacementBlock.Hash(), newLightBlock.Hash()) { 805 c.logger.Error("Replacement provider has a different light block", 806 "newHash", newLightBlock.Hash(), 807 "replHash", replacementBlock.Hash()) 808 // return original error 809 return err 810 } 811 812 // attempt to verify the header again 813 return c.verifySkippingAgainstPrimary(ctx, trustedBlock, replacementBlock, now) 814 case nil: 815 // Compare header with the witnesses to ensure it's not a fork. 816 // More witnesses we have, more chance to notice one. 817 // 818 // CORRECTNESS ASSUMPTION: there's at least 1 correct full node 819 // (primary or one of the witnesses). 820 if cmpErr := c.detectDivergence(ctx, trace, now); cmpErr != nil { 821 return cmpErr 822 } 823 default: 824 return err 825 } 826 827 return nil 828 } 829 830 // LastTrustedHeight returns a last trusted height. -1 and nil are returned if 831 // there are no trusted headers. 832 // 833 // Safe for concurrent use by multiple goroutines. 834 func (c *Client) LastTrustedHeight() (int64, error) { 835 return c.trustedStore.LastLightBlockHeight() 836 } 837 838 // FirstTrustedHeight returns a first trusted height. -1 and nil are returned if 839 // there are no trusted headers. 840 // 841 // Safe for concurrent use by multiple goroutines. 842 func (c *Client) FirstTrustedHeight() (int64, error) { 843 return c.trustedStore.FirstLightBlockHeight() 844 } 845 846 // ChainID returns the chain ID the light client was configured with. 847 // 848 // Safe for concurrent use by multiple goroutines. 849 func (c *Client) ChainID() string { 850 return c.chainID 851 } 852 853 // Primary returns the primary provider. 854 // 855 // NOTE: provider may be not safe for concurrent access. 856 func (c *Client) Primary() provider.Provider { 857 c.providerMutex.Lock() 858 defer c.providerMutex.Unlock() 859 return c.primary 860 } 861 862 // Witnesses returns the witness providers. 863 // 864 // NOTE: providers may be not safe for concurrent access. 865 func (c *Client) Witnesses() []provider.Provider { 866 c.providerMutex.Lock() 867 defer c.providerMutex.Unlock() 868 return c.witnesses 869 } 870 871 // Cleanup removes all the data (headers and validator sets) stored. Note: the 872 // client must be stopped at this point. 873 func (c *Client) Cleanup() error { 874 c.logger.Info("Removing all the data") 875 c.latestTrustedBlock = nil 876 return c.trustedStore.Prune(0) 877 } 878 879 // cleanupAfter deletes all headers & validator sets after +height+. It also 880 // resets latestTrustedBlock to the latest header. 881 func (c *Client) cleanupAfter(height int64) error { 882 prevHeight := c.latestTrustedBlock.Height 883 884 for { 885 h, err := c.trustedStore.LightBlockBefore(prevHeight) 886 if err == store.ErrLightBlockNotFound || (h != nil && h.Height <= height) { 887 break 888 } else if err != nil { 889 return fmt.Errorf("failed to get header before %d: %w", prevHeight, err) 890 } 891 892 err = c.trustedStore.DeleteLightBlock(h.Height) 893 if err != nil { 894 c.logger.Error("can't remove a trusted header & validator set", "err", err, 895 "height", h.Height) 896 } 897 898 prevHeight = h.Height 899 } 900 901 c.latestTrustedBlock = nil 902 err := c.restoreTrustedLightBlock() 903 if err != nil { 904 return err 905 } 906 907 return nil 908 } 909 910 func (c *Client) updateTrustedLightBlock(l *types.LightBlock) error { 911 c.logger.Debug("updating trusted light block", "light_block", l) 912 913 if err := c.trustedStore.SaveLightBlock(l); err != nil { 914 return fmt.Errorf("failed to save trusted header: %w", err) 915 } 916 917 if c.pruningSize > 0 { 918 if err := c.trustedStore.Prune(c.pruningSize); err != nil { 919 return fmt.Errorf("prune: %w", err) 920 } 921 } 922 923 if c.latestTrustedBlock == nil || l.Height > c.latestTrustedBlock.Height { 924 c.latestTrustedBlock = l 925 } 926 927 return nil 928 } 929 930 // backwards verification (see VerifyHeaderBackwards func in the spec) verifies 931 // headers before a trusted header. If a sent header is invalid the primary is 932 // replaced with another provider and the operation is repeated. 933 func (c *Client) backwards( 934 ctx context.Context, 935 trustedHeader *types.Header, 936 newHeader *types.Header) error { 937 938 var ( 939 verifiedHeader = trustedHeader 940 interimHeader *types.Header 941 ) 942 943 for verifiedHeader.Height > newHeader.Height { 944 interimBlock, err := c.lightBlockFromPrimary(ctx, verifiedHeader.Height-1) 945 if err != nil { 946 return fmt.Errorf("failed to obtain the header at height #%d: %w", verifiedHeader.Height-1, err) 947 } 948 interimHeader = interimBlock.Header 949 c.logger.Debug("Verify newHeader against verifiedHeader", 950 "trustedHeight", verifiedHeader.Height, 951 "trustedHash", verifiedHeader.Hash(), 952 "newHeight", interimHeader.Height, 953 "newHash", interimHeader.Hash()) 954 if err := VerifyBackwards(interimHeader, verifiedHeader); err != nil { 955 // verification has failed 956 c.logger.Error("backwards verification failed, replacing primary...", "err", err, "primary", c.primary) 957 958 // the client tries to see if it can get a witness to continue with the request 959 newPrimarysBlock, replaceErr := c.findNewPrimary(ctx, newHeader.Height, true) 960 if replaceErr != nil { 961 c.logger.Debug("failed to replace primary. Returning original error", "err", replaceErr) 962 return err 963 } 964 965 // before continuing we must check that they have the same target header to validate 966 if !bytes.Equal(newPrimarysBlock.Hash(), newHeader.Hash()) { 967 c.logger.Debug("replaced primary but new primary has a different block to the initial one") 968 // return the original error 969 return err 970 } 971 972 // try again with the new primary 973 return c.backwards(ctx, verifiedHeader, newPrimarysBlock.Header) 974 } 975 verifiedHeader = interimHeader 976 } 977 978 return nil 979 } 980 981 // lightBlockFromPrimary retrieves the lightBlock from the primary provider 982 // at the specified height. This method also handles provider behavior as follows: 983 // 984 // 1. If the provider does not respond or does not have the block, it tries again 985 // with a different provider 986 // 2. If all providers return the same error, the light client forwards the error to 987 // where the initial request came from 988 // 3. If the provider provides an invalid light block, is deemed unreliable or returns 989 // any other error, the primary is permanently dropped and is replaced by a witness. 990 func (c *Client) lightBlockFromPrimary(ctx context.Context, height int64) (*types.LightBlock, error) { 991 c.providerMutex.Lock() 992 l, err := c.primary.LightBlock(ctx, height) 993 c.providerMutex.Unlock() 994 995 switch err { 996 case nil: 997 // Everything went smoothly. We reset the lightBlockRequests and return the light block 998 return l, nil 999 1000 case context.Canceled, context.DeadlineExceeded: 1001 return l, err 1002 1003 case provider.ErrNoResponse, provider.ErrLightBlockNotFound, provider.ErrHeightTooHigh: 1004 // we find a new witness to replace the primary 1005 c.logger.Info("error from light block request from primary, replacing...", 1006 "error", err, "height", height, "primary", c.primary) 1007 return c.findNewPrimary(ctx, height, false) 1008 1009 default: 1010 // The light client has most likely received either provider.ErrUnreliableProvider or provider.ErrBadLightBlock 1011 // These errors mean that the light client should drop the primary and try with another provider instead 1012 c.logger.Info("error from light block request from primary, removing...", 1013 "error", err, "height", height, "primary", c.primary) 1014 return c.findNewPrimary(ctx, height, true) 1015 } 1016 } 1017 1018 // NOTE: requires a providerMutex lock 1019 func (c *Client) removeWitnesses(indexes []int) error { 1020 // check that we will still have witnesses remaining 1021 if len(c.witnesses) <= len(indexes) { 1022 return ErrNoWitnesses 1023 } 1024 1025 // we need to make sure that we remove witnesses by index in the reverse 1026 // order so as to not affect the indexes themselves 1027 sort.Ints(indexes) 1028 for i := len(indexes) - 1; i >= 0; i-- { 1029 c.witnesses[indexes[i]] = c.witnesses[len(c.witnesses)-1] 1030 c.witnesses = c.witnesses[:len(c.witnesses)-1] 1031 } 1032 1033 return nil 1034 } 1035 1036 type witnessResponse struct { 1037 lb *types.LightBlock 1038 witnessIndex int 1039 err error 1040 } 1041 1042 // findNewPrimary concurrently sends a light block request, promoting the first witness to return 1043 // a valid light block as the new primary. The remove option indicates whether the primary should be 1044 // entire removed or just appended to the back of the witnesses list. This method also handles witness 1045 // errors. If no witness is available, it returns the last error of the witness. 1046 func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) (*types.LightBlock, error) { 1047 c.providerMutex.Lock() 1048 defer c.providerMutex.Unlock() 1049 1050 if len(c.witnesses) == 0 { 1051 return nil, ErrNoWitnesses 1052 } 1053 1054 var ( 1055 witnessResponsesC = make(chan witnessResponse, len(c.witnesses)) 1056 witnessesToRemove []int 1057 lastError error 1058 wg sync.WaitGroup 1059 ) 1060 1061 // send out a light block request to all witnesses 1062 subctx, cancel := context.WithCancel(ctx) 1063 defer cancel() 1064 for index := range c.witnesses { 1065 wg.Add(1) 1066 go func(witnessIndex int, witnessResponsesC chan witnessResponse) { 1067 defer wg.Done() 1068 1069 lb, err := c.witnesses[witnessIndex].LightBlock(subctx, height) 1070 witnessResponsesC <- witnessResponse{lb, witnessIndex, err} 1071 }(index, witnessResponsesC) 1072 } 1073 1074 // process all the responses as they come in 1075 for i := 0; i < cap(witnessResponsesC); i++ { 1076 response := <-witnessResponsesC 1077 switch response.err { 1078 // success! We have found a new primary 1079 case nil: 1080 cancel() // cancel all remaining requests to other witnesses 1081 1082 wg.Wait() // wait for all goroutines to finish 1083 1084 // if we are not intending on removing the primary then append the old primary to the end of the witness slice 1085 if !remove { 1086 c.witnesses = append(c.witnesses, c.primary) 1087 } 1088 1089 // promote respondent as the new primary 1090 c.logger.Debug("found new primary", "primary", c.witnesses[response.witnessIndex]) 1091 c.primary = c.witnesses[response.witnessIndex] 1092 1093 // add promoted witness to the list of witnesses to be removed 1094 witnessesToRemove = append(witnessesToRemove, response.witnessIndex) 1095 1096 // remove witnesses marked as bad (the client must do this before we alter the witness slice and change the indexes 1097 // of witnesses). Removal is done in descending order 1098 if err := c.removeWitnesses(witnessesToRemove); err != nil { 1099 return nil, err 1100 } 1101 1102 // return the light block that new primary responded with 1103 return response.lb, nil 1104 1105 // process benign errors by logging them only 1106 case provider.ErrNoResponse, provider.ErrLightBlockNotFound, provider.ErrHeightTooHigh: 1107 lastError = response.err 1108 c.logger.Debug("error on light block request from witness", 1109 "error", response.err, "primary", c.witnesses[response.witnessIndex]) 1110 continue 1111 1112 // process malevolent errors like ErrUnreliableProvider and ErrBadLightBlock by removing the witness 1113 default: 1114 lastError = response.err 1115 c.logger.Error("error on light block request from witness, removing...", 1116 "error", response.err, "primary", c.witnesses[response.witnessIndex]) 1117 witnessesToRemove = append(witnessesToRemove, response.witnessIndex) 1118 } 1119 } 1120 1121 // remove witnesses marked as bad. Removal is done in descending order 1122 if err := c.removeWitnesses(witnessesToRemove); err != nil { 1123 c.logger.Error("failed to remove witnesses", "err", err, "witnessesToRemove", witnessesToRemove) 1124 } 1125 1126 return nil, lastError 1127 } 1128 1129 // compareFirstHeaderWithWitnesses compares h with all witnesses. If any 1130 // witness reports a different header than h, the function returns an error. 1131 func (c *Client) compareFirstHeaderWithWitnesses(ctx context.Context, h *types.SignedHeader) error { 1132 compareCtx, cancel := context.WithCancel(ctx) 1133 defer cancel() 1134 1135 c.providerMutex.Lock() 1136 defer c.providerMutex.Unlock() 1137 1138 if len(c.witnesses) == 0 { 1139 return ErrNoWitnesses 1140 } 1141 1142 errc := make(chan error, len(c.witnesses)) 1143 for i, witness := range c.witnesses { 1144 go c.compareNewHeaderWithWitness(compareCtx, errc, h, witness, i) 1145 } 1146 1147 witnessesToRemove := make([]int, 0, len(c.witnesses)) 1148 1149 // handle errors from the header comparisons as they come in 1150 for i := 0; i < cap(errc); i++ { 1151 err := <-errc 1152 1153 switch e := err.(type) { 1154 case nil: 1155 continue 1156 case errConflictingHeaders: 1157 c.logger.Error(fmt.Sprintf(`Witness #%d has a different header. Please check primary is correct 1158 and remove witness. Otherwise, use the different primary`, e.WitnessIndex), "witness", c.witnesses[e.WitnessIndex]) 1159 return err 1160 case errBadWitness: 1161 // If witness sent us an invalid header, then remove it 1162 c.logger.Info("witness sent an invalid light block, removing...", 1163 "witness", c.witnesses[e.WitnessIndex], 1164 "err", err) 1165 witnessesToRemove = append(witnessesToRemove, e.WitnessIndex) 1166 default: // benign errors can be ignored with the exception of context errors 1167 if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 1168 return err 1169 } 1170 1171 // the witness either didn't respond or didn't have the block. We ignore it. 1172 c.logger.Info("error comparing first header with witness. You may want to consider removing the witness", 1173 "err", err) 1174 } 1175 1176 } 1177 1178 // remove witnesses that have misbehaved 1179 if err := c.removeWitnesses(witnessesToRemove); err != nil { 1180 c.logger.Error("failed to remove witnesses", "err", err, "witnessesToRemove", witnessesToRemove) 1181 } 1182 1183 return nil 1184 }