github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvserver/protectedts/ptcache/cache.go (about)

     1  // Copyright 2019 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package ptcache
    12  
    13  import (
    14  	"context"
    15  	"time"
    16  
    17  	"github.com/cockroachdb/cockroach/pkg/kv"
    18  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts"
    19  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb"
    20  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    21  	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
    22  	"github.com/cockroachdb/cockroach/pkg/util/hlc"
    23  	"github.com/cockroachdb/cockroach/pkg/util/log"
    24  	"github.com/cockroachdb/cockroach/pkg/util/stop"
    25  	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
    26  	"github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight"
    27  	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
    28  	"github.com/cockroachdb/cockroach/pkg/util/uuid"
    29  	"github.com/cockroachdb/errors"
    30  )
    31  
    32  // Cache implements protectedts.Cache.
    33  type Cache struct {
    34  	db       *kv.DB
    35  	storage  protectedts.Storage
    36  	stopper  *stop.Stopper
    37  	settings *cluster.Settings
    38  	sf       singleflight.Group
    39  	mu       struct {
    40  		syncutil.RWMutex
    41  
    42  		started bool
    43  
    44  		// Updated in doUpdate().
    45  		lastUpdate hlc.Timestamp
    46  		state      ptpb.State
    47  
    48  		// Updated in doUpdate but mutable. The records in the map are not mutated
    49  		// and should not be by any client.
    50  		recordsByID map[uuid.UUID]*ptpb.Record
    51  
    52  		// TODO(ajwerner): add a more efficient lookup structure such as an
    53  		// interval.Tree for Iterate.
    54  	}
    55  }
    56  
    57  // Config configures a Cache.
    58  type Config struct {
    59  	DB       *kv.DB
    60  	Storage  protectedts.Storage
    61  	Settings *cluster.Settings
    62  }
    63  
    64  // New returns a new cache.
    65  func New(config Config) *Cache {
    66  	c := &Cache{
    67  		db:       config.DB,
    68  		storage:  config.Storage,
    69  		settings: config.Settings,
    70  	}
    71  	c.mu.recordsByID = make(map[uuid.UUID]*ptpb.Record)
    72  	return c
    73  }
    74  
    75  var _ protectedts.Cache = (*Cache)(nil)
    76  
    77  // Iterate is part of the protectedts.Cache interface.
    78  func (c *Cache) Iterate(
    79  	_ context.Context, from, to roachpb.Key, it protectedts.Iterator,
    80  ) (asOf hlc.Timestamp) {
    81  	c.mu.RLock()
    82  	state, lastUpdate := c.mu.state, c.mu.lastUpdate
    83  	c.mu.RUnlock()
    84  
    85  	sp := roachpb.Span{
    86  		Key:    from,
    87  		EndKey: to,
    88  	}
    89  	for i := range state.Records {
    90  		r := &state.Records[i]
    91  		if !overlaps(r, sp) {
    92  			continue
    93  		}
    94  		if wantMore := it(r); !wantMore {
    95  			break
    96  		}
    97  	}
    98  	return lastUpdate
    99  }
   100  
   101  // QueryRecord is part of the protectedts.Cache interface.
   102  func (c *Cache) QueryRecord(_ context.Context, id uuid.UUID) (exists bool, asOf hlc.Timestamp) {
   103  	c.mu.RLock()
   104  	defer c.mu.RUnlock()
   105  
   106  	_, exists = c.mu.recordsByID[id]
   107  	return exists, c.mu.lastUpdate
   108  }
   109  
   110  // refreshKey is used for the singleflight.
   111  const refreshKey = ""
   112  
   113  // Refresh is part of the protectedts.Cache interface.
   114  func (c *Cache) Refresh(ctx context.Context, asOf hlc.Timestamp) error {
   115  	for !c.upToDate(asOf) {
   116  		ch, _ := c.sf.DoChan(refreshKey, c.doSingleFlightUpdate)
   117  		select {
   118  		case <-ctx.Done():
   119  			return ctx.Err()
   120  		case res := <-ch:
   121  			if res.Err != nil {
   122  				return res.Err
   123  			}
   124  		}
   125  	}
   126  	return nil
   127  }
   128  
   129  // Start starts the periodic fetching of the Cache. A Cache must not be used
   130  // until after it has been started. An error will be returned if it has
   131  // already been started.
   132  func (c *Cache) Start(ctx context.Context, stopper *stop.Stopper) error {
   133  	c.mu.Lock()
   134  	defer c.mu.Unlock()
   135  	if c.mu.started {
   136  		return errors.New("cannot start a Cache more than once")
   137  	}
   138  	c.mu.started = true
   139  	c.stopper = stopper
   140  	return c.stopper.RunAsyncTask(ctx, "periodically-refresh-protectedts-cache",
   141  		c.periodicallyRefreshProtectedtsCache)
   142  }
   143  
   144  func (c *Cache) periodicallyRefreshProtectedtsCache(ctx context.Context) {
   145  	settingChanged := make(chan struct{}, 1)
   146  	protectedts.PollInterval.SetOnChange(&c.settings.SV, func() {
   147  		select {
   148  		case settingChanged <- struct{}{}:
   149  		default:
   150  		}
   151  	})
   152  	timer := timeutil.NewTimer()
   153  	defer timer.Stop()
   154  	timer.Reset(0) // Read immediately upon startup
   155  	var lastReset time.Time
   156  	var doneCh <-chan singleflight.Result
   157  	// TODO(ajwerner): consider resetting the timer when the state is updated
   158  	// due to a call to Refresh.
   159  	for {
   160  		select {
   161  		case <-timer.C:
   162  			// Let's not reset the timer until we get our response.
   163  			timer.Read = true
   164  			doneCh, _ = c.sf.DoChan(refreshKey, c.doSingleFlightUpdate)
   165  		case <-settingChanged:
   166  			if timer.Read { // we're currently fetching
   167  				continue
   168  			}
   169  			interval := protectedts.PollInterval.Get(&c.settings.SV)
   170  			// NB: It's okay if nextUpdate is a negative duration; timer.Reset will
   171  			// treat a negative duration as zero and send a notification immediately.
   172  			nextUpdate := interval - timeutil.Since(lastReset)
   173  			timer.Reset(nextUpdate)
   174  			lastReset = timeutil.Now()
   175  		case res := <-doneCh:
   176  			if res.Err != nil {
   177  				if ctx.Err() == nil {
   178  					log.Errorf(ctx, "failed to refresh protected timestamps: %v", res.Err)
   179  				}
   180  			}
   181  			timer.Reset(protectedts.PollInterval.Get(&c.settings.SV))
   182  			lastReset = timeutil.Now()
   183  		case <-c.stopper.ShouldQuiesce():
   184  			return
   185  		}
   186  	}
   187  }
   188  
   189  func (c *Cache) doSingleFlightUpdate() (interface{}, error) {
   190  	// TODO(ajwerner): add log tags to the context.
   191  	ctx, cancel := c.stopper.WithCancelOnQuiesce(context.Background())
   192  	defer cancel()
   193  	return nil, c.stopper.RunTaskWithErr(ctx,
   194  		"refresh-protectedts-cache", c.doUpdate)
   195  }
   196  
   197  func (c *Cache) getMetadata() ptpb.Metadata {
   198  	c.mu.RLock()
   199  	defer c.mu.RUnlock()
   200  	return c.mu.state.Metadata
   201  }
   202  
   203  func (c *Cache) doUpdate(ctx context.Context) error {
   204  	// NB: doUpdate is only ever called underneath c.singleFlight and thus is
   205  	// never called concurrently. Due to the lack of concurrency there are no
   206  	// concerns about races as this is the only method which writes to the Cache's
   207  	// state.
   208  	prev := c.getMetadata()
   209  	var (
   210  		versionChanged bool
   211  		state          ptpb.State
   212  		ts             hlc.Timestamp
   213  	)
   214  	err := c.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) (err error) {
   215  		// NB: because this is a read-only transaction, the commit will be a no-op;
   216  		// returning nil here means the transaction will commit and will never need
   217  		// to change its read timestamp.
   218  		defer func() {
   219  			if err == nil {
   220  				ts = txn.ReadTimestamp()
   221  			}
   222  		}()
   223  		md, err := c.storage.GetMetadata(ctx, txn)
   224  		if err != nil {
   225  			return errors.Wrap(err, "failed to fetch protectedts metadata")
   226  		}
   227  		if versionChanged = md.Version != prev.Version; !versionChanged {
   228  			return nil
   229  		}
   230  		if state, err = c.storage.GetState(ctx, txn); err != nil {
   231  			return errors.Wrap(err, "failed to fetch protectedts state")
   232  		}
   233  		return nil
   234  	})
   235  	if err != nil {
   236  		return err
   237  	}
   238  	c.mu.Lock()
   239  	defer c.mu.Unlock()
   240  	c.mu.lastUpdate = ts
   241  	if versionChanged {
   242  		c.mu.state = state
   243  		for id := range c.mu.recordsByID {
   244  			delete(c.mu.recordsByID, id)
   245  		}
   246  		for i := range state.Records {
   247  			r := &state.Records[i]
   248  			c.mu.recordsByID[r.ID] = r
   249  		}
   250  	}
   251  	return nil
   252  }
   253  
   254  // upToDate returns true if the lastUpdate for the cache is at least asOf.
   255  func (c *Cache) upToDate(asOf hlc.Timestamp) bool {
   256  	c.mu.RLock()
   257  	defer c.mu.RUnlock()
   258  	return asOf.LessEq(c.mu.lastUpdate)
   259  }
   260  
   261  func overlaps(r *ptpb.Record, sp roachpb.Span) bool {
   262  	for i := range r.Spans {
   263  		if r.Spans[i].Overlaps(sp) {
   264  			return true
   265  		}
   266  	}
   267  	return false
   268  }