github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvserver/protectedts/ptcache/cache.go (about) 1 // Copyright 2019 The Cockroach Authors. 2 // 3 // Use of this software is governed by the Business Source License 4 // included in the file licenses/BSL.txt. 5 // 6 // As of the Change Date specified in that file, in accordance with 7 // the Business Source License, use of this software will be governed 8 // by the Apache License, Version 2.0, included in the file 9 // licenses/APL.txt. 10 11 package ptcache 12 13 import ( 14 "context" 15 "time" 16 17 "github.com/cockroachdb/cockroach/pkg/kv" 18 "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts" 19 "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb" 20 "github.com/cockroachdb/cockroach/pkg/roachpb" 21 "github.com/cockroachdb/cockroach/pkg/settings/cluster" 22 "github.com/cockroachdb/cockroach/pkg/util/hlc" 23 "github.com/cockroachdb/cockroach/pkg/util/log" 24 "github.com/cockroachdb/cockroach/pkg/util/stop" 25 "github.com/cockroachdb/cockroach/pkg/util/syncutil" 26 "github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight" 27 "github.com/cockroachdb/cockroach/pkg/util/timeutil" 28 "github.com/cockroachdb/cockroach/pkg/util/uuid" 29 "github.com/cockroachdb/errors" 30 ) 31 32 // Cache implements protectedts.Cache. 33 type Cache struct { 34 db *kv.DB 35 storage protectedts.Storage 36 stopper *stop.Stopper 37 settings *cluster.Settings 38 sf singleflight.Group 39 mu struct { 40 syncutil.RWMutex 41 42 started bool 43 44 // Updated in doUpdate(). 45 lastUpdate hlc.Timestamp 46 state ptpb.State 47 48 // Updated in doUpdate but mutable. The records in the map are not mutated 49 // and should not be by any client. 50 recordsByID map[uuid.UUID]*ptpb.Record 51 52 // TODO(ajwerner): add a more efficient lookup structure such as an 53 // interval.Tree for Iterate. 54 } 55 } 56 57 // Config configures a Cache. 58 type Config struct { 59 DB *kv.DB 60 Storage protectedts.Storage 61 Settings *cluster.Settings 62 } 63 64 // New returns a new cache. 65 func New(config Config) *Cache { 66 c := &Cache{ 67 db: config.DB, 68 storage: config.Storage, 69 settings: config.Settings, 70 } 71 c.mu.recordsByID = make(map[uuid.UUID]*ptpb.Record) 72 return c 73 } 74 75 var _ protectedts.Cache = (*Cache)(nil) 76 77 // Iterate is part of the protectedts.Cache interface. 78 func (c *Cache) Iterate( 79 _ context.Context, from, to roachpb.Key, it protectedts.Iterator, 80 ) (asOf hlc.Timestamp) { 81 c.mu.RLock() 82 state, lastUpdate := c.mu.state, c.mu.lastUpdate 83 c.mu.RUnlock() 84 85 sp := roachpb.Span{ 86 Key: from, 87 EndKey: to, 88 } 89 for i := range state.Records { 90 r := &state.Records[i] 91 if !overlaps(r, sp) { 92 continue 93 } 94 if wantMore := it(r); !wantMore { 95 break 96 } 97 } 98 return lastUpdate 99 } 100 101 // QueryRecord is part of the protectedts.Cache interface. 102 func (c *Cache) QueryRecord(_ context.Context, id uuid.UUID) (exists bool, asOf hlc.Timestamp) { 103 c.mu.RLock() 104 defer c.mu.RUnlock() 105 106 _, exists = c.mu.recordsByID[id] 107 return exists, c.mu.lastUpdate 108 } 109 110 // refreshKey is used for the singleflight. 111 const refreshKey = "" 112 113 // Refresh is part of the protectedts.Cache interface. 114 func (c *Cache) Refresh(ctx context.Context, asOf hlc.Timestamp) error { 115 for !c.upToDate(asOf) { 116 ch, _ := c.sf.DoChan(refreshKey, c.doSingleFlightUpdate) 117 select { 118 case <-ctx.Done(): 119 return ctx.Err() 120 case res := <-ch: 121 if res.Err != nil { 122 return res.Err 123 } 124 } 125 } 126 return nil 127 } 128 129 // Start starts the periodic fetching of the Cache. A Cache must not be used 130 // until after it has been started. An error will be returned if it has 131 // already been started. 132 func (c *Cache) Start(ctx context.Context, stopper *stop.Stopper) error { 133 c.mu.Lock() 134 defer c.mu.Unlock() 135 if c.mu.started { 136 return errors.New("cannot start a Cache more than once") 137 } 138 c.mu.started = true 139 c.stopper = stopper 140 return c.stopper.RunAsyncTask(ctx, "periodically-refresh-protectedts-cache", 141 c.periodicallyRefreshProtectedtsCache) 142 } 143 144 func (c *Cache) periodicallyRefreshProtectedtsCache(ctx context.Context) { 145 settingChanged := make(chan struct{}, 1) 146 protectedts.PollInterval.SetOnChange(&c.settings.SV, func() { 147 select { 148 case settingChanged <- struct{}{}: 149 default: 150 } 151 }) 152 timer := timeutil.NewTimer() 153 defer timer.Stop() 154 timer.Reset(0) // Read immediately upon startup 155 var lastReset time.Time 156 var doneCh <-chan singleflight.Result 157 // TODO(ajwerner): consider resetting the timer when the state is updated 158 // due to a call to Refresh. 159 for { 160 select { 161 case <-timer.C: 162 // Let's not reset the timer until we get our response. 163 timer.Read = true 164 doneCh, _ = c.sf.DoChan(refreshKey, c.doSingleFlightUpdate) 165 case <-settingChanged: 166 if timer.Read { // we're currently fetching 167 continue 168 } 169 interval := protectedts.PollInterval.Get(&c.settings.SV) 170 // NB: It's okay if nextUpdate is a negative duration; timer.Reset will 171 // treat a negative duration as zero and send a notification immediately. 172 nextUpdate := interval - timeutil.Since(lastReset) 173 timer.Reset(nextUpdate) 174 lastReset = timeutil.Now() 175 case res := <-doneCh: 176 if res.Err != nil { 177 if ctx.Err() == nil { 178 log.Errorf(ctx, "failed to refresh protected timestamps: %v", res.Err) 179 } 180 } 181 timer.Reset(protectedts.PollInterval.Get(&c.settings.SV)) 182 lastReset = timeutil.Now() 183 case <-c.stopper.ShouldQuiesce(): 184 return 185 } 186 } 187 } 188 189 func (c *Cache) doSingleFlightUpdate() (interface{}, error) { 190 // TODO(ajwerner): add log tags to the context. 191 ctx, cancel := c.stopper.WithCancelOnQuiesce(context.Background()) 192 defer cancel() 193 return nil, c.stopper.RunTaskWithErr(ctx, 194 "refresh-protectedts-cache", c.doUpdate) 195 } 196 197 func (c *Cache) getMetadata() ptpb.Metadata { 198 c.mu.RLock() 199 defer c.mu.RUnlock() 200 return c.mu.state.Metadata 201 } 202 203 func (c *Cache) doUpdate(ctx context.Context) error { 204 // NB: doUpdate is only ever called underneath c.singleFlight and thus is 205 // never called concurrently. Due to the lack of concurrency there are no 206 // concerns about races as this is the only method which writes to the Cache's 207 // state. 208 prev := c.getMetadata() 209 var ( 210 versionChanged bool 211 state ptpb.State 212 ts hlc.Timestamp 213 ) 214 err := c.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) (err error) { 215 // NB: because this is a read-only transaction, the commit will be a no-op; 216 // returning nil here means the transaction will commit and will never need 217 // to change its read timestamp. 218 defer func() { 219 if err == nil { 220 ts = txn.ReadTimestamp() 221 } 222 }() 223 md, err := c.storage.GetMetadata(ctx, txn) 224 if err != nil { 225 return errors.Wrap(err, "failed to fetch protectedts metadata") 226 } 227 if versionChanged = md.Version != prev.Version; !versionChanged { 228 return nil 229 } 230 if state, err = c.storage.GetState(ctx, txn); err != nil { 231 return errors.Wrap(err, "failed to fetch protectedts state") 232 } 233 return nil 234 }) 235 if err != nil { 236 return err 237 } 238 c.mu.Lock() 239 defer c.mu.Unlock() 240 c.mu.lastUpdate = ts 241 if versionChanged { 242 c.mu.state = state 243 for id := range c.mu.recordsByID { 244 delete(c.mu.recordsByID, id) 245 } 246 for i := range state.Records { 247 r := &state.Records[i] 248 c.mu.recordsByID[r.ID] = r 249 } 250 } 251 return nil 252 } 253 254 // upToDate returns true if the lastUpdate for the cache is at least asOf. 255 func (c *Cache) upToDate(asOf hlc.Timestamp) bool { 256 c.mu.RLock() 257 defer c.mu.RUnlock() 258 return asOf.LessEq(c.mu.lastUpdate) 259 } 260 261 func overlaps(r *ptpb.Record, sp roachpb.Span) bool { 262 for i := range r.Spans { 263 if r.Spans[i].Overlaps(sp) { 264 return true 265 } 266 } 267 return false 268 }