go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/gae/filter/dscache/ds.go (about)

     1  // Copyright 2015 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package dscache
    16  
    17  import (
    18  	"context"
    19  	"time"
    20  
    21  	"go.chromium.org/luci/common/data/rand/mathrand"
    22  	"go.chromium.org/luci/common/errors"
    23  	"go.chromium.org/luci/common/logging"
    24  
    25  	ds "go.chromium.org/luci/gae/service/datastore"
    26  )
    27  
    28  // internalValueSizeLimit is a var for testing purposes.
    29  var internalValueSizeLimit = ValueSizeLimit
    30  
    31  type dsCache struct {
    32  	ds.RawInterface
    33  
    34  	*supportContext
    35  }
    36  
    37  var _ ds.RawInterface = (*dsCache)(nil)
    38  
    39  func (d *dsCache) DeleteMulti(keys []*ds.Key, cb ds.DeleteMultiCB) error {
    40  	return d.mutation(keys, func() error {
    41  		return d.RawInterface.DeleteMulti(keys, cb)
    42  	})
    43  }
    44  
    45  func (d *dsCache) PutMulti(keys []*ds.Key, vals []ds.PropertyMap, cb ds.NewKeyCB) error {
    46  	return d.mutation(keys, func() error {
    47  		return d.RawInterface.PutMulti(keys, vals, cb)
    48  	})
    49  }
    50  
    51  func (d *dsCache) GetMulti(keys []*ds.Key, metas ds.MultiMetaGetter, cb ds.GetMultiCB) error {
    52  	rnd := mathrand.Get(d.c)
    53  
    54  	itemKeys := d.mkRandKeys(keys, metas, rnd)
    55  	if len(itemKeys) == 0 {
    56  		return d.RawInterface.GetMulti(keys, metas, cb)
    57  	}
    58  
    59  	nonce := generateNonce(rnd)
    60  	lockItems, err := d.impl.TryLockAndFetch(d.c, itemKeys, nonce, RefreshLockTimeout)
    61  	if err != nil {
    62  		logging.WithError(err).Debugf(d.c, "dscache: GetMulti: TryLockAndFetch")
    63  	}
    64  
    65  	p := d.makeFetchPlan(facts{keys, metas, lockItems, nonce})
    66  
    67  	if !p.empty() {
    68  		// looks like we have something to pull from datastore, and maybe some work
    69  		// to save stuff back to memcache.
    70  
    71  		var toCas []CacheItem
    72  		err := d.RawInterface.GetMulti(p.toGet, p.toGetMeta, func(j int, pm ds.PropertyMap, err error) {
    73  			i := p.idxMap[j]
    74  
    75  			if err == nil {
    76  				p.decoded[i] = pm
    77  			} else {
    78  				p.lme.Assign(i, err)
    79  				if err != ds.ErrNoSuchEntity {
    80  					return
    81  				}
    82  				pm = nil
    83  			}
    84  
    85  			toSave := p.toSave[j]
    86  			if toSave == nil {
    87  				return
    88  			}
    89  
    90  			expiry := time.Duration(ds.GetMetaDefault(
    91  				metas.GetSingle(i),
    92  				CacheExpirationMeta,
    93  				int64(CacheDuration.Seconds()),
    94  			).(int64)) * time.Second
    95  
    96  			if pm == nil {
    97  				// Missing entities are denoted by an empty data buffer.
    98  				toSave.PromoteToData(toSave.Prefix(), expiry)
    99  			} else {
   100  				// Serialize and compress the PropertyMap, bail if too large.
   101  				buf, err := encodeItemValue(pm, toSave.Prefix())
   102  				if err == nil && len(buf) > internalValueSizeLimit {
   103  					err = errors.Reason("encoded entity too big (%d > %d)", len(buf), internalValueSizeLimit).Err()
   104  				}
   105  				if err == nil {
   106  					// The item should be able to fit into the cache.
   107  					toSave.PromoteToData(buf, expiry)
   108  				} else {
   109  					// The item is "broken". No one else should try to serialize this item
   110  					// until something Put/Delete's it. Set a lock on it.
   111  					logging.WithError(err).Warningf(d.c, "dscache: PropertyMap serialization error")
   112  					toSave.PromoteToIndefiniteLock()
   113  				}
   114  			}
   115  			toCas = append(toCas, toSave)
   116  		})
   117  
   118  		if err != nil {
   119  			// TODO(vadimsh): Should we drop locks owned by us?
   120  			return err
   121  		}
   122  
   123  		if len(toCas) > 0 {
   124  			// Store stuff we fetched back into memcache unless someone (like
   125  			// a concurrent Put) deleted our locks already.
   126  			if err := d.impl.CompareAndSwap(d.c, toCas); err != nil {
   127  				logging.WithError(err).Debugf(d.c, "dscache: GetMulti: CompareAndSwap")
   128  			}
   129  		}
   130  	}
   131  
   132  	// Finally, run the callback for all of the decoded items and the errors,
   133  	// if any.
   134  	for i, dec := range p.decoded {
   135  		cb(i, dec, p.lme.GetOne(i))
   136  	}
   137  
   138  	return nil
   139  }
   140  
   141  func (d *dsCache) RunInTransaction(f func(context.Context) error, opts *ds.TransactionOptions) error {
   142  	txnState := dsTxnState{}
   143  	err := d.RawInterface.RunInTransaction(func(ctx context.Context) error {
   144  		txnState.reset()
   145  		err := f(context.WithValue(ctx, &dsTxnCacheKey, &txnState))
   146  		if err == nil {
   147  			err = txnState.apply(d.supportContext)
   148  		}
   149  		return err
   150  	}, opts)
   151  	// Note: the transaction can *eventually* succeed even if `err` is non-nil
   152  	// here. So on errors we pessimistically keep the locks until they expire.
   153  	if err == nil {
   154  		txnState.release(d.supportContext)
   155  	}
   156  	return err
   157  }