go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/server/tq/txn/datastore/ds.go (about)

     1  // Copyright 2020 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package datastore
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strconv"
    21  	"strings"
    22  	"time"
    23  
    24  	"go.chromium.org/luci/common/errors"
    25  	"go.chromium.org/luci/common/retry/transient"
    26  	"go.chromium.org/luci/gae/filter/txndefer"
    27  	ds "go.chromium.org/luci/gae/service/datastore"
    28  
    29  	"go.chromium.org/luci/server/tq/internal/reminder"
    30  )
    31  
    32  type dsDB struct{}
    33  
    34  func (dsDB) Kind() string {
    35  	return "datastore"
    36  }
    37  
    38  func (dsDB) Defer(ctx context.Context, cb func(context.Context)) {
    39  	txndefer.Defer(ctx, cb)
    40  }
    41  
    42  const reminderKind = "tq.Reminder"
    43  
    44  type dsReminder struct {
    45  	_kind string `gae:"$kind,tq.Reminder"`
    46  
    47  	ID      string `gae:"$id"` // "{Reminder.ID}_{Reminder.FreshUntil}".
    48  	Payload []byte `gae:",noindex"`
    49  }
    50  
    51  func (d *dsReminder) fromReminder(r *reminder.Reminder) *dsReminder {
    52  	d.ID = fmt.Sprintf("%s_%d", r.ID, r.FreshUntil.UnixNano())
    53  	d.Payload = r.RawPayload
    54  	return d
    55  }
    56  
    57  func (d dsReminder) toReminder(r *reminder.Reminder) *reminder.Reminder {
    58  	parts := strings.Split(d.ID, "_")
    59  	if len(parts) != 2 {
    60  		panic(errors.Reason("malformed dsReminder ID %q", d.ID).Err())
    61  	}
    62  	ns, err := strconv.ParseInt(parts[1], 10, 64)
    63  	if err != nil {
    64  		panic(errors.Reason("malformed dsReminder ID %q: %s", d.ID, err).Err())
    65  	}
    66  	if r == nil {
    67  		r = &reminder.Reminder{}
    68  	}
    69  	r.ID = parts[0]
    70  	r.FreshUntil = time.Unix(0, ns).UTC()
    71  	r.RawPayload = d.Payload
    72  	return r
    73  }
    74  
    75  // SaveReminder persists reminder in a transaction context.
    76  func (dsDB) SaveReminder(ctx context.Context, r *reminder.Reminder) error {
    77  	v := dsReminder{}
    78  	if err := ds.Put(ctx, v.fromReminder(r)); err != nil {
    79  		return errors.Annotate(err, "failed to persist to datastore").Tag(transient.Tag).Err()
    80  	}
    81  	return nil
    82  }
    83  
    84  // DeleteReminder deletes reminder in a non-tranasction context.
    85  func (dsDB) DeleteReminder(ctx context.Context, r *reminder.Reminder) error {
    86  	v := dsReminder{}
    87  	if err := ds.Delete(ctx, v.fromReminder(r)); err != nil {
    88  		return errors.Annotate(err, "failed to delete the Reminder %s", r.ID).Tag(transient.Tag).Err()
    89  	}
    90  	return nil
    91  }
    92  
    93  // FetchRemindersMeta fetches Reminders with Ids in [low..high) range.
    94  //
    95  // Payload of Reminders should not be fetched.
    96  // Both fresh & stale reminders should be fetched.
    97  // The reminders should be returned in order of ascending Id.
    98  //
    99  // In case of error, partial result of fetched Reminders so far should be
   100  // returned alongside the error. The caller will later call this method again
   101  // to fetch the remaining of Reminders in range of [<lastReturned.ID+1> .. high).
   102  func (dsDB) FetchRemindersMeta(ctx context.Context, low string, high string, limit int) (items []*reminder.Reminder, err error) {
   103  	q := ds.NewQuery(reminderKind).Order("__key__")
   104  	q = q.Gte("__key__", ds.NewKey(ctx, reminderKind, low, 0, nil))
   105  	q = q.Lt("__key__", ds.NewKey(ctx, reminderKind, high, 0, nil))
   106  	q = q.Limit(int32(limit)).KeysOnly(true)
   107  	err = ds.Run(ctx, q, func(k *ds.Key) {
   108  		items = append(items, dsReminder{ID: k.StringID()}.toReminder(nil))
   109  	})
   110  	if err != nil && err != context.DeadlineExceeded {
   111  		err = errors.Annotate(err, "failed to fetch Reminder keys").Tag(transient.Tag).Err()
   112  	}
   113  	return
   114  }
   115  
   116  // FetchReminderRawPayloads fetches payloads of a batch of Reminders.
   117  //
   118  // The Reminder objects are re-used in the returned batch.
   119  // If any Reminder is no longer found, it is silently omitted in the returned
   120  // batch.
   121  // In case of any other error, partial result of fetched Reminders so far
   122  // should be returned alongside the error.
   123  func (dsDB) FetchReminderRawPayloads(ctx context.Context, batch []*reminder.Reminder) ([]*reminder.Reminder, error) {
   124  	vs := make([]*dsReminder, len(batch))
   125  	for i, r := range batch {
   126  		vs[i] = (&dsReminder{}).fromReminder(r)
   127  	}
   128  	err := ds.Get(ctx, vs)
   129  	merr, ok := err.(errors.MultiError)
   130  	if err != nil && !ok {
   131  		return nil, errors.Annotate(err, "failed to fetch Reminders").Tag(transient.Tag).Err()
   132  	}
   133  
   134  	res := make([]*reminder.Reminder, 0, len(batch))
   135  	// Copy reminders with loaded payloads to result and move to the left errors
   136  	// in MultiError which are not expected.
   137  	ei := 0
   138  	for i, v := range vs {
   139  		switch {
   140  		case merr == nil || merr[i] == nil:
   141  			res = append(res, v.toReminder(batch[i]))
   142  		case merr[i] != ds.ErrNoSuchEntity:
   143  			merr[ei] = merr[i]
   144  			ei++
   145  		}
   146  	}
   147  
   148  	if ei == 0 {
   149  		return res, nil
   150  	}
   151  	return res, merr[:ei]
   152  }