go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/cv/internal/retention/cl.go (about)

     1  // Copyright 2024 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package retention
    16  
    17  import (
    18  	"context"
    19  	"strconv"
    20  
    21  	"google.golang.org/protobuf/proto"
    22  
    23  	"go.chromium.org/luci/common/clock"
    24  	"go.chromium.org/luci/common/errors"
    25  	"go.chromium.org/luci/common/logging"
    26  	"go.chromium.org/luci/common/retry"
    27  	"go.chromium.org/luci/common/retry/transient"
    28  	"go.chromium.org/luci/common/sync/parallel"
    29  	"go.chromium.org/luci/server/tq"
    30  
    31  	"go.chromium.org/luci/cv/internal/changelist"
    32  	"go.chromium.org/luci/cv/internal/common"
    33  	"go.chromium.org/luci/cv/internal/run/runquery"
    34  )
    35  
    36  // clsPerTask controls how many cls to wipeout per TQ task.
    37  const clsPerTask = 200
    38  
    39  // scheduleWipeoutCLTasks schedules tasks to wipe out old CLs that are out of
    40  // the retention period.
    41  //
    42  // The tasks will be uniformly distributed over the next hour.
    43  func scheduleWipeoutCLTasks(ctx context.Context, tqd *tq.Dispatcher) error {
    44  	cls, err := changelist.QueryCLIDsUpdatedBefore(ctx, clock.Now(ctx).Add(-retentionPeriod))
    45  	switch {
    46  	case err != nil:
    47  		return err
    48  	case len(cls) == 0:
    49  		logging.Infof(ctx, "no CLs to wipe out")
    50  		return nil
    51  	}
    52  
    53  	logging.Infof(ctx, "schedule tasks to wipeout %d CLs", len(cls))
    54  	return parallel.WorkPool(min(10, len(cls)/clsPerTask), func(workCh chan<- func() error) {
    55  		for _, chunk := range chunk(cls, clsPerTask) {
    56  			clidStrs := make([]string, len(chunk))
    57  			for i, clid := range chunk {
    58  				clidStrs[i] = strconv.FormatInt(int64(clid), 10)
    59  			}
    60  			task := &tq.Task{
    61  				Payload: &WipeoutCLsTask{
    62  					Ids: common.CLIDsAsInt64s(chunk),
    63  				},
    64  				Delay: common.DistributeOffset(wipeoutTasksDistInterval, clidStrs...),
    65  			}
    66  			workCh <- func() error {
    67  				return retry.Retry(ctx, retry.Default, func() error {
    68  					return tqd.AddTask(ctx, task)
    69  				}, nil)
    70  			}
    71  		}
    72  	})
    73  }
    74  
    75  func registerWipeoutCLsTask(tqd *tq.Dispatcher) {
    76  	tqd.RegisterTaskClass(tq.TaskClass{
    77  		ID:           "wipeout-cls",
    78  		Queue:        "data-retention",
    79  		Prototype:    &WipeoutCLsTask{},
    80  		Kind:         tq.NonTransactional,
    81  		Quiet:        true,
    82  		QuietOnError: true,
    83  		Handler: func(ctx context.Context, payload proto.Message) error {
    84  			task := payload.(*WipeoutCLsTask)
    85  			err := wipeoutCLs(ctx, common.MakeCLIDs(task.GetIds()...))
    86  			return common.TQifyError(ctx, err)
    87  		},
    88  	})
    89  }
    90  
    91  // wipeoutCLs wipes out the provided CLs.
    92  func wipeoutCLs(ctx context.Context, clids common.CLIDs) error {
    93  	return parallel.WorkPool(min(10, len(clids)), func(workCh chan<- func() error) {
    94  		for _, clid := range clids {
    95  			clid := clid
    96  			workCh <- func() error {
    97  				return wipeoutCL(ctx, clid)
    98  			}
    99  		}
   100  	})
   101  }
   102  
   103  // wipeoutCL wipes out the provided CL if CL is no longer referenced by any
   104  // runs.
   105  func wipeoutCL(ctx context.Context, clid common.CLID) error {
   106  	ctx = logging.SetField(ctx, "CL", clid)
   107  	runs, err := runquery.CLQueryBuilder{
   108  		CLID:  clid,
   109  		Limit: 1, // As long as there is 1 run referencing this CL, CV can't delete
   110  	}.GetAllRunKeys(ctx)
   111  	switch {
   112  	case err != nil:
   113  		return errors.Annotate(err, "failed to query runs involving CL %d", clid).Tag(transient.Tag).Err()
   114  	case len(runs) > 0:
   115  		logging.Warningf(ctx, "WipeoutCL: skip wipeout because CL is still referenced by run %s", runs[0].StringID())
   116  		return nil
   117  	}
   118  
   119  	if err := changelist.Delete(ctx, clid); err != nil {
   120  		return errors.Annotate(err, "failed to delete CL %d", clid).Tag(transient.Tag).Err()
   121  	}
   122  	logging.Infof(ctx, "successfully wiped out CL %d", clid)
   123  	return nil
   124  }