go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/buildbucket/appengine/internal/buildcron/retention.go (about)

     1  // Copyright 2022 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package buildcron
    16  
    17  import (
    18  	"context"
    19  	"time"
    20  
    21  	"go.chromium.org/luci/common/clock"
    22  	"go.chromium.org/luci/common/errors"
    23  	"go.chromium.org/luci/common/logging"
    24  	"go.chromium.org/luci/common/sync/parallel"
    25  	"go.chromium.org/luci/gae/service/datastore"
    26  
    27  	"go.chromium.org/luci/buildbucket/appengine/internal/buildid"
    28  	"go.chromium.org/luci/buildbucket/appengine/model"
    29  )
    30  
    31  // buildKeyByAge returns a datastore key for a given age old Build.
    32  func buildKeyByAge(ctx context.Context, age time.Duration) *datastore.Key {
    33  	now := clock.Now(ctx)
    34  	// the low time yields the high ID.
    35  	// i.e., 1st param yields 2nd element in the return tuple.
    36  	_, idHigh := buildid.IDRange(
    37  		now.Add(-age), /* low time */
    38  		now,           /* high time */
    39  	)
    40  	return datastore.MakeKey(ctx, model.BuildKind, idHigh)
    41  }
    42  
    43  func deleteBuilds(ctx context.Context, keys []*datastore.Key) error {
    44  	if len(keys) == 0 {
    45  		return nil
    46  	}
    47  	// flatten first w/o filtering to calculate how many builds were actually
    48  	// removed.
    49  	allErrs := 0
    50  	badErrs := 0
    51  	err := errors.Flatten(datastore.Delete(ctx, keys))
    52  	if err != nil {
    53  		allErrs = len(err.(errors.MultiError))
    54  		// ignore it; multiple crons are running??
    55  		err = errors.Filter(err, datastore.ErrNoSuchEntity)
    56  		err = errors.Flatten(err)
    57  		badErrs = len(err.(errors.MultiError))
    58  	}
    59  	if allErrs < len(keys) {
    60  		logging.Infof(ctx, "Removed %d out of %d builds (%d errors, %d already gone)",
    61  			len(keys)-(allErrs-badErrs), len(keys), badErrs, allErrs-badErrs)
    62  	}
    63  	return err
    64  }
    65  
    66  // DeleteOldBuilds deletes builds and other descendants that were created longer
    67  // than model.BuildStorageDuration ago.
    68  func DeleteOldBuilds(ctx context.Context) error {
    69  	const batchSize = 128
    70  	const nWorkers = 4
    71  	// kindless query for Build and all of its descendants.
    72  	q := datastore.NewQuery("").
    73  		Gt("__key__", buildKeyByAge(ctx, model.BuildStorageDuration)).
    74  		Lt("__key__", datastore.MakeKey(ctx, model.BuildKind, buildid.BuildIDMax)).
    75  		KeysOnly(true)
    76  
    77  	return parallel.WorkPool(nWorkers, func(workC chan<- func() error) {
    78  		ch := make(chan []*datastore.Key, nWorkers)
    79  		workC <- func() error {
    80  			defer close(ch)
    81  
    82  			bks := make([]*datastore.Key, 0, batchSize)
    83  			err := datastore.RunBatch(ctx, int32(batchSize), q, func(bk *datastore.Key) error {
    84  				bks = append(bks, bk)
    85  				if len(bks) == batchSize {
    86  					ch <- bks
    87  					bks = make([]*datastore.Key, 0, batchSize)
    88  				}
    89  				return nil
    90  			})
    91  			if len(bks) > 0 {
    92  				ch <- bks
    93  			}
    94  			return err
    95  		}
    96  
    97  		for bks := range ch {
    98  			bks := bks
    99  			workC <- func() error { return deleteBuilds(ctx, bks) }
   100  		}
   101  	})
   102  }