go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/deploy/service/model/cleanup.go (about)

     1  // Copyright 2022 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package model
    16  
    17  import (
    18  	"context"
    19  	"time"
    20  
    21  	"go.chromium.org/luci/common/clock"
    22  	"go.chromium.org/luci/common/errors"
    23  	"go.chromium.org/luci/common/logging"
    24  	"go.chromium.org/luci/common/retry/transient"
    25  	"go.chromium.org/luci/common/sync/parallel"
    26  	"go.chromium.org/luci/gae/service/datastore"
    27  )
    28  
    29  // How long to keep historical records in the datastore.
    30  const retentionPeriod = 365 * 24 * time.Hour // ~1 year
    31  
    32  // CleanupOldEntities deletes old entities to keep datastore tidy.
    33  func CleanupOldEntities(ctx context.Context) error {
    34  	ctx, done := clock.WithTimeout(ctx, 8*time.Minute)
    35  	defer done()
    36  
    37  	cutoff := clock.Now(ctx).Add(-retentionPeriod)
    38  
    39  	err := parallel.FanOutIn(func(work chan<- func() error) {
    40  		work <- func() error { return cleanupOld(ctx, "Actuation", cutoff) }
    41  		work <- func() error { return cleanupOld(ctx, "AssetHistory", cutoff) }
    42  	})
    43  	return transient.Tag.Apply(err)
    44  }
    45  
    46  func cleanupOld(ctx context.Context, kind string, cutoff time.Time) error {
    47  	q := datastore.NewQuery(kind).
    48  		Lt("Created", cutoff.UTC()).
    49  		KeysOnly(true)
    50  
    51  	const batchSize = 250
    52  	var batch []*datastore.Key
    53  	var errs errors.MultiError
    54  
    55  	flushBatch := func() {
    56  		if len(batch) != 0 {
    57  			logging.Infof(ctx, "Deleting %d old %s entities", len(batch), kind)
    58  			if err := datastore.Delete(ctx, batch); err != nil {
    59  				logging.Errorf(ctx, "Failed to delete some of %d %s entities: %s", len(batch), kind, err)
    60  				errs = append(errs, err)
    61  			}
    62  			batch = batch[:0]
    63  		}
    64  	}
    65  
    66  	err := datastore.RunBatch(ctx, batchSize, q, func(key *datastore.Key) {
    67  		batch = append(batch, key)
    68  		if len(batch) == batchSize {
    69  			flushBatch()
    70  		}
    71  	})
    72  	flushBatch()
    73  
    74  	if err != nil {
    75  		logging.Errorf(ctx, "Query to fetch old %s entities failed: %s", kind, err)
    76  		errs = append(errs, err)
    77  	}
    78  
    79  	return errs.AsError()
    80  }