go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/resultdb/internal/cron/cron.go (about)

     1  // Copyright 2020 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package cron can runs functions periodically.
    16  package cron
    17  
    18  import (
    19  	"context"
    20  	"sync"
    21  	"time"
    22  
    23  	"golang.org/x/time/rate"
    24  
    25  	"go.chromium.org/luci/common/clock"
    26  	"go.chromium.org/luci/common/data/rand/mathrand"
    27  	"go.chromium.org/luci/common/logging"
    28  	"go.chromium.org/luci/common/runtime/paniccatcher"
    29  )
    30  
    31  // Group runs multiple cron jobs concurrently. See also Run function.
    32  func Group(ctx context.Context, replicas int, minInterval time.Duration, f func(ctx context.Context, replica int) error) {
    33  	var wg sync.WaitGroup
    34  	for i := 0; i < replicas; i++ {
    35  		i := i
    36  		ctx := logging.SetField(ctx, "cron_replica", i)
    37  		wg.Add(1)
    38  		go func() {
    39  			defer wg.Done()
    40  			Run(ctx, minInterval, func(ctx context.Context) error {
    41  				return f(ctx, i)
    42  			})
    43  		}()
    44  	}
    45  	wg.Wait()
    46  }
    47  
    48  // Run runs f repeatedly, until the context is cancelled.
    49  //
    50  // Ensures f is not called too often (minInterval).
    51  func Run(ctx context.Context, minInterval time.Duration, f func(context.Context) error) {
    52  	defer logging.Warningf(ctx, "Exiting cron")
    53  
    54  	// call calls f with a timeout and catches a panic.
    55  	call := func(ctx context.Context) error {
    56  		defer paniccatcher.Catch(func(p *paniccatcher.Panic) {
    57  			logging.Errorf(ctx, "Caught panic: %s\n%s", p.Reason, p.Stack)
    58  		})
    59  		return f(ctx)
    60  	}
    61  
    62  	var iterationCounter int
    63  	logLimiter := rate.NewLimiter(rate.Every(5*time.Minute), 1)
    64  	for {
    65  		iterationCounter++
    66  		if logLimiter.Allow() {
    67  			logging.Debugf(ctx, "%d iterations have run since start-up", iterationCounter)
    68  		}
    69  
    70  		start := clock.Now(ctx)
    71  		if err := call(ctx); err != nil {
    72  			logging.Errorf(ctx, "Iteration failed: %s", err)
    73  		}
    74  
    75  		// Ensure minInterval between iterations.
    76  		if sleep := minInterval - clock.Since(ctx, start); sleep > 0 {
    77  			// Add jitter: +-10% of sleep time to desynchronize cron jobs.
    78  			sleep = sleep - sleep/10 + time.Duration(mathrand.Intn(ctx, int(sleep/5)))
    79  			select {
    80  			case <-time.After(sleep):
    81  			case <-ctx.Done():
    82  				return
    83  			}
    84  		}
    85  	}
    86  }