github.com/cilium/cilium@v1.16.2/operator/cmd/kvstore_watchdog.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package cmd
     5  
     6  import (
     7  	"context"
     8  	"strings"
     9  	"time"
    10  
    11  	"github.com/cilium/cilium/pkg/allocator"
    12  	cmtypes "github.com/cilium/cilium/pkg/clustermesh/types"
    13  	cmutils "github.com/cilium/cilium/pkg/clustermesh/utils"
    14  	"github.com/cilium/cilium/pkg/defaults"
    15  	"github.com/cilium/cilium/pkg/identity"
    16  	"github.com/cilium/cilium/pkg/identity/cache"
    17  	"github.com/cilium/cilium/pkg/idpool"
    18  	"github.com/cilium/cilium/pkg/inctimer"
    19  	"github.com/cilium/cilium/pkg/kvstore"
    20  	kvstoreallocator "github.com/cilium/cilium/pkg/kvstore/allocator"
    21  	"github.com/cilium/cilium/pkg/logging/logfields"
    22  	"github.com/cilium/cilium/pkg/option"
    23  )
    24  
    25  // keyPathFromLockPath returns the path of the given key that contains a lease
    26  // prefixed to its path.
    27  func keyPathFromLockPath(k string) string {
    28  	// vendor/go.etcd.io/etcd/clientv3/concurrency/mutex.go:L46
    29  	i := strings.LastIndexByte(k, '/')
    30  	if i >= 0 {
    31  		return k[:i]
    32  	}
    33  	return k
    34  }
    35  
    36  // getOldestLeases returns the value that has the smaller revision for each
    37  // 'path'. A 'path' shares the same common prefix for different locks.
    38  func getOldestLeases(lockPaths map[string]kvstore.Value) map[string]kvstore.Value {
    39  	type LockValue struct {
    40  		kvstore.Value
    41  		keyPath string
    42  	}
    43  	oldestPaths := map[string]LockValue{}
    44  	for lockPath, v := range lockPaths {
    45  		keyPath := keyPathFromLockPath(lockPath)
    46  		oldestKeyPath, ok := oldestPaths[keyPath]
    47  		if !ok || v.ModRevision < oldestKeyPath.ModRevision {
    48  			// Store the oldest common path
    49  			oldestPaths[keyPath] = LockValue{
    50  				keyPath: lockPath,
    51  				Value:   v,
    52  			}
    53  		}
    54  	}
    55  	oldestLeases := map[string]kvstore.Value{}
    56  	for _, v := range oldestPaths {
    57  		// Retrieve the oldest lock path
    58  		oldestLeases[v.keyPath] = v.Value
    59  	}
    60  	return oldestLeases
    61  }
    62  
    63  func startKvstoreWatchdog() {
    64  	log.WithField(logfields.Interval, defaults.LockLeaseTTL).Infof("Starting kvstore watchdog")
    65  	backend, err := kvstoreallocator.NewKVStoreBackend(cache.IdentitiesPath, "", nil, kvstore.Client())
    66  	if err != nil {
    67  		log.WithError(err).Fatal("Unable to initialize kvstore backend for identity garbage collection")
    68  	}
    69  
    70  	minID := idpool.ID(identity.GetMinimalAllocationIdentity(option.Config.ClusterID))
    71  	maxID := idpool.ID(identity.GetMaximumAllocationIdentity(option.Config.ClusterID))
    72  	a := allocator.NewAllocatorForGC(backend, allocator.WithMin(minID), allocator.WithMax(maxID))
    73  
    74  	keysToDelete := map[string]kvstore.Value{}
    75  	go func() {
    76  		lockTimer, lockTimerDone := inctimer.New()
    77  		defer lockTimerDone()
    78  		for {
    79  			keysToDelete = getOldestLeases(keysToDelete)
    80  			ctx, cancel := context.WithTimeout(context.Background(), defaults.LockLeaseTTL)
    81  			keysToDelete2, err := a.RunLocksGC(ctx, keysToDelete)
    82  			if err != nil {
    83  				log.WithError(err).Warning("Unable to run security identity garbage collector")
    84  			} else {
    85  				keysToDelete = keysToDelete2
    86  			}
    87  			cancel()
    88  
    89  			<-lockTimer.After(defaults.LockLeaseTTL)
    90  		}
    91  	}()
    92  
    93  	go func() {
    94  		hbTimer, hbTimerDone := inctimer.New()
    95  		defer hbTimerDone()
    96  		for {
    97  			ctx, cancel := context.WithTimeout(context.Background(), defaults.LockLeaseTTL)
    98  
    99  			err := kvstore.Client().Update(ctx, kvstore.HeartbeatPath, []byte(time.Now().Format(time.RFC3339)), true)
   100  			if err != nil {
   101  				log.WithError(err).Warning("Unable to update heartbeat key")
   102  			}
   103  
   104  			if option.Config.ClusterName != defaults.ClusterName && option.Config.ClusterID != 0 {
   105  				// The cluster config continues to be enforced also after the initial successful
   106  				// insertion to prevent issues in case of, e.g., unexpected lease expiration.
   107  				cfg := cmtypes.CiliumClusterConfig{
   108  					ID:           option.Config.ClusterID,
   109  					Capabilities: cmtypes.CiliumClusterConfigCapabilities{MaxConnectedClusters: option.Config.MaxConnectedClusters}}
   110  				if err := cmutils.SetClusterConfig(ctx, option.Config.ClusterName, cfg, kvstore.Client()); err != nil {
   111  					log.WithError(err).Warning("Unable to set local cluster config")
   112  				}
   113  			}
   114  
   115  			cancel()
   116  			<-hbTimer.After(kvstore.HeartbeatWriteInterval)
   117  		}
   118  	}()
   119  }