github.com/cilium/cilium@v1.16.2/operator/identitygc/kvstore_gc.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package identitygc
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"time"
    10  
    11  	"github.com/sirupsen/logrus"
    12  
    13  	"github.com/cilium/cilium/pkg/allocator"
    14  	ciliumIdentity "github.com/cilium/cilium/pkg/identity"
    15  	"github.com/cilium/cilium/pkg/identity/cache"
    16  	"github.com/cilium/cilium/pkg/idpool"
    17  	"github.com/cilium/cilium/pkg/inctimer"
    18  	"github.com/cilium/cilium/pkg/kvstore"
    19  	kvstoreallocator "github.com/cilium/cilium/pkg/kvstore/allocator"
    20  	"github.com/cilium/cilium/pkg/logging/logfields"
    21  )
    22  
    23  func (igc *GC) startKVStoreModeGC(ctx context.Context) error {
    24  	log.WithField(logfields.Interval, igc.gcInterval).Info("Starting kvstore identity garbage collector")
    25  	backend, err := kvstoreallocator.NewKVStoreBackend(cache.IdentitiesPath, "", nil, kvstore.Client())
    26  	if err != nil {
    27  		return fmt.Errorf("unable to initialize kvstore backend for identity allocation")
    28  	}
    29  
    30  	minID := idpool.ID(ciliumIdentity.GetMinimalAllocationIdentity(igc.clusterInfo.ID))
    31  	maxID := idpool.ID(ciliumIdentity.GetMaximumAllocationIdentity(igc.clusterInfo.ID))
    32  	log.WithFields(map[string]interface{}{
    33  		"min":        minID,
    34  		"max":        maxID,
    35  		"cluster-id": igc.clusterInfo.ID,
    36  	}).Info("Garbage Collecting identities between range")
    37  
    38  	igc.allocator = allocator.NewAllocatorForGC(backend, allocator.WithMin(minID), allocator.WithMax(maxID))
    39  
    40  	return igc.wp.Submit("kvstore-identity-gc", igc.runKVStoreModeGC)
    41  }
    42  
    43  func (igc *GC) runKVStoreModeGC(ctx context.Context) error {
    44  	keysToDeletePrev := map[string]uint64{}
    45  
    46  	gcTimer, gcTimerDone := inctimer.New()
    47  	defer gcTimerDone()
    48  	for {
    49  		now := time.Now()
    50  
    51  		keysToDelete, gcStats, err := igc.allocator.RunGC(igc.rateLimiter, keysToDeletePrev)
    52  		gcDuration := time.Since(now)
    53  		if err != nil {
    54  			igc.logger.WithError(err).Warning("Unable to run security identity garbage collector")
    55  
    56  			igc.failedRuns++
    57  			igc.metrics.IdentityGCRuns.WithLabelValues(LabelValueOutcomeFail).Set(float64(igc.failedRuns))
    58  		} else {
    59  			// Best effort to run auth identity GC
    60  			err = igc.runAuthGC(ctx, keysToDeletePrev)
    61  			if err != nil {
    62  				igc.logger.WithField("identities-to-delete", keysToDeletePrev).
    63  					WithError(err).
    64  					Warning("Unable to run auth identity garbage collector")
    65  			}
    66  
    67  			keysToDeletePrev = keysToDelete
    68  
    69  			igc.successfulRuns++
    70  			igc.metrics.IdentityGCRuns.WithLabelValues(LabelValueOutcomeSuccess).Set(float64(igc.successfulRuns))
    71  
    72  			igc.metrics.IdentityGCSize.WithLabelValues(LabelValueOutcomeAlive).Set(float64(gcStats.Alive))
    73  			igc.metrics.IdentityGCSize.WithLabelValues(LabelValueOutcomeDeleted).Set(float64(gcStats.Deleted))
    74  		}
    75  
    76  		if igc.gcInterval <= gcDuration {
    77  			igc.logger.WithFields(logrus.Fields{
    78  				logfields.Interval: igc.gcInterval,
    79  				logfields.Duration: gcDuration,
    80  				logfields.Hint:     "Is there a ratelimit configured on the kvstore client or server?",
    81  			}).Warning("Identity garbage collection took longer than the GC interval")
    82  
    83  			// Don't sleep because we have a lot of work to do,
    84  			// but check if the context was canceled before running
    85  			// another gc cycle.
    86  			if ctx.Err() != nil {
    87  				return nil
    88  			}
    89  		} else {
    90  			select {
    91  			case <-ctx.Done():
    92  				return nil
    93  			case <-gcTimer.After(igc.gcInterval - gcDuration):
    94  			}
    95  		}
    96  
    97  		igc.logger.WithFields(logrus.Fields{
    98  			"identities-to-delete": keysToDeletePrev,
    99  		}).Debug("Will delete identities if they are still unused")
   100  	}
   101  }
   102  
   103  func (igc *GC) runAuthGC(ctx context.Context, staleKeys map[string]uint64) error {
   104  	// Wait until we can delete an identity
   105  	if err := igc.rateLimiter.Wait(ctx); err != nil {
   106  		return err
   107  	}
   108  
   109  	for k := range staleKeys {
   110  		if err := igc.authIdentityClient.Delete(ctx, k); err != nil {
   111  			return err
   112  		}
   113  	}
   114  	return nil
   115  }