github.com/cilium/cilium@v1.16.2/pkg/fqdn/name_manager_gc.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package fqdn
     5  
     6  import (
     7  	"context"
     8  	"encoding/json"
     9  	"errors"
    10  	"fmt"
    11  	"net/netip"
    12  	"os"
    13  	"path/filepath"
    14  	"regexp"
    15  	"strings"
    16  
    17  	"k8s.io/apimachinery/pkg/util/sets"
    18  
    19  	"github.com/cilium/cilium/pkg/controller"
    20  	"github.com/cilium/cilium/pkg/fqdn/matchpattern"
    21  	"github.com/cilium/cilium/pkg/ipcache"
    22  	ipcacheTypes "github.com/cilium/cilium/pkg/ipcache/types"
    23  	"github.com/cilium/cilium/pkg/labels"
    24  	"github.com/cilium/cilium/pkg/logging/logfields"
    25  	"github.com/cilium/cilium/pkg/metrics"
    26  	"github.com/cilium/cilium/pkg/option"
    27  	"github.com/cilium/cilium/pkg/policy/api"
    28  	"github.com/cilium/cilium/pkg/source"
    29  	"github.com/cilium/cilium/pkg/time"
    30  )
    31  
    32  const DNSGCJobInterval = 1 * time.Minute
    33  
    34  const dnsGCJobName = "dns-garbage-collector-job"
    35  
    36  var (
    37  	dnsGCControllerGroup = controller.NewGroup(dnsGCJobName)
    38  )
    39  
    40  var (
    41  	checkpointFile = "fqdn-name-manager-selectors.json"
    42  
    43  	restorationIPCacheResource = ipcacheTypes.NewResourceID(ipcacheTypes.ResourceKindDaemon, "", "fqdn-name-manager-restoration")
    44  )
    45  
    46  // serializedSelector is the schema of the serialized selectors on disk
    47  type serializedSelector struct {
    48  	Regex    string           `json:"re"`
    49  	Selector api.FQDNSelector `json:"sel"`
    50  }
    51  
    52  // This implements some garbage collection and cleanup functions for the NameManager
    53  
    54  // GC cleans up TTL expired entries from the DNS policies.
    55  // It removes stale or undesired entries from the DNS caches.
    56  // This is done for all per-EP DNSCache
    57  // instances (ep.DNSHistory) with evictions (whether due to TTL expiry or
    58  // overlimit eviction) cascaded into ep.DNSZombies. Data in DNSHistory and
    59  // DNSZombies is further collected into the global DNSCache instance. The
    60  // data there drives toFQDNs policy via NameManager and ToFQDNs selectors.
    61  // DNSCache entries expire data when the TTL elapses and when the entries for
    62  // a DNS name are above a limit. The data is then placed into
    63  // DNSZombieMappings instances. These rely on the CT GC loop to update
    64  // liveness for each to-delete IP. When an IP is not in-use it is finally
    65  // deleted from the global DNSCache. Until then, each of these IPs is
    66  // inserted into the global cache as a synthetic DNS lookup.
    67  func (n *NameManager) GC(ctx context.Context) error {
    68  	var (
    69  		GCStart = time.Now()
    70  
    71  		// activeConnections holds DNSName -> single IP entries that have been
    72  		// marked active by the CT GC. Since we expire in this controller, we
    73  		// give these entries 2 cycles of TTL to allow for timing mismatches
    74  		// with the CT GC.
    75  		activeConnectionsTTL = int(2 * DNSGCJobInterval.Seconds())
    76  		activeConnections    = NewDNSCache(activeConnectionsTTL)
    77  	)
    78  	namesToClean := make(sets.Set[string])
    79  
    80  	// Take a snapshot of the *entire* reverse cache, so we can compute the set of
    81  	// IPs that have been completely removed and safely delete their metadata.
    82  	maybeStaleIPs := n.cache.GetIPs()
    83  
    84  	// Cleanup each endpoint cache, deferring deletions via DNSZombies.
    85  	endpoints := n.config.GetEndpointsDNSInfo("")
    86  	for _, ep := range endpoints {
    87  		epID := ep.ID
    88  		if metrics.FQDNActiveNames.IsEnabled() || metrics.FQDNActiveIPs.IsEnabled() {
    89  			countFQDNs, countIPs := ep.DNSHistory.Count()
    90  			if metrics.FQDNActiveNames.IsEnabled() {
    91  				metrics.FQDNActiveNames.WithLabelValues(epID).Set(float64(countFQDNs))
    92  			}
    93  			if metrics.FQDNActiveIPs.IsEnabled() {
    94  				metrics.FQDNActiveIPs.WithLabelValues(epID).Set(float64(countIPs))
    95  			}
    96  		}
    97  		affectedNames := ep.DNSHistory.GC(GCStart, ep.DNSZombies)
    98  		namesToClean = namesToClean.Union(affectedNames)
    99  
   100  		alive, dead := ep.DNSZombies.GC()
   101  		if metrics.FQDNAliveZombieConnections.IsEnabled() {
   102  			metrics.FQDNAliveZombieConnections.WithLabelValues(epID).Set(float64(len(alive)))
   103  		}
   104  
   105  		// Alive zombie need to be added to the global cache as name->IP
   106  		// entries.
   107  		//
   108  		// NB: The following  comment is _no longer true_ (see
   109  		// DNSZombies.GC()).  We keep it to maintain the original intention
   110  		// of the code for future reference:
   111  		//    We accumulate the names into namesToClean to ensure that the
   112  		//    original full DNS lookup (name -> many IPs) is expired and
   113  		//    only the active connections (name->single IP) are re-added.
   114  		//    Note: Other DNS lookups may also use an active IP. This is
   115  		//    fine.
   116  		//
   117  		lookupTime := time.Now()
   118  		for _, zombie := range alive {
   119  			for _, name := range zombie.Names {
   120  				namesToClean.Insert(name)
   121  				activeConnections.Update(lookupTime, name, []netip.Addr{zombie.IP}, activeConnectionsTTL)
   122  			}
   123  		}
   124  
   125  		// Dead entries can be deleted outright, without any replacement.
   126  		// Entries here have been evicted from the DNS cache (via .GC due to
   127  		// TTL expiration or overlimit) and are no longer active connections.
   128  		for _, zombie := range dead {
   129  			namesToClean.Insert(zombie.Names...)
   130  		}
   131  	}
   132  
   133  	if namesToClean.Len() == 0 {
   134  		return nil
   135  	}
   136  
   137  	// Collect DNS data into the global cache. This aggregates all endpoint
   138  	// and existing connection data into one place for use elsewhere.
   139  	// In the case where a lookup occurs in a race with .ReplaceFromCache the
   140  	// result is consistent:
   141  	// - If before, the ReplaceFromCache will use the new data when pulling
   142  	// in from each EP cache.
   143  	// - If after, the normal update process occurs after .ReplaceFromCache
   144  	// releases its locks.
   145  	caches := []*DNSCache{activeConnections}
   146  	for _, ep := range endpoints {
   147  		caches = append(caches, ep.DNSHistory)
   148  	}
   149  
   150  	namesToCleanSlice := namesToClean.UnsortedList()
   151  
   152  	n.cache.ReplaceFromCacheByNames(namesToCleanSlice, caches...)
   153  
   154  	metrics.FQDNGarbageCollectorCleanedTotal.Add(float64(len(namesToCleanSlice)))
   155  	namesCount := len(namesToCleanSlice)
   156  	// Limit the amount of info level logging to some sane amount
   157  	if namesCount > 20 {
   158  		// namedsToClean is only used for logging after this so we can reslice it in place
   159  		namesToCleanSlice = namesToCleanSlice[:20]
   160  	}
   161  	log.WithField(logfields.Controller, dnsGCJobName).Infof(
   162  		"FQDN garbage collector work deleted %d name entries: %s", namesCount, strings.Join(namesToCleanSlice, ","))
   163  
   164  	// Remove any now-stale ipcache metadata.
   165  	// Need to RLock here so we don't race on re-insertion.
   166  	n.maybeRemoveMetadata(maybeStaleIPs)
   167  
   168  	return nil
   169  }
   170  
   171  func (n *NameManager) StartGC(ctx context.Context) {
   172  	n.manager.UpdateController(dnsGCJobName, controller.ControllerParams{
   173  		Group:       dnsGCControllerGroup,
   174  		RunInterval: DNSGCJobInterval,
   175  		DoFunc:      n.GC,
   176  		Context:     ctx,
   177  	})
   178  }
   179  
   180  // DeleteDNSLookups force-removes any entries in *all* caches that are not currently actively
   181  // passing traffic.
   182  func (n *NameManager) DeleteDNSLookups(expireLookupsBefore time.Time, matchPatternStr string) error {
   183  	var nameMatcher *regexp.Regexp // nil matches all in our implementation
   184  	if matchPatternStr != "" {
   185  		var err error
   186  		nameMatcher, err = matchpattern.ValidateWithoutCache(matchPatternStr)
   187  		if err != nil {
   188  			return err
   189  		}
   190  	}
   191  
   192  	maybeStaleIPs := n.cache.GetIPs()
   193  
   194  	// Clear any to-delete entries globally
   195  	// Clear any to-delete entries in each endpoint, then update globally to
   196  	// insert any entries that now should be in the global cache (because they
   197  	// provide an IP at the latest expiration time).
   198  	namesToRegen := n.cache.ForceExpire(expireLookupsBefore, nameMatcher)
   199  	for _, ep := range n.config.GetEndpointsDNSInfo("") {
   200  		namesToRegen = namesToRegen.Union(ep.DNSHistory.ForceExpire(expireLookupsBefore, nameMatcher))
   201  		n.cache.UpdateFromCache(ep.DNSHistory, nil)
   202  
   203  		namesToRegen.Insert(ep.DNSZombies.ForceExpire(expireLookupsBefore, nameMatcher)...)
   204  		activeConnections := NewDNSCache(0)
   205  		zombies, _ := ep.DNSZombies.GC()
   206  		lookupTime := time.Now()
   207  		for _, zombie := range zombies {
   208  			namesToRegen.Insert(zombie.Names...)
   209  			for _, name := range zombie.Names {
   210  				activeConnections.Update(lookupTime, name, []netip.Addr{zombie.IP}, 0)
   211  			}
   212  		}
   213  		n.cache.UpdateFromCache(activeConnections, nil)
   214  	}
   215  
   216  	// We may have removed entries; remove them from the ipcache metadata layer
   217  	n.maybeRemoveMetadata(maybeStaleIPs)
   218  	return nil
   219  }
   220  
   221  // RestoreCache loads cache state from the restored system:
   222  // - adds any pre-cached DNS entries
   223  // - repopulates the cache from the (persisted) endpoint DNS cache and zombies
   224  func (n *NameManager) RestoreCache(preCachePath string, restoredEPs []EndpointDNSInfo) {
   225  	// Prefill the cache with the CLI provided pre-cache data. This allows various bridging arrangements during upgrades, or just ensure critical DNS mappings remain.
   226  	if preCachePath != "" {
   227  		log.WithField(logfields.Path, preCachePath).Info("Reading toFQDNs pre-cache data")
   228  		precache, err := readPreCache(preCachePath)
   229  		if err != nil {
   230  			// FIXME: add a link to the "documented format"
   231  			log.WithError(err).WithField(logfields.Path, preCachePath).Error("Cannot parse toFQDNs pre-cache data. Please ensure the file is JSON and follows the documented format")
   232  			// We do not stop the agent here. It is safer to continue with best effort
   233  			// than to enter crash backoffs when this file is broken.
   234  		} else {
   235  			n.cache.UpdateFromCache(precache, nil)
   236  		}
   237  	}
   238  
   239  	// Prefill the cache with DNS lookups from restored endpoints. This is needed
   240  	// to maintain continuity of which IPs are allowed. The GC cascade logic
   241  	// below mimics the logic found in the dns-garbage-collector controller.
   242  	// Note: This is TTL aware, and expired data will not be used (e.g. when
   243  	// restoring after a long delay).
   244  	now := time.Now()
   245  	for _, possibleEP := range restoredEPs {
   246  		// Upgrades from old ciliums have this nil
   247  		if possibleEP.DNSHistory != nil {
   248  			n.cache.UpdateFromCache(possibleEP.DNSHistory, []string{})
   249  
   250  			// GC any connections that have expired, but propagate it to the zombies
   251  			// list. DNSCache.GC can handle a nil DNSZombies parameter. We use the
   252  			// actual now time because we are checkpointing at restore time.
   253  			possibleEP.DNSHistory.GC(now, possibleEP.DNSZombies)
   254  		}
   255  
   256  		if possibleEP.DNSZombies != nil {
   257  			lookupTime := time.Now()
   258  			alive, _ := possibleEP.DNSZombies.GC()
   259  			for _, zombie := range alive {
   260  				for _, name := range zombie.Names {
   261  					n.cache.Update(lookupTime, name, []netip.Addr{zombie.IP}, int(2*DNSGCJobInterval.Seconds()))
   262  				}
   263  			}
   264  		}
   265  	}
   266  
   267  	if option.Config.RestoreState {
   268  		checkpointPath := filepath.Join(option.Config.StateDir, checkpointFile)
   269  
   270  		// Restore selector labels in IPCache when upgrading from v1.15. This is needed
   271  		// because Cilium v1.15 and older used to use CIDR identities for ToFQDN identities.
   272  		// This can be removed once Cilium v1.15 is end of life. When restoring from
   273  		// Cilium v1.16 and newer, we can rely on identity restoration to inject the
   274  		// correct identities into IPCache
   275  		oldSelectors, err := restoreSelectors(checkpointPath)
   276  		if err != nil {
   277  			log.WithError(err).WithField(logfields.Path, checkpointPath).Error("Failed to restore FQDN selectors. " +
   278  				"Expect brief traffic disruptions for ToFQDN destinations during initial endpoint regeneration")
   279  			return
   280  		}
   281  		if len(oldSelectors) == 0 {
   282  			log.Info("No FQDN selectors to restore from previous Cilium v1.15 installations")
   283  			return
   284  		}
   285  
   286  		// If we are upgrading from Cilium v1.15, we need to provide the expected
   287  		// FQDN labels into IPCache before label injection and endpoint restoration starts.
   288  		// This ensures that the prefix labels (and thus the numeric identity of the prefix)
   289  		// will not change once the real selectors are discovered during initial endpoint regeneration.
   290  		// Without this, the initial endpoint regeneration might cause drops as old and new IPCache
   291  		// will use different identities for the prefixes in IPCache.
   292  		// These restored labels are removed in CompleteBootstrap, which is invoked after the real
   293  		// labels have been discovered in endpoint regeneration.
   294  		log.Info("Detected upgrade from Cilium v1.15. Building IPCache metadata from restored FQDN selectors")
   295  
   296  		ipsToNames := n.cache.GetIPs()
   297  		ipcacheUpdates := make([]ipcache.MU, 0, len(ipsToNames))
   298  		n.restoredPrefixes = make(sets.Set[netip.Prefix], len(ipsToNames))
   299  
   300  		for addr, names := range ipsToNames {
   301  			lbls := labels.Labels{}
   302  			for _, name := range names {
   303  				lbls.MergeLabels(deriveLabelsForName(name, oldSelectors))
   304  			}
   305  
   306  			prefix := netip.PrefixFrom(addr, addr.BitLen())
   307  			ipcacheUpdates = append(ipcacheUpdates, ipcache.MU{
   308  				Prefix:   prefix,
   309  				Source:   source.Restored,
   310  				Resource: restorationIPCacheResource,
   311  				Metadata: []ipcache.IPMetadata{
   312  					lbls,
   313  				},
   314  			})
   315  			n.restoredPrefixes.Insert(prefix)
   316  		}
   317  		n.config.IPCache.UpsertMetadataBatch(ipcacheUpdates...)
   318  	}
   319  }
   320  
   321  func restoreSelectors(checkpointPath string) (map[api.FQDNSelector]*regexp.Regexp, error) {
   322  	f, err := os.Open(checkpointPath)
   323  	if err != nil {
   324  		// If not upgrading from Cilium v1.15, the file will not exist
   325  		if errors.Is(err, os.ErrNotExist) {
   326  			return nil, nil
   327  		}
   328  		return nil, fmt.Errorf("failed to open selector checkpoint file: %w", err)
   329  	}
   330  
   331  	var restoredSelectors []serializedSelector
   332  	err = json.NewDecoder(f).Decode(&restoredSelectors)
   333  	if err != nil {
   334  		return nil, fmt.Errorf("failed to decode checkpointed selectors from: %w", err)
   335  	}
   336  
   337  	oldSelectors := make(map[api.FQDNSelector]*regexp.Regexp, len(restoredSelectors))
   338  	for _, s := range restoredSelectors {
   339  		re, err := regexp.Compile(s.Regex)
   340  		if err != nil {
   341  			return nil, fmt.Errorf("invalid regex %q in checkpointed selectors: %w", s.Regex, err)
   342  		}
   343  		oldSelectors[s.Selector] = re
   344  	}
   345  	return oldSelectors, nil
   346  }
   347  
   348  // readPreCache returns a fqdn.DNSCache object created from the json data at
   349  // preCachePath
   350  func readPreCache(preCachePath string) (cache *DNSCache, err error) {
   351  	data, err := os.ReadFile(preCachePath)
   352  	if err != nil {
   353  		return nil, err
   354  	}
   355  
   356  	cache = NewDNSCache(0) // no per-host limit here
   357  	if err = cache.UnmarshalJSON(data); err != nil {
   358  		return nil, err
   359  	}
   360  	return cache, nil
   361  }