github.com/cilium/cilium@v1.16.2/pkg/fqdn/name_manager_gc.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package fqdn 5 6 import ( 7 "context" 8 "encoding/json" 9 "errors" 10 "fmt" 11 "net/netip" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "k8s.io/apimachinery/pkg/util/sets" 18 19 "github.com/cilium/cilium/pkg/controller" 20 "github.com/cilium/cilium/pkg/fqdn/matchpattern" 21 "github.com/cilium/cilium/pkg/ipcache" 22 ipcacheTypes "github.com/cilium/cilium/pkg/ipcache/types" 23 "github.com/cilium/cilium/pkg/labels" 24 "github.com/cilium/cilium/pkg/logging/logfields" 25 "github.com/cilium/cilium/pkg/metrics" 26 "github.com/cilium/cilium/pkg/option" 27 "github.com/cilium/cilium/pkg/policy/api" 28 "github.com/cilium/cilium/pkg/source" 29 "github.com/cilium/cilium/pkg/time" 30 ) 31 32 const DNSGCJobInterval = 1 * time.Minute 33 34 const dnsGCJobName = "dns-garbage-collector-job" 35 36 var ( 37 dnsGCControllerGroup = controller.NewGroup(dnsGCJobName) 38 ) 39 40 var ( 41 checkpointFile = "fqdn-name-manager-selectors.json" 42 43 restorationIPCacheResource = ipcacheTypes.NewResourceID(ipcacheTypes.ResourceKindDaemon, "", "fqdn-name-manager-restoration") 44 ) 45 46 // serializedSelector is the schema of the serialized selectors on disk 47 type serializedSelector struct { 48 Regex string `json:"re"` 49 Selector api.FQDNSelector `json:"sel"` 50 } 51 52 // This implements some garbage collection and cleanup functions for the NameManager 53 54 // GC cleans up TTL expired entries from the DNS policies. 55 // It removes stale or undesired entries from the DNS caches. 56 // This is done for all per-EP DNSCache 57 // instances (ep.DNSHistory) with evictions (whether due to TTL expiry or 58 // overlimit eviction) cascaded into ep.DNSZombies. Data in DNSHistory and 59 // DNSZombies is further collected into the global DNSCache instance. The 60 // data there drives toFQDNs policy via NameManager and ToFQDNs selectors. 61 // DNSCache entries expire data when the TTL elapses and when the entries for 62 // a DNS name are above a limit. The data is then placed into 63 // DNSZombieMappings instances. These rely on the CT GC loop to update 64 // liveness for each to-delete IP. When an IP is not in-use it is finally 65 // deleted from the global DNSCache. Until then, each of these IPs is 66 // inserted into the global cache as a synthetic DNS lookup. 67 func (n *NameManager) GC(ctx context.Context) error { 68 var ( 69 GCStart = time.Now() 70 71 // activeConnections holds DNSName -> single IP entries that have been 72 // marked active by the CT GC. Since we expire in this controller, we 73 // give these entries 2 cycles of TTL to allow for timing mismatches 74 // with the CT GC. 75 activeConnectionsTTL = int(2 * DNSGCJobInterval.Seconds()) 76 activeConnections = NewDNSCache(activeConnectionsTTL) 77 ) 78 namesToClean := make(sets.Set[string]) 79 80 // Take a snapshot of the *entire* reverse cache, so we can compute the set of 81 // IPs that have been completely removed and safely delete their metadata. 82 maybeStaleIPs := n.cache.GetIPs() 83 84 // Cleanup each endpoint cache, deferring deletions via DNSZombies. 85 endpoints := n.config.GetEndpointsDNSInfo("") 86 for _, ep := range endpoints { 87 epID := ep.ID 88 if metrics.FQDNActiveNames.IsEnabled() || metrics.FQDNActiveIPs.IsEnabled() { 89 countFQDNs, countIPs := ep.DNSHistory.Count() 90 if metrics.FQDNActiveNames.IsEnabled() { 91 metrics.FQDNActiveNames.WithLabelValues(epID).Set(float64(countFQDNs)) 92 } 93 if metrics.FQDNActiveIPs.IsEnabled() { 94 metrics.FQDNActiveIPs.WithLabelValues(epID).Set(float64(countIPs)) 95 } 96 } 97 affectedNames := ep.DNSHistory.GC(GCStart, ep.DNSZombies) 98 namesToClean = namesToClean.Union(affectedNames) 99 100 alive, dead := ep.DNSZombies.GC() 101 if metrics.FQDNAliveZombieConnections.IsEnabled() { 102 metrics.FQDNAliveZombieConnections.WithLabelValues(epID).Set(float64(len(alive))) 103 } 104 105 // Alive zombie need to be added to the global cache as name->IP 106 // entries. 107 // 108 // NB: The following comment is _no longer true_ (see 109 // DNSZombies.GC()). We keep it to maintain the original intention 110 // of the code for future reference: 111 // We accumulate the names into namesToClean to ensure that the 112 // original full DNS lookup (name -> many IPs) is expired and 113 // only the active connections (name->single IP) are re-added. 114 // Note: Other DNS lookups may also use an active IP. This is 115 // fine. 116 // 117 lookupTime := time.Now() 118 for _, zombie := range alive { 119 for _, name := range zombie.Names { 120 namesToClean.Insert(name) 121 activeConnections.Update(lookupTime, name, []netip.Addr{zombie.IP}, activeConnectionsTTL) 122 } 123 } 124 125 // Dead entries can be deleted outright, without any replacement. 126 // Entries here have been evicted from the DNS cache (via .GC due to 127 // TTL expiration or overlimit) and are no longer active connections. 128 for _, zombie := range dead { 129 namesToClean.Insert(zombie.Names...) 130 } 131 } 132 133 if namesToClean.Len() == 0 { 134 return nil 135 } 136 137 // Collect DNS data into the global cache. This aggregates all endpoint 138 // and existing connection data into one place for use elsewhere. 139 // In the case where a lookup occurs in a race with .ReplaceFromCache the 140 // result is consistent: 141 // - If before, the ReplaceFromCache will use the new data when pulling 142 // in from each EP cache. 143 // - If after, the normal update process occurs after .ReplaceFromCache 144 // releases its locks. 145 caches := []*DNSCache{activeConnections} 146 for _, ep := range endpoints { 147 caches = append(caches, ep.DNSHistory) 148 } 149 150 namesToCleanSlice := namesToClean.UnsortedList() 151 152 n.cache.ReplaceFromCacheByNames(namesToCleanSlice, caches...) 153 154 metrics.FQDNGarbageCollectorCleanedTotal.Add(float64(len(namesToCleanSlice))) 155 namesCount := len(namesToCleanSlice) 156 // Limit the amount of info level logging to some sane amount 157 if namesCount > 20 { 158 // namedsToClean is only used for logging after this so we can reslice it in place 159 namesToCleanSlice = namesToCleanSlice[:20] 160 } 161 log.WithField(logfields.Controller, dnsGCJobName).Infof( 162 "FQDN garbage collector work deleted %d name entries: %s", namesCount, strings.Join(namesToCleanSlice, ",")) 163 164 // Remove any now-stale ipcache metadata. 165 // Need to RLock here so we don't race on re-insertion. 166 n.maybeRemoveMetadata(maybeStaleIPs) 167 168 return nil 169 } 170 171 func (n *NameManager) StartGC(ctx context.Context) { 172 n.manager.UpdateController(dnsGCJobName, controller.ControllerParams{ 173 Group: dnsGCControllerGroup, 174 RunInterval: DNSGCJobInterval, 175 DoFunc: n.GC, 176 Context: ctx, 177 }) 178 } 179 180 // DeleteDNSLookups force-removes any entries in *all* caches that are not currently actively 181 // passing traffic. 182 func (n *NameManager) DeleteDNSLookups(expireLookupsBefore time.Time, matchPatternStr string) error { 183 var nameMatcher *regexp.Regexp // nil matches all in our implementation 184 if matchPatternStr != "" { 185 var err error 186 nameMatcher, err = matchpattern.ValidateWithoutCache(matchPatternStr) 187 if err != nil { 188 return err 189 } 190 } 191 192 maybeStaleIPs := n.cache.GetIPs() 193 194 // Clear any to-delete entries globally 195 // Clear any to-delete entries in each endpoint, then update globally to 196 // insert any entries that now should be in the global cache (because they 197 // provide an IP at the latest expiration time). 198 namesToRegen := n.cache.ForceExpire(expireLookupsBefore, nameMatcher) 199 for _, ep := range n.config.GetEndpointsDNSInfo("") { 200 namesToRegen = namesToRegen.Union(ep.DNSHistory.ForceExpire(expireLookupsBefore, nameMatcher)) 201 n.cache.UpdateFromCache(ep.DNSHistory, nil) 202 203 namesToRegen.Insert(ep.DNSZombies.ForceExpire(expireLookupsBefore, nameMatcher)...) 204 activeConnections := NewDNSCache(0) 205 zombies, _ := ep.DNSZombies.GC() 206 lookupTime := time.Now() 207 for _, zombie := range zombies { 208 namesToRegen.Insert(zombie.Names...) 209 for _, name := range zombie.Names { 210 activeConnections.Update(lookupTime, name, []netip.Addr{zombie.IP}, 0) 211 } 212 } 213 n.cache.UpdateFromCache(activeConnections, nil) 214 } 215 216 // We may have removed entries; remove them from the ipcache metadata layer 217 n.maybeRemoveMetadata(maybeStaleIPs) 218 return nil 219 } 220 221 // RestoreCache loads cache state from the restored system: 222 // - adds any pre-cached DNS entries 223 // - repopulates the cache from the (persisted) endpoint DNS cache and zombies 224 func (n *NameManager) RestoreCache(preCachePath string, restoredEPs []EndpointDNSInfo) { 225 // Prefill the cache with the CLI provided pre-cache data. This allows various bridging arrangements during upgrades, or just ensure critical DNS mappings remain. 226 if preCachePath != "" { 227 log.WithField(logfields.Path, preCachePath).Info("Reading toFQDNs pre-cache data") 228 precache, err := readPreCache(preCachePath) 229 if err != nil { 230 // FIXME: add a link to the "documented format" 231 log.WithError(err).WithField(logfields.Path, preCachePath).Error("Cannot parse toFQDNs pre-cache data. Please ensure the file is JSON and follows the documented format") 232 // We do not stop the agent here. It is safer to continue with best effort 233 // than to enter crash backoffs when this file is broken. 234 } else { 235 n.cache.UpdateFromCache(precache, nil) 236 } 237 } 238 239 // Prefill the cache with DNS lookups from restored endpoints. This is needed 240 // to maintain continuity of which IPs are allowed. The GC cascade logic 241 // below mimics the logic found in the dns-garbage-collector controller. 242 // Note: This is TTL aware, and expired data will not be used (e.g. when 243 // restoring after a long delay). 244 now := time.Now() 245 for _, possibleEP := range restoredEPs { 246 // Upgrades from old ciliums have this nil 247 if possibleEP.DNSHistory != nil { 248 n.cache.UpdateFromCache(possibleEP.DNSHistory, []string{}) 249 250 // GC any connections that have expired, but propagate it to the zombies 251 // list. DNSCache.GC can handle a nil DNSZombies parameter. We use the 252 // actual now time because we are checkpointing at restore time. 253 possibleEP.DNSHistory.GC(now, possibleEP.DNSZombies) 254 } 255 256 if possibleEP.DNSZombies != nil { 257 lookupTime := time.Now() 258 alive, _ := possibleEP.DNSZombies.GC() 259 for _, zombie := range alive { 260 for _, name := range zombie.Names { 261 n.cache.Update(lookupTime, name, []netip.Addr{zombie.IP}, int(2*DNSGCJobInterval.Seconds())) 262 } 263 } 264 } 265 } 266 267 if option.Config.RestoreState { 268 checkpointPath := filepath.Join(option.Config.StateDir, checkpointFile) 269 270 // Restore selector labels in IPCache when upgrading from v1.15. This is needed 271 // because Cilium v1.15 and older used to use CIDR identities for ToFQDN identities. 272 // This can be removed once Cilium v1.15 is end of life. When restoring from 273 // Cilium v1.16 and newer, we can rely on identity restoration to inject the 274 // correct identities into IPCache 275 oldSelectors, err := restoreSelectors(checkpointPath) 276 if err != nil { 277 log.WithError(err).WithField(logfields.Path, checkpointPath).Error("Failed to restore FQDN selectors. " + 278 "Expect brief traffic disruptions for ToFQDN destinations during initial endpoint regeneration") 279 return 280 } 281 if len(oldSelectors) == 0 { 282 log.Info("No FQDN selectors to restore from previous Cilium v1.15 installations") 283 return 284 } 285 286 // If we are upgrading from Cilium v1.15, we need to provide the expected 287 // FQDN labels into IPCache before label injection and endpoint restoration starts. 288 // This ensures that the prefix labels (and thus the numeric identity of the prefix) 289 // will not change once the real selectors are discovered during initial endpoint regeneration. 290 // Without this, the initial endpoint regeneration might cause drops as old and new IPCache 291 // will use different identities for the prefixes in IPCache. 292 // These restored labels are removed in CompleteBootstrap, which is invoked after the real 293 // labels have been discovered in endpoint regeneration. 294 log.Info("Detected upgrade from Cilium v1.15. Building IPCache metadata from restored FQDN selectors") 295 296 ipsToNames := n.cache.GetIPs() 297 ipcacheUpdates := make([]ipcache.MU, 0, len(ipsToNames)) 298 n.restoredPrefixes = make(sets.Set[netip.Prefix], len(ipsToNames)) 299 300 for addr, names := range ipsToNames { 301 lbls := labels.Labels{} 302 for _, name := range names { 303 lbls.MergeLabels(deriveLabelsForName(name, oldSelectors)) 304 } 305 306 prefix := netip.PrefixFrom(addr, addr.BitLen()) 307 ipcacheUpdates = append(ipcacheUpdates, ipcache.MU{ 308 Prefix: prefix, 309 Source: source.Restored, 310 Resource: restorationIPCacheResource, 311 Metadata: []ipcache.IPMetadata{ 312 lbls, 313 }, 314 }) 315 n.restoredPrefixes.Insert(prefix) 316 } 317 n.config.IPCache.UpsertMetadataBatch(ipcacheUpdates...) 318 } 319 } 320 321 func restoreSelectors(checkpointPath string) (map[api.FQDNSelector]*regexp.Regexp, error) { 322 f, err := os.Open(checkpointPath) 323 if err != nil { 324 // If not upgrading from Cilium v1.15, the file will not exist 325 if errors.Is(err, os.ErrNotExist) { 326 return nil, nil 327 } 328 return nil, fmt.Errorf("failed to open selector checkpoint file: %w", err) 329 } 330 331 var restoredSelectors []serializedSelector 332 err = json.NewDecoder(f).Decode(&restoredSelectors) 333 if err != nil { 334 return nil, fmt.Errorf("failed to decode checkpointed selectors from: %w", err) 335 } 336 337 oldSelectors := make(map[api.FQDNSelector]*regexp.Regexp, len(restoredSelectors)) 338 for _, s := range restoredSelectors { 339 re, err := regexp.Compile(s.Regex) 340 if err != nil { 341 return nil, fmt.Errorf("invalid regex %q in checkpointed selectors: %w", s.Regex, err) 342 } 343 oldSelectors[s.Selector] = re 344 } 345 return oldSelectors, nil 346 } 347 348 // readPreCache returns a fqdn.DNSCache object created from the json data at 349 // preCachePath 350 func readPreCache(preCachePath string) (cache *DNSCache, err error) { 351 data, err := os.ReadFile(preCachePath) 352 if err != nil { 353 return nil, err 354 } 355 356 cache = NewDNSCache(0) // no per-host limit here 357 if err = cache.UnmarshalJSON(data); err != nil { 358 return nil, err 359 } 360 return cache, nil 361 }