k8s.io/kubernetes@v1.29.3/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner.go

/*
Copyright 2023 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serviceaccount

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/types"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	listersv1 "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"
	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
	"k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking"
	"k8s.io/kubernetes/pkg/serviceaccount"
	"k8s.io/utils/clock"
)

const (
	dateFormat                 = "2006-01-02"
	DefaultCleanerSyncInterval = 24 * time.Hour
)

// TokenCleanerOptions contains options for the LegacySATokenCleaner
type LegacySATokenCleanerOptions struct {
	// CleanUpPeriod is the period of time since the last usage of an legacy token before it can be deleted.
	CleanUpPeriod time.Duration
	SyncInterval  time.Duration
}

// LegacySATokenCleaner is a controller that deletes legacy serviceaccount tokens that are not in use for a specified period of time.
type LegacySATokenCleaner struct {
	client           clientset.Interface
	clock            clock.Clock
	saLister         listersv1.ServiceAccountLister
	saInformerSynced cache.InformerSynced

	secretLister         listersv1.SecretLister
	secretInformerSynced cache.InformerSynced

	podLister         listersv1.PodLister
	podInformerSynced cache.InformerSynced

	syncInterval         time.Duration
	minimumSinceLastUsed time.Duration
}

// NewLegacySATokenCleaner returns a new *NewLegacySATokenCleaner.
func NewLegacySATokenCleaner(saInformer coreinformers.ServiceAccountInformer, secretInformer coreinformers.SecretInformer, podInformer coreinformers.PodInformer, client clientset.Interface, cl clock.Clock, options LegacySATokenCleanerOptions) (*LegacySATokenCleaner, error) {
	if !(options.CleanUpPeriod > 0) {
		return nil, fmt.Errorf("invalid CleanUpPeriod: %v", options.CleanUpPeriod)
	}
	if !(options.SyncInterval > 0) {
		return nil, fmt.Errorf("invalid SyncInterval: %v", options.SyncInterval)
	}

	tc := &LegacySATokenCleaner{
		client:               client,
		clock:                cl,
		saLister:             saInformer.Lister(),
		saInformerSynced:     saInformer.Informer().HasSynced,
		secretLister:         secretInformer.Lister(),
		secretInformerSynced: secretInformer.Informer().HasSynced,
		podLister:            podInformer.Lister(),
		podInformerSynced:    podInformer.Informer().HasSynced,
		minimumSinceLastUsed: options.CleanUpPeriod,
		syncInterval:         options.SyncInterval,
	}

	return tc, nil
}

func (tc *LegacySATokenCleaner) Run(ctx context.Context) {
	defer utilruntime.HandleCrash()

	logger := klog.FromContext(ctx)
	logger.Info("Starting legacy service account token cleaner controller")
	defer logger.Info("Shutting down legacy service account token cleaner controller")

	if !cache.WaitForNamedCacheSync("legacy-service-account-token-cleaner", ctx.Done(), tc.saInformerSynced, tc.secretInformerSynced, tc.podInformerSynced) {
		return
	}

	go wait.UntilWithContext(ctx, tc.evaluateSATokens, tc.syncInterval)

	<-ctx.Done()
}

func (tc *LegacySATokenCleaner) evaluateSATokens(ctx context.Context) {
	logger := klog.FromContext(ctx)

	now := tc.clock.Now().UTC()
	trackedSince, err := tc.latestPossibleTrackedSinceTime(ctx)
	if err != nil {
		logger.Error(err, "Getting lastest possible tracked_since time")
		return
	}

	if now.Before(trackedSince.Add(tc.minimumSinceLastUsed)) {
		// we haven't been tracking long enough
		return
	}

	preserveCreatedOnOrAfter := now.Add(-tc.minimumSinceLastUsed)
	preserveUsedOnOrAfter := now.Add(-tc.minimumSinceLastUsed).Format(dateFormat)

	secretList, err := tc.secretLister.Secrets(metav1.NamespaceAll).List(labels.Everything())
	if err != nil {
		logger.Error(err, "Getting cached secret list")
		return
	}

	namespaceToUsedSecretNames := make(map[string]sets.String)
	for _, secret := range secretList {
		if secret.Type != v1.SecretTypeServiceAccountToken {
			continue
		}
		if !secret.CreationTimestamp.Time.Before(preserveCreatedOnOrAfter) {
			continue
		}

		if secret.DeletionTimestamp != nil {
			continue
		}

		// if LastUsedLabelKey does not exist, we think the secret has not been used
		// since the legacy token starts to track.
		lastUsed, ok := secret.Labels[serviceaccount.LastUsedLabelKey]
		if ok {
			_, err := time.Parse(dateFormat, lastUsed)
			if err != nil {
				// the lastUsed value is not well-formed thus we cannot determine it
				logger.Error(err, "Parsing lastUsed time", "secret", klog.KRef(secret.Namespace, secret.Name))
				continue
			}
			if lastUsed >= preserveUsedOnOrAfter {
				continue
			}
		}

		sa, saErr := tc.getServiceAccount(secret)

		if saErr != nil {
			logger.Error(saErr, "Getting service account", "secret", klog.KRef(secret.Namespace, secret.Name))
			continue
		}
		if sa == nil || !hasSecretReference(sa, secret.Name) {
			// can't determine if this is an auto-generated token
			continue
		}

		mountedSecretNames, err := tc.getMountedSecretNames(secret.Namespace, namespaceToUsedSecretNames)
		if err != nil {
			logger.Error(err, "Resolving mounted secrets", "secret", klog.KRef(secret.Namespace, secret.Name))
			continue
		}
		if mountedSecretNames.Has(secret.Name) {
			// still used by pods
			continue
		}

		invalidSince := secret.Labels[serviceaccount.InvalidSinceLabelKey]
		// If the secret has not been labeled with invalid since date or the label value has invalid format, update the invalidSince label with the current date value.
		_, err = time.Parse(dateFormat, invalidSince)
		if err != nil {
			invalidSince = now.Format(dateFormat)
			logger.Info("Mark the auto-generated service account token as invalid", "invalidSince", invalidSince, "secret", klog.KRef(secret.Namespace, secret.Name))
			patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithUID(secret.UID).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: invalidSince}))
			if err != nil {
				logger.Error(err, "Failed to marshal invalid since label")
			} else {
				if _, err := tc.client.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
					logger.Error(err, "Failed to label legacy service account token secret with invalid since date")
				}
			}
			continue
		}

		if invalidSince >= preserveUsedOnOrAfter {
			continue
		}

		logger.Info("Delete auto-generated service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "creationTime", secret.CreationTimestamp, "lastUsed", lastUsed, "invalidSince", invalidSince)
		if err := tc.client.CoreV1().Secrets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &secret.ResourceVersion}}); err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) {
			logger.Error(err, "Deleting legacy service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "serviceaccount", sa.Name)
		}
	}
}

func (tc *LegacySATokenCleaner) getMountedSecretNames(secretNamespace string, namespaceToUsedSecretNames map[string]sets.String) (sets.String, error) {
	if secrets, ok := namespaceToUsedSecretNames[secretNamespace]; ok {
		return secrets, nil
	}

	podList, err := tc.podLister.Pods(secretNamespace).List(labels.Everything())
	if err != nil {
		return nil, fmt.Errorf("failed to get pod list from pod cache: %v", err)
	}

	var secrets sets.String
	for _, pod := range podList {
		podutil.VisitPodSecretNames(pod, func(secretName string) bool {
			if secrets == nil {
				secrets = sets.NewString()
			}
			secrets.Insert(secretName)
			return true
		})
	}
	if secrets != nil {
		namespaceToUsedSecretNames[secretNamespace] = secrets
	}
	return secrets, nil
}

func (tc *LegacySATokenCleaner) getServiceAccount(secret *v1.Secret) (*v1.ServiceAccount, error) {
	saName := secret.Annotations[v1.ServiceAccountNameKey]
	if len(saName) == 0 {
		return nil, nil
	}
	saUID := types.UID(secret.Annotations[v1.ServiceAccountUIDKey])
	sa, err := tc.saLister.ServiceAccounts(secret.Namespace).Get(saName)
	if apierrors.IsNotFound(err) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}

	// Ensure UID matches if given
	if len(saUID) == 0 || saUID == sa.UID {
		return sa, nil
	}

	return nil, nil
}

// get the latest possible TrackedSince time information from the configMap label.
func (tc *LegacySATokenCleaner) latestPossibleTrackedSinceTime(ctx context.Context) (time.Time, error) {
	configMap, err := tc.client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{})
	if err != nil {
		return time.Time{}, err
	}
	trackedSince, exist := configMap.Data[legacytokentracking.ConfigMapDataKey]
	if !exist {
		return time.Time{}, fmt.Errorf("configMap does not have since label")
	}
	trackedSinceTime, err := time.Parse(dateFormat, trackedSince)
	if err != nil {
		return time.Time{}, fmt.Errorf("error parsing trackedSince time: %v", err)
	}
	// make sure the time to be 00:00 on the day just after the date starts to track
	return trackedSinceTime.AddDate(0, 0, 1), nil
}

func hasSecretReference(serviceAccount *v1.ServiceAccount, secretName string) bool {
	for _, secret := range serviceAccount.Secrets {
		if secret.Name == secretName {
			return true
		}
	}
	return false
}