k8s.io/apiserver@v0.31.1/pkg/server/dynamiccertificates/dynamic_cafile_content.go (about)

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package dynamiccertificates
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/x509"
    23  	"fmt"
    24  	"os"
    25  	"sync/atomic"
    26  	"time"
    27  
    28  	"github.com/fsnotify/fsnotify"
    29  	"k8s.io/client-go/util/cert"
    30  
    31  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    32  	"k8s.io/apimachinery/pkg/util/wait"
    33  	"k8s.io/client-go/util/workqueue"
    34  	"k8s.io/klog/v2"
    35  )
    36  
    37  // FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
    38  var FileRefreshDuration = 1 * time.Minute
    39  
    40  // ControllerRunner is a generic interface for starting a controller
    41  type ControllerRunner interface {
    42  	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
    43  	RunOnce(ctx context.Context) error
    44  
    45  	// Run should be called a go .Run
    46  	Run(ctx context.Context, workers int)
    47  }
    48  
    49  // DynamicFileCAContent provides a CAContentProvider that can dynamically react to new file content
    50  // It also fulfills the authenticator interface to provide verifyoptions
    51  type DynamicFileCAContent struct {
    52  	name string
    53  
    54  	// filename is the name the file to read.
    55  	filename string
    56  
    57  	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
    58  	caBundle atomic.Value
    59  
    60  	listeners []Listener
    61  
    62  	// queue only ever has one item, but it has nice error handling backoff/retry semantics
    63  	queue workqueue.TypedRateLimitingInterface[string]
    64  }
    65  
    66  var _ Notifier = &DynamicFileCAContent{}
    67  var _ CAContentProvider = &DynamicFileCAContent{}
    68  var _ ControllerRunner = &DynamicFileCAContent{}
    69  
    70  type caBundleAndVerifier struct {
    71  	caBundle      []byte
    72  	verifyOptions x509.VerifyOptions
    73  }
    74  
    75  // NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
    76  func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
    77  	if len(filename) == 0 {
    78  		return nil, fmt.Errorf("missing filename for ca bundle")
    79  	}
    80  	name := fmt.Sprintf("%s::%s", purpose, filename)
    81  
    82  	ret := &DynamicFileCAContent{
    83  		name:     name,
    84  		filename: filename,
    85  		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
    86  			workqueue.DefaultTypedControllerRateLimiter[string](),
    87  			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicCABundle-%s", purpose)},
    88  		),
    89  	}
    90  	if err := ret.loadCABundle(); err != nil {
    91  		return nil, err
    92  	}
    93  
    94  	return ret, nil
    95  }
    96  
    97  // AddListener adds a listener to be notified when the CA content changes.
    98  func (c *DynamicFileCAContent) AddListener(listener Listener) {
    99  	c.listeners = append(c.listeners, listener)
   100  }
   101  
   102  // loadCABundle determines the next set of content for the file.
   103  func (c *DynamicFileCAContent) loadCABundle() error {
   104  	caBundle, err := os.ReadFile(c.filename)
   105  	if err != nil {
   106  		return err
   107  	}
   108  	if len(caBundle) == 0 {
   109  		return fmt.Errorf("missing content for CA bundle %q", c.Name())
   110  	}
   111  
   112  	// check to see if we have a change. If the values are the same, do nothing.
   113  	if !c.hasCAChanged(caBundle) {
   114  		return nil
   115  	}
   116  
   117  	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle)
   118  	if err != nil {
   119  		return err
   120  	}
   121  	c.caBundle.Store(caBundleAndVerifier)
   122  	klog.V(2).InfoS("Loaded a new CA Bundle and Verifier", "name", c.Name())
   123  
   124  	for _, listener := range c.listeners {
   125  		listener.Enqueue()
   126  	}
   127  
   128  	return nil
   129  }
   130  
   131  // hasCAChanged returns true if the caBundle is different than the current.
   132  func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
   133  	uncastExisting := c.caBundle.Load()
   134  	if uncastExisting == nil {
   135  		return true
   136  	}
   137  
   138  	// check to see if we have a change. If the values are the same, do nothing.
   139  	existing, ok := uncastExisting.(*caBundleAndVerifier)
   140  	if !ok {
   141  		return true
   142  	}
   143  	if !bytes.Equal(existing.caBundle, caBundle) {
   144  		return true
   145  	}
   146  
   147  	return false
   148  }
   149  
   150  // RunOnce runs a single sync loop
   151  func (c *DynamicFileCAContent) RunOnce(ctx context.Context) error {
   152  	return c.loadCABundle()
   153  }
   154  
   155  // Run starts the controller and blocks until stopCh is closed.
   156  func (c *DynamicFileCAContent) Run(ctx context.Context, workers int) {
   157  	defer utilruntime.HandleCrash()
   158  	defer c.queue.ShutDown()
   159  
   160  	klog.InfoS("Starting controller", "name", c.name)
   161  	defer klog.InfoS("Shutting down controller", "name", c.name)
   162  
   163  	// doesn't matter what workers say, only start one.
   164  	go wait.Until(c.runWorker, time.Second, ctx.Done())
   165  
   166  	// start the loop that watches the CA file until stopCh is closed.
   167  	go wait.Until(func() {
   168  		if err := c.watchCAFile(ctx.Done()); err != nil {
   169  			klog.ErrorS(err, "Failed to watch CA file, will retry later")
   170  		}
   171  	}, time.Minute, ctx.Done())
   172  
   173  	<-ctx.Done()
   174  }
   175  
   176  func (c *DynamicFileCAContent) watchCAFile(stopCh <-chan struct{}) error {
   177  	// Trigger a check here to ensure the content will be checked periodically even if the following watch fails.
   178  	c.queue.Add(workItemKey)
   179  
   180  	w, err := fsnotify.NewWatcher()
   181  	if err != nil {
   182  		return fmt.Errorf("error creating fsnotify watcher: %v", err)
   183  	}
   184  	defer w.Close()
   185  
   186  	if err = w.Add(c.filename); err != nil {
   187  		return fmt.Errorf("error adding watch for file %s: %v", c.filename, err)
   188  	}
   189  	// Trigger a check in case the file is updated before the watch starts.
   190  	c.queue.Add(workItemKey)
   191  
   192  	for {
   193  		select {
   194  		case e := <-w.Events:
   195  			if err := c.handleWatchEvent(e, w); err != nil {
   196  				return err
   197  			}
   198  		case err := <-w.Errors:
   199  			return fmt.Errorf("received fsnotify error: %v", err)
   200  		case <-stopCh:
   201  			return nil
   202  		}
   203  	}
   204  }
   205  
   206  // handleWatchEvent triggers reloading the CA file, and restarts a new watch if it's a Remove or Rename event.
   207  func (c *DynamicFileCAContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
   208  	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
   209  	defer c.queue.Add(workItemKey)
   210  	if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
   211  		return nil
   212  	}
   213  	if err := w.Remove(c.filename); err != nil {
   214  		klog.InfoS("Failed to remove file watch, it may have been deleted", "file", c.filename, "err", err)
   215  	}
   216  	if err := w.Add(c.filename); err != nil {
   217  		return fmt.Errorf("error adding watch for file %s: %v", c.filename, err)
   218  	}
   219  	return nil
   220  }
   221  
   222  func (c *DynamicFileCAContent) runWorker() {
   223  	for c.processNextWorkItem() {
   224  	}
   225  }
   226  
   227  func (c *DynamicFileCAContent) processNextWorkItem() bool {
   228  	dsKey, quit := c.queue.Get()
   229  	if quit {
   230  		return false
   231  	}
   232  	defer c.queue.Done(dsKey)
   233  
   234  	err := c.loadCABundle()
   235  	if err == nil {
   236  		c.queue.Forget(dsKey)
   237  		return true
   238  	}
   239  
   240  	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
   241  	c.queue.AddRateLimited(dsKey)
   242  
   243  	return true
   244  }
   245  
   246  // Name is just an identifier
   247  func (c *DynamicFileCAContent) Name() string {
   248  	return c.name
   249  }
   250  
   251  // CurrentCABundleContent provides ca bundle byte content
   252  func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
   253  	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
   254  }
   255  
   256  // VerifyOptions provides verifyoptions compatible with authenticators
   257  func (c *DynamicFileCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
   258  	uncastObj := c.caBundle.Load()
   259  	if uncastObj == nil {
   260  		return x509.VerifyOptions{}, false
   261  	}
   262  
   263  	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
   264  }
   265  
   266  // newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.
   267  // It will return a nil function if you pass an empty CA file.
   268  func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) {
   269  	if len(caBundle) == 0 {
   270  		return nil, fmt.Errorf("missing content for CA bundle %q", name)
   271  	}
   272  
   273  	// Wrap with an x509 verifier
   274  	var err error
   275  	verifyOptions := defaultVerifyOptions()
   276  	verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle)
   277  	if err != nil {
   278  		return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err)
   279  	}
   280  
   281  	return &caBundleAndVerifier{
   282  		caBundle:      caBundle,
   283  		verifyOptions: verifyOptions,
   284  	}, nil
   285  }
   286  
   287  // defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time,
   288  // and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth)
   289  func defaultVerifyOptions() x509.VerifyOptions {
   290  	return x509.VerifyOptions{
   291  		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
   292  	}
   293  }