k8s.io/apiserver@v0.31.1/pkg/server/dynamiccertificates/dynamic_cafile_content.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package dynamiccertificates 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/x509" 23 "fmt" 24 "os" 25 "sync/atomic" 26 "time" 27 28 "github.com/fsnotify/fsnotify" 29 "k8s.io/client-go/util/cert" 30 31 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 32 "k8s.io/apimachinery/pkg/util/wait" 33 "k8s.io/client-go/util/workqueue" 34 "k8s.io/klog/v2" 35 ) 36 37 // FileRefreshDuration is exposed so that integration tests can crank up the reload speed. 38 var FileRefreshDuration = 1 * time.Minute 39 40 // ControllerRunner is a generic interface for starting a controller 41 type ControllerRunner interface { 42 // RunOnce runs the sync loop a single time. This useful for synchronous priming 43 RunOnce(ctx context.Context) error 44 45 // Run should be called a go .Run 46 Run(ctx context.Context, workers int) 47 } 48 49 // DynamicFileCAContent provides a CAContentProvider that can dynamically react to new file content 50 // It also fulfills the authenticator interface to provide verifyoptions 51 type DynamicFileCAContent struct { 52 name string 53 54 // filename is the name the file to read. 55 filename string 56 57 // caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file 58 caBundle atomic.Value 59 60 listeners []Listener 61 62 // queue only ever has one item, but it has nice error handling backoff/retry semantics 63 queue workqueue.TypedRateLimitingInterface[string] 64 } 65 66 var _ Notifier = &DynamicFileCAContent{} 67 var _ CAContentProvider = &DynamicFileCAContent{} 68 var _ ControllerRunner = &DynamicFileCAContent{} 69 70 type caBundleAndVerifier struct { 71 caBundle []byte 72 verifyOptions x509.VerifyOptions 73 } 74 75 // NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content 76 func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) { 77 if len(filename) == 0 { 78 return nil, fmt.Errorf("missing filename for ca bundle") 79 } 80 name := fmt.Sprintf("%s::%s", purpose, filename) 81 82 ret := &DynamicFileCAContent{ 83 name: name, 84 filename: filename, 85 queue: workqueue.NewTypedRateLimitingQueueWithConfig( 86 workqueue.DefaultTypedControllerRateLimiter[string](), 87 workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicCABundle-%s", purpose)}, 88 ), 89 } 90 if err := ret.loadCABundle(); err != nil { 91 return nil, err 92 } 93 94 return ret, nil 95 } 96 97 // AddListener adds a listener to be notified when the CA content changes. 98 func (c *DynamicFileCAContent) AddListener(listener Listener) { 99 c.listeners = append(c.listeners, listener) 100 } 101 102 // loadCABundle determines the next set of content for the file. 103 func (c *DynamicFileCAContent) loadCABundle() error { 104 caBundle, err := os.ReadFile(c.filename) 105 if err != nil { 106 return err 107 } 108 if len(caBundle) == 0 { 109 return fmt.Errorf("missing content for CA bundle %q", c.Name()) 110 } 111 112 // check to see if we have a change. If the values are the same, do nothing. 113 if !c.hasCAChanged(caBundle) { 114 return nil 115 } 116 117 caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle) 118 if err != nil { 119 return err 120 } 121 c.caBundle.Store(caBundleAndVerifier) 122 klog.V(2).InfoS("Loaded a new CA Bundle and Verifier", "name", c.Name()) 123 124 for _, listener := range c.listeners { 125 listener.Enqueue() 126 } 127 128 return nil 129 } 130 131 // hasCAChanged returns true if the caBundle is different than the current. 132 func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool { 133 uncastExisting := c.caBundle.Load() 134 if uncastExisting == nil { 135 return true 136 } 137 138 // check to see if we have a change. If the values are the same, do nothing. 139 existing, ok := uncastExisting.(*caBundleAndVerifier) 140 if !ok { 141 return true 142 } 143 if !bytes.Equal(existing.caBundle, caBundle) { 144 return true 145 } 146 147 return false 148 } 149 150 // RunOnce runs a single sync loop 151 func (c *DynamicFileCAContent) RunOnce(ctx context.Context) error { 152 return c.loadCABundle() 153 } 154 155 // Run starts the controller and blocks until stopCh is closed. 156 func (c *DynamicFileCAContent) Run(ctx context.Context, workers int) { 157 defer utilruntime.HandleCrash() 158 defer c.queue.ShutDown() 159 160 klog.InfoS("Starting controller", "name", c.name) 161 defer klog.InfoS("Shutting down controller", "name", c.name) 162 163 // doesn't matter what workers say, only start one. 164 go wait.Until(c.runWorker, time.Second, ctx.Done()) 165 166 // start the loop that watches the CA file until stopCh is closed. 167 go wait.Until(func() { 168 if err := c.watchCAFile(ctx.Done()); err != nil { 169 klog.ErrorS(err, "Failed to watch CA file, will retry later") 170 } 171 }, time.Minute, ctx.Done()) 172 173 <-ctx.Done() 174 } 175 176 func (c *DynamicFileCAContent) watchCAFile(stopCh <-chan struct{}) error { 177 // Trigger a check here to ensure the content will be checked periodically even if the following watch fails. 178 c.queue.Add(workItemKey) 179 180 w, err := fsnotify.NewWatcher() 181 if err != nil { 182 return fmt.Errorf("error creating fsnotify watcher: %v", err) 183 } 184 defer w.Close() 185 186 if err = w.Add(c.filename); err != nil { 187 return fmt.Errorf("error adding watch for file %s: %v", c.filename, err) 188 } 189 // Trigger a check in case the file is updated before the watch starts. 190 c.queue.Add(workItemKey) 191 192 for { 193 select { 194 case e := <-w.Events: 195 if err := c.handleWatchEvent(e, w); err != nil { 196 return err 197 } 198 case err := <-w.Errors: 199 return fmt.Errorf("received fsnotify error: %v", err) 200 case <-stopCh: 201 return nil 202 } 203 } 204 } 205 206 // handleWatchEvent triggers reloading the CA file, and restarts a new watch if it's a Remove or Rename event. 207 func (c *DynamicFileCAContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error { 208 // This should be executed after restarting the watch (if applicable) to ensure no file event will be missing. 209 defer c.queue.Add(workItemKey) 210 if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) { 211 return nil 212 } 213 if err := w.Remove(c.filename); err != nil { 214 klog.InfoS("Failed to remove file watch, it may have been deleted", "file", c.filename, "err", err) 215 } 216 if err := w.Add(c.filename); err != nil { 217 return fmt.Errorf("error adding watch for file %s: %v", c.filename, err) 218 } 219 return nil 220 } 221 222 func (c *DynamicFileCAContent) runWorker() { 223 for c.processNextWorkItem() { 224 } 225 } 226 227 func (c *DynamicFileCAContent) processNextWorkItem() bool { 228 dsKey, quit := c.queue.Get() 229 if quit { 230 return false 231 } 232 defer c.queue.Done(dsKey) 233 234 err := c.loadCABundle() 235 if err == nil { 236 c.queue.Forget(dsKey) 237 return true 238 } 239 240 utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) 241 c.queue.AddRateLimited(dsKey) 242 243 return true 244 } 245 246 // Name is just an identifier 247 func (c *DynamicFileCAContent) Name() string { 248 return c.name 249 } 250 251 // CurrentCABundleContent provides ca bundle byte content 252 func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) { 253 return c.caBundle.Load().(*caBundleAndVerifier).caBundle 254 } 255 256 // VerifyOptions provides verifyoptions compatible with authenticators 257 func (c *DynamicFileCAContent) VerifyOptions() (x509.VerifyOptions, bool) { 258 uncastObj := c.caBundle.Load() 259 if uncastObj == nil { 260 return x509.VerifyOptions{}, false 261 } 262 263 return uncastObj.(*caBundleAndVerifier).verifyOptions, true 264 } 265 266 // newVerifyOptions creates a new verification func from a file. It reads the content and then fails. 267 // It will return a nil function if you pass an empty CA file. 268 func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) { 269 if len(caBundle) == 0 { 270 return nil, fmt.Errorf("missing content for CA bundle %q", name) 271 } 272 273 // Wrap with an x509 verifier 274 var err error 275 verifyOptions := defaultVerifyOptions() 276 verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle) 277 if err != nil { 278 return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err) 279 } 280 281 return &caBundleAndVerifier{ 282 caBundle: caBundle, 283 verifyOptions: verifyOptions, 284 }, nil 285 } 286 287 // defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time, 288 // and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth) 289 func defaultVerifyOptions() x509.VerifyOptions { 290 return x509.VerifyOptions{ 291 KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, 292 } 293 }