vitess.io/vitess@v0.16.2/go/vt/topo/k8stopo/server.go (about) 1 /* 2 Copyright 2020 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 /* 18 Package k8stopo implements topo.Server with the Kubernetes API as the backend. 19 20 We expect the following behavior from the kubernetes client library: 21 22 - TODO 23 24 We follow these conventions within this package: 25 26 - TODO 27 */ 28 package k8stopo 29 30 import ( 31 "fmt" 32 "os" 33 "path/filepath" 34 "strings" 35 36 "vitess.io/vitess/go/vt/servenv" 37 38 "github.com/spf13/pflag" 39 40 "k8s.io/apimachinery/pkg/fields" 41 "k8s.io/client-go/kubernetes" 42 "k8s.io/client-go/rest" 43 "k8s.io/client-go/tools/cache" 44 "k8s.io/client-go/tools/clientcmd" 45 46 "vitess.io/vitess/go/vt/log" 47 "vitess.io/vitess/go/vt/topo" 48 vtv1beta1 "vitess.io/vitess/go/vt/topo/k8stopo/apis/topo/v1beta1" 49 vtkube "vitess.io/vitess/go/vt/topo/k8stopo/client/clientset/versioned" 50 vttyped "vitess.io/vitess/go/vt/topo/k8stopo/client/clientset/versioned/typed/topo/v1beta1" 51 ) 52 53 var kubeconfigPath, configContext, configNamespace string 54 55 func init() { 56 servenv.RegisterFlagsForTopoBinaries(registerK8STopoFlags) 57 } 58 59 func registerK8STopoFlags(fs *pflag.FlagSet) { 60 // kubeconfigPath is a string that gives the location of a valid kubeconfig file 61 fs.StringVar(&kubeconfigPath, "topo_k8s_kubeconfig", kubeconfigPath, "Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod") 62 63 // configContext is a string that can be used to override the default context 64 fs.StringVar(&configContext, "topo_k8s_context", configContext, "The kubeconfig context to use, overrides the 'current-context' from the config") 65 66 // configNamespace is a string that can be used to override the default namespace for objects 67 fs.StringVar(&configNamespace, "topo_k8s_namespace", configNamespace, "The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config") 68 } 69 70 // Factory is the Kubernetes topo.Factory implementation. 71 type Factory struct{} 72 73 // HasGlobalReadOnlyCell is part of the topo.Factory interface. 74 func (f Factory) HasGlobalReadOnlyCell(serverAddr, root string) bool { 75 return false 76 } 77 78 // Create is part of the topo.Factory interface. 79 func (f Factory) Create(cell, serverAddr, root string) (topo.Conn, error) { 80 return NewServer(serverAddr, root) 81 } 82 83 // Server is the implementation of topo.Server for Kubernetes. 84 type Server struct { 85 // kubeClient is the entire kubernetes interface 86 kubeClient kubernetes.Interface 87 88 // vtKubeClient is the client for vitess api types 89 vtKubeClient vtkube.Interface 90 91 // resource is a scoped-down kubernetes.Interface used for convenience 92 resourceClient vttyped.VitessTopoNodeInterface 93 94 // stopChan is used to tell the client-go informers to quit 95 stopChan chan struct{} 96 97 // memberInformer is the controller that syncronized the cache of data 98 memberInformer cache.Controller 99 100 // memberIndexer is the cache of tree data 101 memberIndexer cache.Indexer 102 103 // namespace is the Kubernetes namespace to be used for all resources 104 namespace string 105 106 // root is the root path for this client. 107 // used for resource prefixing 108 root string 109 } 110 111 // Close implements topo.Server.Close. 112 func (s *Server) Close() { 113 close(s.stopChan) 114 } 115 116 func getKeyParents(key string) []string { 117 parents := []string{""} 118 parent := []string{} 119 for _, segment := range strings.Split(filepath.Dir(key), "/") { 120 parent = append(parent, segment) 121 parents = append(parents, strings.Join(parent, "/")) 122 } 123 return parents 124 } 125 126 func indexByParent(obj any) ([]string, error) { 127 return getKeyParents(obj.(*vtv1beta1.VitessTopoNode).Data.Key), nil 128 } 129 130 // syncTree starts and syncs the member objects that form the directory "tree" 131 func (s *Server) syncTree() error { 132 // Create the informer / indexer 133 restClient := s.vtKubeClient.TopoV1beta1().RESTClient() 134 listwatch := cache.NewListWatchFromClient(restClient, "vitesstoponodes", s.namespace, fields.Everything()) 135 136 // set up index funcs 137 indexers := cache.Indexers{} 138 indexers["by_parent"] = indexByParent 139 140 s.memberIndexer, s.memberInformer = cache.NewIndexerInformer(listwatch, &vtv1beta1.VitessTopoNode{}, 0, 141 cache.ResourceEventHandlerFuncs{}, indexers) 142 143 // Start indexer 144 go s.memberInformer.Run(s.stopChan) 145 146 // Wait for sync 147 log.Info("Waiting for Kubernetes topo cache sync") 148 if !cache.WaitForCacheSync(s.stopChan, s.memberInformer.HasSynced) { 149 return fmt.Errorf("timed out waiting for caches to sync") 150 } 151 log.Info("Kubernetes topo cache sync completed") 152 153 return nil 154 } 155 156 // NewServer returns a new k8stopo.Server. 157 func NewServer(_, root string) (*Server, error) { 158 log.Info("Creating new Kubernetes topo server with root: ", root) 159 160 var config *rest.Config 161 var err error 162 namespace := "default" //nolint 163 164 if kubeconfigPath == "" { 165 log.Info("Creating new in-cluster Kubernetes config") 166 167 config, err = rest.InClusterConfig() 168 if err != nil { 169 return nil, fmt.Errorf("error getting Kubernetes in-cluster client config: %s", err) 170 } 171 172 // When running in the cluster, use the namespace file to detect the current namespace 173 nsBytes, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") 174 if err != nil { 175 return nil, err 176 } 177 namespace = string(nsBytes) 178 } else { 179 log.Info("Creating new Kubernetes config from kubeconfig", kubeconfigPath) 180 181 configOverrides := &clientcmd.ConfigOverrides{} 182 183 // respect the context flag 184 if configContext != "" { 185 configOverrides.CurrentContext = configContext 186 log.V(7).Info("Overriding Kubernetes config context with: ", configOverrides.CurrentContext) 187 } 188 189 configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 190 &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, 191 configOverrides, 192 ) 193 194 config, err = configLoader.ClientConfig() 195 if err != nil { 196 return nil, fmt.Errorf("error getting Kubernetes client config: %s", err) 197 } 198 199 // When given a kubeconfig file, use the namespace from the current context 200 namespace, _, err = configLoader.Namespace() 201 if err != nil { 202 return nil, fmt.Errorf("error getting namespace from Kubernetes client config: %s", err) 203 } 204 } 205 206 // respect the namespace flag 207 if configNamespace != "" { 208 namespace = configNamespace 209 log.V(7).Info("Overriding Kubernetes config namespace with: ", namespace) 210 } 211 212 // create the kubernetes client 213 kubeClientset, err := kubernetes.NewForConfig(config) 214 if err != nil { 215 return nil, fmt.Errorf("error creating official Kubernetes client: %s", err) 216 } 217 218 vtKubeClientset, err := vtkube.NewForConfig(config) 219 if err != nil { 220 return nil, fmt.Errorf("error creating vitess Kubernetes client: %s", err) 221 } 222 223 // Create the server 224 s := &Server{ 225 namespace: namespace, 226 kubeClient: kubeClientset, 227 vtKubeClient: vtKubeClientset, 228 resourceClient: vtKubeClientset.TopoV1beta1().VitessTopoNodes(namespace), 229 root: root, 230 stopChan: make(chan struct{}), 231 } 232 233 // Sync cache 234 if err = s.syncTree(); err != nil { 235 return nil, err 236 } 237 238 return s, nil 239 } 240 241 func init() { 242 topo.RegisterFactory("k8s", Factory{}) 243 }