github.com/deemoprobe/k8s-first-commit@v0.0.0-20230430165612-a541f1982be3/pkg/kubelet/kubelet.go (about) 1 /* 2 Copyright 2014 Google Inc. All rights reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package kubelet is ... 18 package kubelet 19 20 import ( 21 "bytes" 22 "encoding/json" 23 "fmt" 24 "io/ioutil" 25 "log" 26 "math/rand" 27 "net/http" 28 "os/exec" 29 "strconv" 30 "strings" 31 "sync" 32 "time" 33 34 "github.com/GoogleCloudPlatform/kubernetes/pkg/api" 35 "github.com/GoogleCloudPlatform/kubernetes/pkg/registry" 36 "github.com/GoogleCloudPlatform/kubernetes/pkg/util" 37 "github.com/coreos/go-etcd/etcd" 38 "github.com/fsouza/go-dockerclient" 39 "gopkg.in/v1/yaml" 40 ) 41 42 // State, sub object of the Docker JSON data 43 type State struct { 44 Running bool 45 } 46 47 // The structured representation of the JSON object returned by Docker inspect 48 type DockerContainerData struct { 49 state State 50 } 51 52 // Interface for testability 53 type DockerInterface interface { 54 ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) 55 InspectContainer(id string) (*docker.Container, error) 56 CreateContainer(docker.CreateContainerOptions) (*docker.Container, error) 57 StartContainer(id string, hostConfig *docker.HostConfig) error 58 StopContainer(id string, timeout uint) error 59 } 60 61 // The main kubelet implementation 62 type Kubelet struct { 63 Client registry.EtcdClient 64 DockerClient DockerInterface 65 FileCheckFrequency time.Duration 66 SyncFrequency time.Duration 67 HTTPCheckFrequency time.Duration 68 pullLock sync.Mutex 69 } 70 71 // Starts background goroutines. If file, manifest_url, or address are empty, 72 // they are not watched. Never returns. 73 func (sl *Kubelet) RunKubelet(file, manifest_url, etcd_servers, address string, port uint) { 74 fileChannel := make(chan api.ContainerManifest) 75 etcdChannel := make(chan []api.ContainerManifest) 76 httpChannel := make(chan api.ContainerManifest) 77 serverChannel := make(chan api.ContainerManifest) 78 79 go util.Forever(func() { sl.WatchFile(file, fileChannel) }, 20*time.Second) 80 if manifest_url != "" { 81 go util.Forever(func() { sl.WatchHTTP(manifest_url, httpChannel) }, 20*time.Second) 82 } 83 if etcd_servers != "" { 84 servers := []string{etcd_servers} 85 log.Printf("Creating etcd client pointing to %v", servers) 86 sl.Client = etcd.NewClient(servers) 87 go util.Forever(func() { sl.SyncAndSetupEtcdWatch(etcdChannel) }, 20*time.Second) 88 } 89 if address != "" { 90 log.Printf("Starting to listen on %s:%d", address, port) 91 handler := KubeletServer{ 92 Kubelet: sl, 93 UpdateChannel: serverChannel, 94 } 95 s := &http.Server{ 96 // TODO: This is broken if address is an ipv6 address. 97 Addr: fmt.Sprintf("%s:%d", address, port), 98 Handler: &handler, 99 ReadTimeout: 10 * time.Second, 100 WriteTimeout: 10 * time.Second, 101 MaxHeaderBytes: 1 << 20, 102 } 103 go util.Forever(func() { s.ListenAndServe() }, 0) 104 } 105 sl.RunSyncLoop(etcdChannel, fileChannel, serverChannel, httpChannel, sl) 106 } 107 108 // Interface implemented by Kubelet, for testability 109 type SyncHandler interface { 110 SyncManifests([]api.ContainerManifest) error 111 } 112 113 // Log an event to the etcd backend. 114 func (sl *Kubelet) LogEvent(event *api.Event) error { 115 if sl.Client == nil { 116 return fmt.Errorf("no etcd client connection.") 117 } 118 event.Timestamp = time.Now().Unix() 119 data, err := json.Marshal(event) 120 if err != nil { 121 return err 122 } 123 124 var response *etcd.Response 125 response, err = sl.Client.AddChild(fmt.Sprintf("/events/%s", event.Container.Name), string(data), 60*60*48 /* 2 days */) 126 // TODO(bburns) : examine response here. 127 if err != nil { 128 log.Printf("Error writing event: %s\n", err) 129 if response != nil { 130 log.Printf("Response was: %#v\n", *response) 131 } 132 } 133 return err 134 } 135 136 // Does this container exist on this host? Returns true if so, and the name under which the container is running. 137 // Returns an error if one occurs. 138 func (sl *Kubelet) ContainerExists(manifest *api.ContainerManifest, container *api.Container) (exists bool, foundName string, err error) { 139 containers, err := sl.ListContainers() 140 if err != nil { 141 return false, "", err 142 } 143 for _, name := range containers { 144 manifestId, containerName := dockerNameToManifestAndContainer(name) 145 if manifestId == manifest.Id && containerName == container.Name { 146 // TODO(bburns) : This leads to an extra list. Convert this to use the returned ID and a straight call 147 // to inspect 148 data, err := sl.GetContainerByName(name) 149 return data != nil, name, err 150 } 151 } 152 return false, "", nil 153 } 154 155 func (sl *Kubelet) GetContainerID(name string) (string, error) { 156 containerList, err := sl.DockerClient.ListContainers(docker.ListContainersOptions{}) 157 if err != nil { 158 return "", err 159 } 160 for _, value := range containerList { 161 if strings.Contains(value.Names[0], name) { 162 return value.ID, nil 163 } 164 } 165 return "", fmt.Errorf("couldn't find name: %s", name) 166 } 167 168 // Get a container by name. 169 // returns the container data from Docker, or an error if one exists. 170 func (sl *Kubelet) GetContainerByName(name string) (*docker.Container, error) { 171 id, err := sl.GetContainerID(name) 172 if err != nil { 173 return nil, err 174 } 175 return sl.DockerClient.InspectContainer(id) 176 } 177 178 func (sl *Kubelet) ListContainers() ([]string, error) { 179 result := []string{} 180 containerList, err := sl.DockerClient.ListContainers(docker.ListContainersOptions{}) 181 if err != nil { 182 return result, err 183 } 184 for _, value := range containerList { 185 result = append(result, value.Names[0]) 186 } 187 return result, err 188 } 189 190 func (sl *Kubelet) pullImage(image string) error { 191 sl.pullLock.Lock() 192 defer sl.pullLock.Unlock() 193 cmd := exec.Command("docker", "pull", image) 194 err := cmd.Start() 195 if err != nil { 196 return err 197 } 198 return cmd.Wait() 199 } 200 201 // Converts "-" to "_-_" and "_" to "___" so that we can use "--" to meaningfully separate parts of a docker name. 202 func escapeDash(in string) (out string) { 203 out = strings.Replace(in, "_", "___", -1) 204 out = strings.Replace(out, "-", "_-_", -1) 205 return 206 } 207 208 // Reverses the transformation of escapeDash. 209 func unescapeDash(in string) (out string) { 210 out = strings.Replace(in, "_-_", "-", -1) 211 out = strings.Replace(out, "___", "_", -1) 212 return 213 } 214 215 // Creates a name which can be reversed to identify both manifest id and container name. 216 func manifestAndContainerToDockerName(manifest *api.ContainerManifest, container *api.Container) string { 217 // Note, manifest.Id could be blank. 218 return fmt.Sprintf("%s--%s--%x", escapeDash(container.Name), escapeDash(manifest.Id), rand.Uint32()) 219 } 220 221 // Upacks a container name, returning the manifest id and container name we would have used to 222 // construct the docker name. If the docker name isn't one we created, we may return empty strings. 223 func dockerNameToManifestAndContainer(name string) (manifestId, containerName string) { 224 // For some reason docker appears to be appending '/' to names. 225 // If its there, strip it. 226 if name[0] == '/' { 227 name = name[1:] 228 } 229 parts := strings.Split(name, "--") 230 if len(parts) > 0 { 231 containerName = unescapeDash(parts[0]) 232 } 233 if len(parts) > 1 { 234 manifestId = unescapeDash(parts[1]) 235 } 236 return 237 } 238 239 func (sl *Kubelet) RunContainer(manifest *api.ContainerManifest, container *api.Container) (name string, err error) { 240 err = sl.pullImage(container.Image) 241 if err != nil { 242 return "", err 243 } 244 245 name = manifestAndContainerToDockerName(manifest, container) 246 envVariables := []string{} 247 for _, value := range container.Env { 248 envVariables = append(envVariables, fmt.Sprintf("%s=%s", value.Name, value.Value)) 249 } 250 251 volumes := map[string]struct{}{} 252 binds := []string{} 253 for _, volume := range container.VolumeMounts { 254 volumes[volume.MountPath] = struct{}{} 255 basePath := "/exports/" + volume.Name + ":" + volume.MountPath 256 if volume.ReadOnly { 257 basePath += ":ro" 258 } 259 binds = append(binds, basePath) 260 } 261 262 exposedPorts := map[docker.Port]struct{}{} 263 portBindings := map[docker.Port][]docker.PortBinding{} 264 for _, port := range container.Ports { 265 interiorPort := port.ContainerPort 266 exteriorPort := port.HostPort 267 // Some of this port stuff is under-documented voodoo. 268 // See http://stackoverflow.com/questions/20428302/binding-a-port-to-a-host-interface-using-the-rest-api 269 dockerPort := docker.Port(strconv.Itoa(interiorPort) + "/tcp") 270 exposedPorts[dockerPort] = struct{}{} 271 portBindings[dockerPort] = []docker.PortBinding{ 272 docker.PortBinding{ 273 HostPort: strconv.Itoa(exteriorPort), 274 }, 275 } 276 } 277 var cmdList []string 278 if len(container.Command) > 0 { 279 cmdList = strings.Split(container.Command, " ") 280 } 281 opts := docker.CreateContainerOptions{ 282 Name: name, 283 Config: &docker.Config{ 284 Image: container.Image, 285 ExposedPorts: exposedPorts, 286 Env: envVariables, 287 Volumes: volumes, 288 WorkingDir: container.WorkingDir, 289 Cmd: cmdList, 290 }, 291 } 292 dockerContainer, err := sl.DockerClient.CreateContainer(opts) 293 if err != nil { 294 return "", err 295 } 296 return name, sl.DockerClient.StartContainer(dockerContainer.ID, &docker.HostConfig{ 297 PortBindings: portBindings, 298 Binds: binds, 299 }) 300 } 301 302 func (sl *Kubelet) KillContainer(name string) error { 303 id, err := sl.GetContainerID(name) 304 if err != nil { 305 return err 306 } 307 err = sl.DockerClient.StopContainer(id, 10) 308 manifestId, containerName := dockerNameToManifestAndContainer(name) 309 sl.LogEvent(&api.Event{ 310 Event: "STOP", 311 Manifest: &api.ContainerManifest{ 312 Id: manifestId, 313 }, 314 Container: &api.Container{ 315 Name: containerName, 316 }, 317 }) 318 319 return err 320 } 321 322 // Watch a file for changes to the set of tasks that should run on this Kubelet 323 // This function loops forever and is intended to be run as a goroutine 324 func (sl *Kubelet) WatchFile(file string, changeChannel chan<- api.ContainerManifest) { 325 var lastData []byte 326 for { 327 time.Sleep(sl.FileCheckFrequency) 328 var manifest api.ContainerManifest 329 data, err := ioutil.ReadFile(file) 330 if err != nil { 331 log.Printf("Couldn't read file: %s : %v", file, err) 332 continue 333 } 334 if err = sl.ExtractYAMLData(data, &manifest); err != nil { 335 continue 336 } 337 if !bytes.Equal(lastData, data) { 338 lastData = data 339 // Ok, we have a valid configuration, send to channel for 340 // rejiggering. 341 changeChannel <- manifest 342 continue 343 } 344 } 345 } 346 347 // Watch an HTTP endpoint for changes to the set of tasks that should run on this Kubelet 348 // This function runs forever and is intended to be run as a goroutine 349 func (sl *Kubelet) WatchHTTP(url string, changeChannel chan<- api.ContainerManifest) { 350 var lastData []byte 351 client := &http.Client{} 352 for { 353 time.Sleep(sl.HTTPCheckFrequency) 354 var config api.ContainerManifest 355 data, err := sl.SyncHTTP(client, url, &config) 356 log.Printf("Containers: %#v", config) 357 if err != nil { 358 log.Printf("Error syncing HTTP: %#v", err) 359 continue 360 } 361 if !bytes.Equal(lastData, data) { 362 lastData = data 363 changeChannel <- config 364 continue 365 } 366 } 367 } 368 369 // SyncHTTP reads from url a yaml manifest and populates config. Returns the 370 // raw bytes, if something was read. Returns an error if something goes wrong. 371 // 'client' is used to execute the request, to allow caching of clients. 372 func (sl *Kubelet) SyncHTTP(client *http.Client, url string, config *api.ContainerManifest) ([]byte, error) { 373 request, err := http.NewRequest("GET", url, nil) 374 if err != nil { 375 return nil, err 376 } 377 response, err := client.Do(request) 378 if err != nil { 379 return nil, err 380 } 381 defer response.Body.Close() 382 body, err := ioutil.ReadAll(response.Body) 383 if err != nil { 384 return nil, err 385 } 386 if err = sl.ExtractYAMLData(body, &config); err != nil { 387 return body, err 388 } 389 return body, nil 390 } 391 392 // Take an etcd Response object, and turn it into a structured list of containers 393 // Return a list of containers, or an error if one occurs. 394 func (sl *Kubelet) ResponseToManifests(response *etcd.Response) ([]api.ContainerManifest, error) { 395 if response.Node == nil || len(response.Node.Value) == 0 { 396 return nil, fmt.Errorf("no nodes field: %#v", response) 397 } 398 var manifests []api.ContainerManifest 399 err := sl.ExtractYAMLData([]byte(response.Node.Value), &manifests) 400 return manifests, err 401 } 402 403 func (sl *Kubelet) getKubeletStateFromEtcd(key string, changeChannel chan<- []api.ContainerManifest) error { 404 response, err := sl.Client.Get(key+"/kubelet", true, false) 405 if err != nil { 406 log.Printf("Error on get on %s: %#v", key, err) 407 switch err.(type) { 408 case *etcd.EtcdError: 409 etcdError := err.(*etcd.EtcdError) 410 if etcdError.ErrorCode == 100 { 411 return nil 412 } 413 } 414 return err 415 } 416 manifests, err := sl.ResponseToManifests(response) 417 if err != nil { 418 log.Printf("Error parsing response (%#v): %s", response, err) 419 return err 420 } 421 log.Printf("Got initial state from etcd: %+v", manifests) 422 changeChannel <- manifests 423 return nil 424 } 425 426 // Sync with etcd, and set up an etcd watch for new configurations 427 // The channel to send new configurations across 428 // This function loops forever and is intended to be run in a go routine. 429 func (sl *Kubelet) SyncAndSetupEtcdWatch(changeChannel chan<- []api.ContainerManifest) { 430 hostname, err := exec.Command("hostname", "-f").Output() 431 if err != nil { 432 log.Printf("Couldn't determine hostname : %v", err) 433 return 434 } 435 key := "/registry/hosts/" + strings.TrimSpace(string(hostname)) 436 // First fetch the initial configuration (watch only gives changes...) 437 for { 438 err = sl.getKubeletStateFromEtcd(key, changeChannel) 439 if err == nil { 440 // We got a successful response, etcd is up, set up the watch. 441 break 442 } 443 time.Sleep(30 * time.Second) 444 } 445 446 done := make(chan bool) 447 go util.Forever(func() { sl.TimeoutWatch(done) }, 0) 448 for { 449 // The etcd client will close the watch channel when it exits. So we need 450 // to create and service a new one every time. 451 watchChannel := make(chan *etcd.Response) 452 // We don't push this through Forever because if it dies, we just do it again in 30 secs. 453 // anyway. 454 go sl.WatchEtcd(watchChannel, changeChannel) 455 456 sl.getKubeletStateFromEtcd(key, changeChannel) 457 log.Printf("Setting up a watch for configuration changes in etcd for %s", key) 458 sl.Client.Watch(key, 0, true, watchChannel, done) 459 } 460 } 461 462 // Timeout the watch after 30 seconds 463 func (sl *Kubelet) TimeoutWatch(done chan bool) { 464 t := time.Tick(30 * time.Second) 465 for _ = range t { 466 done <- true 467 } 468 } 469 470 // Extract data from YAML file into a list of containers. 471 func (sl *Kubelet) ExtractYAMLData(buf []byte, output interface{}) error { 472 err := yaml.Unmarshal(buf, output) 473 if err != nil { 474 log.Printf("Couldn't unmarshal configuration: %v", err) 475 return err 476 } 477 return nil 478 } 479 480 // Watch etcd for changes, receives config objects from the etcd client watch. 481 // This function loops forever and is intended to be run as a goroutine. 482 func (sl *Kubelet) WatchEtcd(watchChannel <-chan *etcd.Response, changeChannel chan<- []api.ContainerManifest) { 483 defer util.HandleCrash() 484 for { 485 watchResponse := <-watchChannel 486 log.Printf("Got change: %#v", watchResponse) 487 488 // This means the channel has been closed. 489 if watchResponse == nil { 490 return 491 } 492 493 if watchResponse.Node == nil || len(watchResponse.Node.Value) == 0 { 494 log.Printf("No nodes field: %#v", watchResponse) 495 if watchResponse.Node != nil { 496 log.Printf("Node: %#v", watchResponse.Node) 497 } 498 } 499 log.Printf("Got data: %v", watchResponse.Node.Value) 500 var manifests []api.ContainerManifest 501 if err := sl.ExtractYAMLData([]byte(watchResponse.Node.Value), &manifests); err != nil { 502 continue 503 } 504 // Ok, we have a valid configuration, send to channel for 505 // rejiggering. 506 changeChannel <- manifests 507 } 508 } 509 510 // Sync the configured list of containers (desired state) with the host current state 511 func (sl *Kubelet) SyncManifests(config []api.ContainerManifest) error { 512 log.Printf("Desired:%#v", config) 513 var err error 514 desired := map[string]bool{} 515 for _, manifest := range config { 516 for _, element := range manifest.Containers { 517 var exists bool 518 exists, actualName, err := sl.ContainerExists(&manifest, &element) 519 if err != nil { 520 log.Printf("Error detecting container: %#v skipping.", err) 521 continue 522 } 523 if !exists { 524 log.Printf("%#v doesn't exist, creating", element) 525 actualName, err = sl.RunContainer(&manifest, &element) 526 // For some reason, list gives back names that start with '/' 527 actualName = "/" + actualName 528 529 if err != nil { 530 // TODO(bburns) : Perhaps blacklist a container after N failures? 531 log.Printf("Error creating container: %#v", err) 532 desired[actualName] = true 533 continue 534 } 535 } else { 536 log.Printf("%#v exists as %v", element.Name, actualName) 537 } 538 desired[actualName] = true 539 } 540 } 541 existingContainers, _ := sl.ListContainers() 542 log.Printf("Existing:\n%#v Desired: %#v", existingContainers, desired) 543 for _, container := range existingContainers { 544 if !desired[container] { 545 log.Printf("Killing: %s", container) 546 err = sl.KillContainer(container) 547 if err != nil { 548 log.Printf("Error killing container: %#v", err) 549 } 550 } 551 } 552 return err 553 } 554 555 // runSyncLoop is the main loop for processing changes. It watches for changes from 556 // four channels (file, etcd, server, and http) and creates a union of the two. For 557 // any new change seen, will run a sync against desired state and running state. If 558 // no changes are seen to the configuration, will synchronize the last known desired 559 // state every sync_frequency seconds. 560 // Never returns. 561 func (sl *Kubelet) RunSyncLoop(etcdChannel <-chan []api.ContainerManifest, fileChannel, serverChannel, httpChannel <-chan api.ContainerManifest, handler SyncHandler) { 562 var lastFile, lastEtcd, lastHttp, lastServer []api.ContainerManifest 563 for { 564 select { 565 case manifest := <-fileChannel: 566 log.Printf("Got new manifest from file... %v", manifest) 567 lastFile = []api.ContainerManifest{manifest} 568 case manifests := <-etcdChannel: 569 log.Printf("Got new configuration from etcd... %v", manifests) 570 lastEtcd = manifests 571 case manifest := <-httpChannel: 572 log.Printf("Got new manifest from external http... %v", manifest) 573 lastHttp = []api.ContainerManifest{manifest} 574 case manifest := <-serverChannel: 575 log.Printf("Got new manifest from our server... %v", manifest) 576 lastServer = []api.ContainerManifest{manifest} 577 case <-time.After(sl.SyncFrequency): 578 } 579 580 manifests := append([]api.ContainerManifest{}, lastFile...) 581 manifests = append(manifests, lastEtcd...) 582 manifests = append(manifests, lastHttp...) 583 manifests = append(manifests, lastServer...) 584 err := handler.SyncManifests(manifests) 585 if err != nil { 586 log.Printf("Couldn't sync containers : %#v", err) 587 } 588 } 589 } 590 591 func (sl *Kubelet) GetContainerInfo(name string) (string, error) { 592 info, err := sl.DockerClient.InspectContainer(name) 593 if err != nil { 594 return "{}", err 595 } 596 data, err := json.Marshal(info) 597 return string(data), err 598 }