github.com/deemoprobe/k8s-first-commit@v0.0.0-20230430165612-a541f1982be3/pkg/kubelet/kubelet.go (about)

     1  /*
     2  Copyright 2014 Google Inc. All rights reserved.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package kubelet is ...
    18  package kubelet
    19  
    20  import (
    21  	"bytes"
    22  	"encoding/json"
    23  	"fmt"
    24  	"io/ioutil"
    25  	"log"
    26  	"math/rand"
    27  	"net/http"
    28  	"os/exec"
    29  	"strconv"
    30  	"strings"
    31  	"sync"
    32  	"time"
    33  
    34  	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
    35  	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry"
    36  	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
    37  	"github.com/coreos/go-etcd/etcd"
    38  	"github.com/fsouza/go-dockerclient"
    39  	"gopkg.in/v1/yaml"
    40  )
    41  
    42  // State, sub object of the Docker JSON data
    43  type State struct {
    44  	Running bool
    45  }
    46  
    47  // The structured representation of the JSON object returned by Docker inspect
    48  type DockerContainerData struct {
    49  	state State
    50  }
    51  
    52  // Interface for testability
    53  type DockerInterface interface {
    54  	ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error)
    55  	InspectContainer(id string) (*docker.Container, error)
    56  	CreateContainer(docker.CreateContainerOptions) (*docker.Container, error)
    57  	StartContainer(id string, hostConfig *docker.HostConfig) error
    58  	StopContainer(id string, timeout uint) error
    59  }
    60  
    61  // The main kubelet implementation
    62  type Kubelet struct {
    63  	Client             registry.EtcdClient
    64  	DockerClient       DockerInterface
    65  	FileCheckFrequency time.Duration
    66  	SyncFrequency      time.Duration
    67  	HTTPCheckFrequency time.Duration
    68  	pullLock           sync.Mutex
    69  }
    70  
    71  // Starts background goroutines. If file, manifest_url, or address are empty,
    72  // they are not watched. Never returns.
    73  func (sl *Kubelet) RunKubelet(file, manifest_url, etcd_servers, address string, port uint) {
    74  	fileChannel := make(chan api.ContainerManifest)
    75  	etcdChannel := make(chan []api.ContainerManifest)
    76  	httpChannel := make(chan api.ContainerManifest)
    77  	serverChannel := make(chan api.ContainerManifest)
    78  
    79  	go util.Forever(func() { sl.WatchFile(file, fileChannel) }, 20*time.Second)
    80  	if manifest_url != "" {
    81  		go util.Forever(func() { sl.WatchHTTP(manifest_url, httpChannel) }, 20*time.Second)
    82  	}
    83  	if etcd_servers != "" {
    84  		servers := []string{etcd_servers}
    85  		log.Printf("Creating etcd client pointing to %v", servers)
    86  		sl.Client = etcd.NewClient(servers)
    87  		go util.Forever(func() { sl.SyncAndSetupEtcdWatch(etcdChannel) }, 20*time.Second)
    88  	}
    89  	if address != "" {
    90  		log.Printf("Starting to listen on %s:%d", address, port)
    91  		handler := KubeletServer{
    92  			Kubelet:       sl,
    93  			UpdateChannel: serverChannel,
    94  		}
    95  		s := &http.Server{
    96  			// TODO: This is broken if address is an ipv6 address.
    97  			Addr:           fmt.Sprintf("%s:%d", address, port),
    98  			Handler:        &handler,
    99  			ReadTimeout:    10 * time.Second,
   100  			WriteTimeout:   10 * time.Second,
   101  			MaxHeaderBytes: 1 << 20,
   102  		}
   103  		go util.Forever(func() { s.ListenAndServe() }, 0)
   104  	}
   105  	sl.RunSyncLoop(etcdChannel, fileChannel, serverChannel, httpChannel, sl)
   106  }
   107  
   108  // Interface implemented by Kubelet, for testability
   109  type SyncHandler interface {
   110  	SyncManifests([]api.ContainerManifest) error
   111  }
   112  
   113  // Log an event to the etcd backend.
   114  func (sl *Kubelet) LogEvent(event *api.Event) error {
   115  	if sl.Client == nil {
   116  		return fmt.Errorf("no etcd client connection.")
   117  	}
   118  	event.Timestamp = time.Now().Unix()
   119  	data, err := json.Marshal(event)
   120  	if err != nil {
   121  		return err
   122  	}
   123  
   124  	var response *etcd.Response
   125  	response, err = sl.Client.AddChild(fmt.Sprintf("/events/%s", event.Container.Name), string(data), 60*60*48 /* 2 days */)
   126  	// TODO(bburns) : examine response here.
   127  	if err != nil {
   128  		log.Printf("Error writing event: %s\n", err)
   129  		if response != nil {
   130  			log.Printf("Response was: %#v\n", *response)
   131  		}
   132  	}
   133  	return err
   134  }
   135  
   136  // Does this container exist on this host? Returns true if so, and the name under which the container is running.
   137  // Returns an error if one occurs.
   138  func (sl *Kubelet) ContainerExists(manifest *api.ContainerManifest, container *api.Container) (exists bool, foundName string, err error) {
   139  	containers, err := sl.ListContainers()
   140  	if err != nil {
   141  		return false, "", err
   142  	}
   143  	for _, name := range containers {
   144  		manifestId, containerName := dockerNameToManifestAndContainer(name)
   145  		if manifestId == manifest.Id && containerName == container.Name {
   146  			// TODO(bburns) : This leads to an extra list.  Convert this to use the returned ID and a straight call
   147  			// to inspect
   148  			data, err := sl.GetContainerByName(name)
   149  			return data != nil, name, err
   150  		}
   151  	}
   152  	return false, "", nil
   153  }
   154  
   155  func (sl *Kubelet) GetContainerID(name string) (string, error) {
   156  	containerList, err := sl.DockerClient.ListContainers(docker.ListContainersOptions{})
   157  	if err != nil {
   158  		return "", err
   159  	}
   160  	for _, value := range containerList {
   161  		if strings.Contains(value.Names[0], name) {
   162  			return value.ID, nil
   163  		}
   164  	}
   165  	return "", fmt.Errorf("couldn't find name: %s", name)
   166  }
   167  
   168  // Get a container by name.
   169  // returns the container data from Docker, or an error if one exists.
   170  func (sl *Kubelet) GetContainerByName(name string) (*docker.Container, error) {
   171  	id, err := sl.GetContainerID(name)
   172  	if err != nil {
   173  		return nil, err
   174  	}
   175  	return sl.DockerClient.InspectContainer(id)
   176  }
   177  
   178  func (sl *Kubelet) ListContainers() ([]string, error) {
   179  	result := []string{}
   180  	containerList, err := sl.DockerClient.ListContainers(docker.ListContainersOptions{})
   181  	if err != nil {
   182  		return result, err
   183  	}
   184  	for _, value := range containerList {
   185  		result = append(result, value.Names[0])
   186  	}
   187  	return result, err
   188  }
   189  
   190  func (sl *Kubelet) pullImage(image string) error {
   191  	sl.pullLock.Lock()
   192  	defer sl.pullLock.Unlock()
   193  	cmd := exec.Command("docker", "pull", image)
   194  	err := cmd.Start()
   195  	if err != nil {
   196  		return err
   197  	}
   198  	return cmd.Wait()
   199  }
   200  
   201  // Converts "-" to "_-_" and "_" to "___" so that we can use "--" to meaningfully separate parts of a docker name.
   202  func escapeDash(in string) (out string) {
   203  	out = strings.Replace(in, "_", "___", -1)
   204  	out = strings.Replace(out, "-", "_-_", -1)
   205  	return
   206  }
   207  
   208  // Reverses the transformation of escapeDash.
   209  func unescapeDash(in string) (out string) {
   210  	out = strings.Replace(in, "_-_", "-", -1)
   211  	out = strings.Replace(out, "___", "_", -1)
   212  	return
   213  }
   214  
   215  // Creates a name which can be reversed to identify both manifest id and container name.
   216  func manifestAndContainerToDockerName(manifest *api.ContainerManifest, container *api.Container) string {
   217  	// Note, manifest.Id could be blank.
   218  	return fmt.Sprintf("%s--%s--%x", escapeDash(container.Name), escapeDash(manifest.Id), rand.Uint32())
   219  }
   220  
   221  // Upacks a container name, returning the manifest id and container name we would have used to
   222  // construct the docker name. If the docker name isn't one we created, we may return empty strings.
   223  func dockerNameToManifestAndContainer(name string) (manifestId, containerName string) {
   224  	// For some reason docker appears to be appending '/' to names.
   225  	// If its there, strip it.
   226  	if name[0] == '/' {
   227  		name = name[1:]
   228  	}
   229  	parts := strings.Split(name, "--")
   230  	if len(parts) > 0 {
   231  		containerName = unescapeDash(parts[0])
   232  	}
   233  	if len(parts) > 1 {
   234  		manifestId = unescapeDash(parts[1])
   235  	}
   236  	return
   237  }
   238  
   239  func (sl *Kubelet) RunContainer(manifest *api.ContainerManifest, container *api.Container) (name string, err error) {
   240  	err = sl.pullImage(container.Image)
   241  	if err != nil {
   242  		return "", err
   243  	}
   244  
   245  	name = manifestAndContainerToDockerName(manifest, container)
   246  	envVariables := []string{}
   247  	for _, value := range container.Env {
   248  		envVariables = append(envVariables, fmt.Sprintf("%s=%s", value.Name, value.Value))
   249  	}
   250  
   251  	volumes := map[string]struct{}{}
   252  	binds := []string{}
   253  	for _, volume := range container.VolumeMounts {
   254  		volumes[volume.MountPath] = struct{}{}
   255  		basePath := "/exports/" + volume.Name + ":" + volume.MountPath
   256  		if volume.ReadOnly {
   257  			basePath += ":ro"
   258  		}
   259  		binds = append(binds, basePath)
   260  	}
   261  
   262  	exposedPorts := map[docker.Port]struct{}{}
   263  	portBindings := map[docker.Port][]docker.PortBinding{}
   264  	for _, port := range container.Ports {
   265  		interiorPort := port.ContainerPort
   266  		exteriorPort := port.HostPort
   267  		// Some of this port stuff is under-documented voodoo.
   268  		// See http://stackoverflow.com/questions/20428302/binding-a-port-to-a-host-interface-using-the-rest-api
   269  		dockerPort := docker.Port(strconv.Itoa(interiorPort) + "/tcp")
   270  		exposedPorts[dockerPort] = struct{}{}
   271  		portBindings[dockerPort] = []docker.PortBinding{
   272  			docker.PortBinding{
   273  				HostPort: strconv.Itoa(exteriorPort),
   274  			},
   275  		}
   276  	}
   277  	var cmdList []string
   278  	if len(container.Command) > 0 {
   279  		cmdList = strings.Split(container.Command, " ")
   280  	}
   281  	opts := docker.CreateContainerOptions{
   282  		Name: name,
   283  		Config: &docker.Config{
   284  			Image:        container.Image,
   285  			ExposedPorts: exposedPorts,
   286  			Env:          envVariables,
   287  			Volumes:      volumes,
   288  			WorkingDir:   container.WorkingDir,
   289  			Cmd:          cmdList,
   290  		},
   291  	}
   292  	dockerContainer, err := sl.DockerClient.CreateContainer(opts)
   293  	if err != nil {
   294  		return "", err
   295  	}
   296  	return name, sl.DockerClient.StartContainer(dockerContainer.ID, &docker.HostConfig{
   297  		PortBindings: portBindings,
   298  		Binds:        binds,
   299  	})
   300  }
   301  
   302  func (sl *Kubelet) KillContainer(name string) error {
   303  	id, err := sl.GetContainerID(name)
   304  	if err != nil {
   305  		return err
   306  	}
   307  	err = sl.DockerClient.StopContainer(id, 10)
   308  	manifestId, containerName := dockerNameToManifestAndContainer(name)
   309  	sl.LogEvent(&api.Event{
   310  		Event: "STOP",
   311  		Manifest: &api.ContainerManifest{
   312  			Id: manifestId,
   313  		},
   314  		Container: &api.Container{
   315  			Name: containerName,
   316  		},
   317  	})
   318  
   319  	return err
   320  }
   321  
   322  // Watch a file for changes to the set of tasks that should run on this Kubelet
   323  // This function loops forever and is intended to be run as a goroutine
   324  func (sl *Kubelet) WatchFile(file string, changeChannel chan<- api.ContainerManifest) {
   325  	var lastData []byte
   326  	for {
   327  		time.Sleep(sl.FileCheckFrequency)
   328  		var manifest api.ContainerManifest
   329  		data, err := ioutil.ReadFile(file)
   330  		if err != nil {
   331  			log.Printf("Couldn't read file: %s : %v", file, err)
   332  			continue
   333  		}
   334  		if err = sl.ExtractYAMLData(data, &manifest); err != nil {
   335  			continue
   336  		}
   337  		if !bytes.Equal(lastData, data) {
   338  			lastData = data
   339  			// Ok, we have a valid configuration, send to channel for
   340  			// rejiggering.
   341  			changeChannel <- manifest
   342  			continue
   343  		}
   344  	}
   345  }
   346  
   347  // Watch an HTTP endpoint for changes to the set of tasks that should run on this Kubelet
   348  // This function runs forever and is intended to be run as a goroutine
   349  func (sl *Kubelet) WatchHTTP(url string, changeChannel chan<- api.ContainerManifest) {
   350  	var lastData []byte
   351  	client := &http.Client{}
   352  	for {
   353  		time.Sleep(sl.HTTPCheckFrequency)
   354  		var config api.ContainerManifest
   355  		data, err := sl.SyncHTTP(client, url, &config)
   356  		log.Printf("Containers: %#v", config)
   357  		if err != nil {
   358  			log.Printf("Error syncing HTTP: %#v", err)
   359  			continue
   360  		}
   361  		if !bytes.Equal(lastData, data) {
   362  			lastData = data
   363  			changeChannel <- config
   364  			continue
   365  		}
   366  	}
   367  }
   368  
   369  // SyncHTTP reads from url a yaml manifest and populates config. Returns the
   370  // raw bytes, if something was read. Returns an error if something goes wrong.
   371  // 'client' is used to execute the request, to allow caching of clients.
   372  func (sl *Kubelet) SyncHTTP(client *http.Client, url string, config *api.ContainerManifest) ([]byte, error) {
   373  	request, err := http.NewRequest("GET", url, nil)
   374  	if err != nil {
   375  		return nil, err
   376  	}
   377  	response, err := client.Do(request)
   378  	if err != nil {
   379  		return nil, err
   380  	}
   381  	defer response.Body.Close()
   382  	body, err := ioutil.ReadAll(response.Body)
   383  	if err != nil {
   384  		return nil, err
   385  	}
   386  	if err = sl.ExtractYAMLData(body, &config); err != nil {
   387  		return body, err
   388  	}
   389  	return body, nil
   390  }
   391  
   392  // Take an etcd Response object, and turn it into a structured list of containers
   393  // Return a list of containers, or an error if one occurs.
   394  func (sl *Kubelet) ResponseToManifests(response *etcd.Response) ([]api.ContainerManifest, error) {
   395  	if response.Node == nil || len(response.Node.Value) == 0 {
   396  		return nil, fmt.Errorf("no nodes field: %#v", response)
   397  	}
   398  	var manifests []api.ContainerManifest
   399  	err := sl.ExtractYAMLData([]byte(response.Node.Value), &manifests)
   400  	return manifests, err
   401  }
   402  
   403  func (sl *Kubelet) getKubeletStateFromEtcd(key string, changeChannel chan<- []api.ContainerManifest) error {
   404  	response, err := sl.Client.Get(key+"/kubelet", true, false)
   405  	if err != nil {
   406  		log.Printf("Error on get on %s: %#v", key, err)
   407  		switch err.(type) {
   408  		case *etcd.EtcdError:
   409  			etcdError := err.(*etcd.EtcdError)
   410  			if etcdError.ErrorCode == 100 {
   411  				return nil
   412  			}
   413  		}
   414  		return err
   415  	}
   416  	manifests, err := sl.ResponseToManifests(response)
   417  	if err != nil {
   418  		log.Printf("Error parsing response (%#v): %s", response, err)
   419  		return err
   420  	}
   421  	log.Printf("Got initial state from etcd: %+v", manifests)
   422  	changeChannel <- manifests
   423  	return nil
   424  }
   425  
   426  // Sync with etcd, and set up an etcd watch for new configurations
   427  // The channel to send new configurations across
   428  // This function loops forever and is intended to be run in a go routine.
   429  func (sl *Kubelet) SyncAndSetupEtcdWatch(changeChannel chan<- []api.ContainerManifest) {
   430  	hostname, err := exec.Command("hostname", "-f").Output()
   431  	if err != nil {
   432  		log.Printf("Couldn't determine hostname : %v", err)
   433  		return
   434  	}
   435  	key := "/registry/hosts/" + strings.TrimSpace(string(hostname))
   436  	// First fetch the initial configuration (watch only gives changes...)
   437  	for {
   438  		err = sl.getKubeletStateFromEtcd(key, changeChannel)
   439  		if err == nil {
   440  			// We got a successful response, etcd is up, set up the watch.
   441  			break
   442  		}
   443  		time.Sleep(30 * time.Second)
   444  	}
   445  
   446  	done := make(chan bool)
   447  	go util.Forever(func() { sl.TimeoutWatch(done) }, 0)
   448  	for {
   449  		// The etcd client will close the watch channel when it exits.  So we need
   450  		// to create and service a new one every time.
   451  		watchChannel := make(chan *etcd.Response)
   452  		// We don't push this through Forever because if it dies, we just do it again in 30 secs.
   453  		// anyway.
   454  		go sl.WatchEtcd(watchChannel, changeChannel)
   455  
   456  		sl.getKubeletStateFromEtcd(key, changeChannel)
   457  		log.Printf("Setting up a watch for configuration changes in etcd for %s", key)
   458  		sl.Client.Watch(key, 0, true, watchChannel, done)
   459  	}
   460  }
   461  
   462  // Timeout the watch after 30 seconds
   463  func (sl *Kubelet) TimeoutWatch(done chan bool) {
   464  	t := time.Tick(30 * time.Second)
   465  	for _ = range t {
   466  		done <- true
   467  	}
   468  }
   469  
   470  // Extract data from YAML file into a list of containers.
   471  func (sl *Kubelet) ExtractYAMLData(buf []byte, output interface{}) error {
   472  	err := yaml.Unmarshal(buf, output)
   473  	if err != nil {
   474  		log.Printf("Couldn't unmarshal configuration: %v", err)
   475  		return err
   476  	}
   477  	return nil
   478  }
   479  
   480  // Watch etcd for changes, receives config objects from the etcd client watch.
   481  // This function loops forever and is intended to be run as a goroutine.
   482  func (sl *Kubelet) WatchEtcd(watchChannel <-chan *etcd.Response, changeChannel chan<- []api.ContainerManifest) {
   483  	defer util.HandleCrash()
   484  	for {
   485  		watchResponse := <-watchChannel
   486  		log.Printf("Got change: %#v", watchResponse)
   487  
   488  		// This means the channel has been closed.
   489  		if watchResponse == nil {
   490  			return
   491  		}
   492  
   493  		if watchResponse.Node == nil || len(watchResponse.Node.Value) == 0 {
   494  			log.Printf("No nodes field: %#v", watchResponse)
   495  			if watchResponse.Node != nil {
   496  				log.Printf("Node: %#v", watchResponse.Node)
   497  			}
   498  		}
   499  		log.Printf("Got data: %v", watchResponse.Node.Value)
   500  		var manifests []api.ContainerManifest
   501  		if err := sl.ExtractYAMLData([]byte(watchResponse.Node.Value), &manifests); err != nil {
   502  			continue
   503  		}
   504  		// Ok, we have a valid configuration, send to channel for
   505  		// rejiggering.
   506  		changeChannel <- manifests
   507  	}
   508  }
   509  
   510  // Sync the configured list of containers (desired state) with the host current state
   511  func (sl *Kubelet) SyncManifests(config []api.ContainerManifest) error {
   512  	log.Printf("Desired:%#v", config)
   513  	var err error
   514  	desired := map[string]bool{}
   515  	for _, manifest := range config {
   516  		for _, element := range manifest.Containers {
   517  			var exists bool
   518  			exists, actualName, err := sl.ContainerExists(&manifest, &element)
   519  			if err != nil {
   520  				log.Printf("Error detecting container: %#v skipping.", err)
   521  				continue
   522  			}
   523  			if !exists {
   524  				log.Printf("%#v doesn't exist, creating", element)
   525  				actualName, err = sl.RunContainer(&manifest, &element)
   526  				// For some reason, list gives back names that start with '/'
   527  				actualName = "/" + actualName
   528  
   529  				if err != nil {
   530  					// TODO(bburns) : Perhaps blacklist a container after N failures?
   531  					log.Printf("Error creating container: %#v", err)
   532  					desired[actualName] = true
   533  					continue
   534  				}
   535  			} else {
   536  				log.Printf("%#v exists as %v", element.Name, actualName)
   537  			}
   538  			desired[actualName] = true
   539  		}
   540  	}
   541  	existingContainers, _ := sl.ListContainers()
   542  	log.Printf("Existing:\n%#v Desired: %#v", existingContainers, desired)
   543  	for _, container := range existingContainers {
   544  		if !desired[container] {
   545  			log.Printf("Killing: %s", container)
   546  			err = sl.KillContainer(container)
   547  			if err != nil {
   548  				log.Printf("Error killing container: %#v", err)
   549  			}
   550  		}
   551  	}
   552  	return err
   553  }
   554  
   555  // runSyncLoop is the main loop for processing changes. It watches for changes from
   556  // four channels (file, etcd, server, and http) and creates a union of the two. For
   557  // any new change seen, will run a sync against desired state and running state. If
   558  // no changes are seen to the configuration, will synchronize the last known desired
   559  // state every sync_frequency seconds.
   560  // Never returns.
   561  func (sl *Kubelet) RunSyncLoop(etcdChannel <-chan []api.ContainerManifest, fileChannel, serverChannel, httpChannel <-chan api.ContainerManifest, handler SyncHandler) {
   562  	var lastFile, lastEtcd, lastHttp, lastServer []api.ContainerManifest
   563  	for {
   564  		select {
   565  		case manifest := <-fileChannel:
   566  			log.Printf("Got new manifest from file... %v", manifest)
   567  			lastFile = []api.ContainerManifest{manifest}
   568  		case manifests := <-etcdChannel:
   569  			log.Printf("Got new configuration from etcd... %v", manifests)
   570  			lastEtcd = manifests
   571  		case manifest := <-httpChannel:
   572  			log.Printf("Got new manifest from external http... %v", manifest)
   573  			lastHttp = []api.ContainerManifest{manifest}
   574  		case manifest := <-serverChannel:
   575  			log.Printf("Got new manifest from our server... %v", manifest)
   576  			lastServer = []api.ContainerManifest{manifest}
   577  		case <-time.After(sl.SyncFrequency):
   578  		}
   579  
   580  		manifests := append([]api.ContainerManifest{}, lastFile...)
   581  		manifests = append(manifests, lastEtcd...)
   582  		manifests = append(manifests, lastHttp...)
   583  		manifests = append(manifests, lastServer...)
   584  		err := handler.SyncManifests(manifests)
   585  		if err != nil {
   586  			log.Printf("Couldn't sync containers : %#v", err)
   587  		}
   588  	}
   589  }
   590  
   591  func (sl *Kubelet) GetContainerInfo(name string) (string, error) {
   592  	info, err := sl.DockerClient.InspectContainer(name)
   593  	if err != nil {
   594  		return "{}", err
   595  	}
   596  	data, err := json.Marshal(info)
   597  	return string(data), err
   598  }