github.com/google/cadvisor@v0.49.1/integration/tests/api/docker_test.go (about)

     1  // Copyright 2014 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package api
    16  
    17  import (
    18  	"fmt"
    19  	"os"
    20  	"strconv"
    21  	"testing"
    22  	"time"
    23  
    24  	info "github.com/google/cadvisor/info/v1"
    25  	v2 "github.com/google/cadvisor/info/v2"
    26  	"github.com/google/cadvisor/integration/framework"
    27  
    28  	"github.com/opencontainers/runc/libcontainer/cgroups"
    29  	"github.com/stretchr/testify/assert"
    30  	"github.com/stretchr/testify/require"
    31  )
    32  
    33  // Sanity check the container by:
    34  // - Checking that the specified alias is a valid one for this container.
    35  // - Verifying that stats are not empty.
    36  func sanityCheck(alias string, containerInfo info.ContainerInfo, t *testing.T) {
    37  	assert.Contains(t, containerInfo.Aliases, alias, "Alias %q should be in list of aliases %v", alias, containerInfo.Aliases)
    38  	assert.NotEmpty(t, containerInfo.Stats, "Expected container to have stats")
    39  }
    40  
    41  // Sanity check the container by:
    42  // - Checking that the specified alias is a valid one for this container.
    43  // - Verifying that stats are not empty.
    44  func sanityCheckV2(alias string, info v2.ContainerInfo, t *testing.T) {
    45  	assert.Contains(t, info.Spec.Aliases, alias, "Alias %q should be in list of aliases %v", alias, info.Spec.Aliases)
    46  	assert.NotEmpty(t, info.Stats, "Expected container to have stats")
    47  }
    48  
    49  // Waits up to 5s for a container with the specified alias to appear.
    50  func waitForContainer(alias string, fm framework.Framework) {
    51  	err := framework.RetryForDuration(func() error {
    52  		ret, err := fm.Cadvisor().Client().DockerContainer(alias, &info.ContainerInfoRequest{
    53  			NumStats: 1,
    54  		})
    55  		if err != nil {
    56  			return err
    57  		}
    58  		if len(ret.Stats) != 1 {
    59  			return fmt.Errorf("no stats returned for container %q", alias)
    60  		}
    61  
    62  		return nil
    63  	}, 5*time.Second)
    64  	require.NoError(fm.T(), err, "Timed out waiting for container %q to be available in cAdvisor: %v", alias, err)
    65  }
    66  
    67  // A Docker container in /docker/<ID>
    68  func TestDockerContainerById(t *testing.T) {
    69  	fm := framework.New(t)
    70  	defer fm.Cleanup()
    71  
    72  	containerID := fm.Docker().RunPause()
    73  
    74  	// Wait for the container to show up.
    75  	waitForContainer(containerID, fm)
    76  
    77  	request := &info.ContainerInfoRequest{
    78  		NumStats: 1,
    79  	}
    80  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerID, request)
    81  	require.NoError(t, err)
    82  
    83  	sanityCheck(containerID, containerInfo, t)
    84  }
    85  
    86  // A Docker container in /docker/<name>
    87  func TestDockerContainerByName(t *testing.T) {
    88  	fm := framework.New(t)
    89  	defer fm.Cleanup()
    90  
    91  	containerName := fmt.Sprintf("test-docker-container-by-name-%d", os.Getpid())
    92  	fm.Docker().Run(framework.DockerRunArgs{
    93  		Image: "registry.k8s.io/pause",
    94  		Args:  []string{"--name", containerName},
    95  	})
    96  
    97  	// Wait for the container to show up.
    98  	waitForContainer(containerName, fm)
    99  
   100  	request := &info.ContainerInfoRequest{
   101  		NumStats: 1,
   102  	}
   103  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerName, request)
   104  	require.NoError(t, err)
   105  
   106  	sanityCheck(containerName, containerInfo, t)
   107  }
   108  
   109  // Find the first container with the specified alias in containers.
   110  func findContainer(alias string, containers []info.ContainerInfo, t *testing.T) info.ContainerInfo {
   111  	for _, cont := range containers {
   112  		for _, a := range cont.Aliases {
   113  			if alias == a {
   114  				return cont
   115  			}
   116  		}
   117  	}
   118  	t.Fatalf("Failed to find container %q in %+v", alias, containers)
   119  	return info.ContainerInfo{}
   120  }
   121  
   122  // All Docker containers through /docker
   123  func TestGetAllDockerContainers(t *testing.T) {
   124  	fm := framework.New(t)
   125  	defer fm.Cleanup()
   126  
   127  	// Wait for the containers to show up.
   128  	containerID1 := fm.Docker().RunPause()
   129  	containerID2 := fm.Docker().RunPause()
   130  	waitForContainer(containerID1, fm)
   131  	waitForContainer(containerID2, fm)
   132  
   133  	request := &info.ContainerInfoRequest{
   134  		NumStats: 1,
   135  	}
   136  	containersInfo, err := fm.Cadvisor().Client().AllDockerContainers(request)
   137  	require.NoError(t, err)
   138  
   139  	if len(containersInfo) < 2 {
   140  		t.Fatalf("At least 2 Docker containers should exist, received %d: %+v", len(containersInfo), containersInfo)
   141  	}
   142  	sanityCheck(containerID1, findContainer(containerID1, containersInfo, t), t)
   143  	sanityCheck(containerID2, findContainer(containerID2, containersInfo, t), t)
   144  }
   145  
   146  // Check expected properties of a Docker container.
   147  func TestBasicDockerContainer(t *testing.T) {
   148  	fm := framework.New(t)
   149  	defer fm.Cleanup()
   150  
   151  	containerName := fmt.Sprintf("test-basic-docker-container-%d", os.Getpid())
   152  	containerID := fm.Docker().Run(framework.DockerRunArgs{
   153  		Image: "registry.k8s.io/pause",
   154  		Args: []string{
   155  			"--name", containerName,
   156  		},
   157  	})
   158  
   159  	// Wait for the container to show up.
   160  	waitForContainer(containerID, fm)
   161  
   162  	request := &info.ContainerInfoRequest{
   163  		NumStats: 1,
   164  	}
   165  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerID, request)
   166  	require.NoError(t, err)
   167  
   168  	// Check that the contianer is known by both its name and ID.
   169  	sanityCheck(containerID, containerInfo, t)
   170  	sanityCheck(containerName, containerInfo, t)
   171  
   172  	assert.Empty(t, containerInfo.Subcontainers, "Should not have subcontainers")
   173  	assert.Len(t, containerInfo.Stats, 1, "Should have exactly one stat")
   174  }
   175  
   176  // TODO(vmarmol): Handle if CPU or memory is not isolated on this system.
   177  // Check the ContainerSpec.
   178  func TestDockerContainerSpec(t *testing.T) {
   179  	fm := framework.New(t)
   180  	defer fm.Cleanup()
   181  
   182  	var (
   183  		cpuShares   = uint64(2048)
   184  		cpuMask     = "0"
   185  		memoryLimit = uint64(1 << 30) // 1GB
   186  		image       = "registry.k8s.io/pause"
   187  		env         = map[string]string{"test_var": "FOO"}
   188  		labels      = map[string]string{"bar": "baz"}
   189  	)
   190  
   191  	containerID := fm.Docker().Run(framework.DockerRunArgs{
   192  		Image: image,
   193  		Args: []string{
   194  			"--cpu-shares", strconv.FormatUint(cpuShares, 10),
   195  			"--cpuset-cpus", cpuMask,
   196  			"--memory", strconv.FormatUint(memoryLimit, 10),
   197  			"--env", "TEST_VAR=FOO",
   198  			"--label", "bar=baz",
   199  		},
   200  	})
   201  
   202  	// Wait for the container to show up.
   203  	waitForContainer(containerID, fm)
   204  
   205  	request := &info.ContainerInfoRequest{
   206  		NumStats: 1,
   207  	}
   208  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerID, request)
   209  	require.NoError(t, err)
   210  	sanityCheck(containerID, containerInfo, t)
   211  
   212  	assert := assert.New(t)
   213  
   214  	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
   215  	if cgroups.IsCgroup2UnifiedMode() {
   216  		// cpu shares are rounded slighly on cgroupv2 due to conversion between cgroupv1 (cpu.shares) and cgroupv2 (cpu.weight)
   217  		// When container is created via docker, runc will convert cpu shares to cpu.weight https://github.com/opencontainers/runc/blob/d11f4d756e85ece5cdba8bb69f8bd4db3cdcbeab/libcontainer/cgroups/utils.go#L423-L428
   218  		// And cAdvisor will convert cpu.weight back to cpu shares in https://github.com/google/cadvisor/blob/24e7a9883d12f944fd4403861707f4bafcaf4f3d/container/common/helpers.go#L249-L260
   219  		// Worked example:
   220  		// cpuShares = 2048 (input to docker --cpu-shares)
   221  		// cpuWeight = int((1 + ((cpuShares-2)*9999)/262142))=79 (conversion done by runc)
   222  		// cpuWeight back to cpuShares = int(2 + ((cpuWeight-1)*262142)/9999)= 2046
   223  		var cgroupV2Shares uint64 = 2046
   224  		assert.Equal(cgroupV2Shares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cgroupV2Shares, containerInfo.Spec.Cpu.Limit)
   225  	} else {
   226  		assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
   227  	}
   228  
   229  	assert.Equal(cpuMask, containerInfo.Spec.Cpu.Mask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
   230  	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
   231  	assert.Equal(memoryLimit, containerInfo.Spec.Memory.Limit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)
   232  	assert.True(containerInfo.Spec.HasNetwork, "Network should be isolated")
   233  	assert.True(containerInfo.Spec.HasDiskIo, "Blkio should be isolated")
   234  
   235  	assert.Equal(image, containerInfo.Spec.Image, "Spec should include container image")
   236  	assert.Equal(env, containerInfo.Spec.Envs, "Spec should include environment variables")
   237  	assert.Equal(labels, containerInfo.Spec.Labels, "Spec should include labels")
   238  }
   239  
   240  // Check the CPU ContainerStats.
   241  func TestDockerContainerCpuStats(t *testing.T) {
   242  	fm := framework.New(t)
   243  	defer fm.Cleanup()
   244  
   245  	// Wait for the container to show up.
   246  	containerID := fm.Docker().RunBusybox("ping", "www.google.com")
   247  	waitForContainer(containerID, fm)
   248  
   249  	request := &info.ContainerInfoRequest{
   250  		NumStats: 1,
   251  	}
   252  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerID, request)
   253  	if err != nil {
   254  		t.Fatal(err)
   255  	}
   256  	sanityCheck(containerID, containerInfo, t)
   257  
   258  	// Checks for CpuStats.
   259  	checkCPUStats(t, containerInfo.Stats[0].Cpu)
   260  }
   261  
   262  // Check the memory ContainerStats.
   263  func TestDockerContainerMemoryStats(t *testing.T) {
   264  	fm := framework.New(t)
   265  	defer fm.Cleanup()
   266  
   267  	// Wait for the container to show up.
   268  	containerID := fm.Docker().RunBusybox("ping", "www.google.com")
   269  	waitForContainer(containerID, fm)
   270  
   271  	request := &info.ContainerInfoRequest{
   272  		NumStats: 1,
   273  	}
   274  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerID, request)
   275  	require.NoError(t, err)
   276  	sanityCheck(containerID, containerInfo, t)
   277  
   278  	// Checks for MemoryStats.
   279  	checkMemoryStats(t, containerInfo.Stats[0].Memory)
   280  }
   281  
   282  // Check the network ContainerStats.
   283  func TestDockerContainerNetworkStats(t *testing.T) {
   284  	fm := framework.New(t)
   285  	defer fm.Cleanup()
   286  
   287  	// Wait for the container to show up.
   288  	containerID := fm.Docker().RunBusybox("watch", "-n1", "wget", "http://www.google.com/")
   289  	waitForContainer(containerID, fm)
   290  
   291  	// Wait for at least one additional housekeeping interval
   292  	time.Sleep(20 * time.Second)
   293  	request := &info.ContainerInfoRequest{
   294  		NumStats: 1,
   295  	}
   296  	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerID, request)
   297  	require.NoError(t, err)
   298  	sanityCheck(containerID, containerInfo, t)
   299  
   300  	stat := containerInfo.Stats[0]
   301  	ifaceStats := stat.Network.InterfaceStats
   302  	// macOS we have more than one interface, since traffic is
   303  	// only on eth0 we need to pick that one
   304  	if len(stat.Network.Interfaces) > 0 {
   305  		for _, iface := range stat.Network.Interfaces {
   306  			if iface.Name == "eth0" {
   307  				ifaceStats = iface
   308  			}
   309  		}
   310  	}
   311  
   312  	// Checks for NetworkStats.
   313  	assert := assert.New(t)
   314  	assert.NotEqual(0, ifaceStats.TxBytes, "Network tx bytes should not be zero")
   315  	assert.NotEqual(0, ifaceStats.TxPackets, "Network tx packets should not be zero")
   316  	assert.NotEqual(0, ifaceStats.RxBytes, "Network rx bytes should not be zero")
   317  	assert.NotEqual(0, ifaceStats.RxPackets, "Network rx packets should not be zero")
   318  	assert.NotEqual(ifaceStats.RxBytes, ifaceStats.TxBytes, fmt.Sprintf("Network tx (%d) and rx (%d) bytes should not be equal", ifaceStats.TxBytes, ifaceStats.RxBytes))
   319  	assert.NotEqual(ifaceStats.RxPackets, ifaceStats.TxPackets, fmt.Sprintf("Network tx (%d) and rx (%d) packets should not be equal", ifaceStats.TxPackets, ifaceStats.RxPackets))
   320  }
   321  
   322  func TestDockerFilesystemStats(t *testing.T) {
   323  	fm := framework.New(t)
   324  	defer fm.Cleanup()
   325  
   326  	storageDriver := fm.Docker().StorageDriver()
   327  	if storageDriver == framework.DeviceMapper {
   328  		// Filesystem stats not supported with devicemapper, yet
   329  		return
   330  	}
   331  
   332  	const (
   333  		ddUsage       = uint64(1 << 3) // 1 KB
   334  		sleepDuration = 10 * time.Second
   335  	)
   336  	// Wait for the container to show up.
   337  	// FIXME: Tests should be bundled and run on the remote host instead of being run over ssh.
   338  	// Escaping bash over ssh is ugly.
   339  	// Once github issue 1130 is fixed, this logic can be removed.
   340  	dockerCmd := fmt.Sprintf("dd if=/dev/zero of=/file count=2 bs=%d & ping google.com", ddUsage)
   341  	if fm.Hostname().Host != "localhost" {
   342  		dockerCmd = fmt.Sprintf("'%s'", dockerCmd)
   343  	}
   344  	containerID := fm.Docker().RunBusybox("/bin/sh", "-c", dockerCmd)
   345  	waitForContainer(containerID, fm)
   346  	request := &v2.RequestOptions{
   347  		IdType: v2.TypeDocker,
   348  		Count:  1,
   349  	}
   350  	needsBaseUsageCheck := false
   351  	switch storageDriver {
   352  	case framework.Aufs, framework.Overlay, framework.Overlay2, framework.DeviceMapper:
   353  		needsBaseUsageCheck = true
   354  	}
   355  	pass := false
   356  	// We need to wait for the `dd` operation to complete.
   357  	for i := 0; i < 10; i++ {
   358  		containerInfo, err := fm.Cadvisor().ClientV2().Stats(containerID, request)
   359  		if err != nil {
   360  			t.Logf("%v stats unavailable - %v", time.Now().String(), err)
   361  			t.Logf("retrying after %s...", sleepDuration.String())
   362  			time.Sleep(sleepDuration)
   363  
   364  			continue
   365  		}
   366  		require.Equal(t, len(containerInfo), 1)
   367  		var info v2.ContainerInfo
   368  		// There is only one container in containerInfo. Since it is a map with unknown key,
   369  		// use the value blindly.
   370  		for _, cInfo := range containerInfo {
   371  			info = cInfo
   372  		}
   373  		sanityCheckV2(containerID, info, t)
   374  
   375  		require.NotNil(t, info.Stats[0], "got info: %+v", info)
   376  		require.NotNil(t, info.Stats[0].Filesystem, "got info: %+v", info)
   377  		require.NotNil(t, info.Stats[0].Filesystem.TotalUsageBytes, "got info: %+v", info.Stats[0].Filesystem)
   378  		if *info.Stats[0].Filesystem.TotalUsageBytes >= ddUsage {
   379  			if !needsBaseUsageCheck {
   380  				pass = true
   381  				break
   382  			}
   383  			require.NotNil(t, info.Stats[0].Filesystem.BaseUsageBytes)
   384  			if *info.Stats[0].Filesystem.BaseUsageBytes >= ddUsage {
   385  				pass = true
   386  				break
   387  			}
   388  		}
   389  		t.Logf("expected total usage %d bytes to be greater than %d bytes", *info.Stats[0].Filesystem.TotalUsageBytes, ddUsage)
   390  		if needsBaseUsageCheck {
   391  			t.Logf("expected base %d bytes to be greater than %d bytes", *info.Stats[0].Filesystem.BaseUsageBytes, ddUsage)
   392  		}
   393  		t.Logf("retrying after %s...", sleepDuration.String())
   394  		time.Sleep(sleepDuration)
   395  	}
   396  
   397  	if !pass {
   398  		t.Fail()
   399  	}
   400  }