github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/client/lib/cgutil/cpuset_manager_v1_test.go (about)

     1  //go:build linux
     2  
     3  package cgutil
     4  
     5  import (
     6  	"io/ioutil"
     7  	"path/filepath"
     8  	"testing"
     9  
    10  	"github.com/hashicorp/nomad/client/testutil"
    11  	"github.com/hashicorp/nomad/helper/testlog"
    12  	"github.com/hashicorp/nomad/helper/uuid"
    13  	"github.com/hashicorp/nomad/lib/cpuset"
    14  	"github.com/hashicorp/nomad/nomad/mock"
    15  	"github.com/opencontainers/runc/libcontainer/cgroups"
    16  	"github.com/stretchr/testify/require"
    17  )
    18  
    19  func tmpCpusetManagerV1(t *testing.T) (*cpusetManagerV1, func()) {
    20  	mount, err := FindCgroupMountpointDir()
    21  	if err != nil || mount == "" {
    22  		t.Skipf("Failed to find cgroup mount: %v %v", mount, err)
    23  	}
    24  
    25  	parent := "/gotest-" + uuid.Short()
    26  	require.NoError(t, cpusetEnsureParentV1(parent))
    27  
    28  	parentPath, err := GetCgroupPathHelperV1("cpuset", parent)
    29  	require.NoError(t, err)
    30  
    31  	manager := NewCpusetManagerV1(parent, nil, testlog.HCLogger(t)).(*cpusetManagerV1)
    32  	return manager, func() { require.NoError(t, cgroups.RemovePaths(map[string]string{"cpuset": parentPath})) }
    33  }
    34  
    35  func TestCpusetManager_V1_Init(t *testing.T) {
    36  	testutil.CgroupsCompatibleV1(t)
    37  
    38  	manager, cleanup := tmpCpusetManagerV1(t)
    39  	defer cleanup()
    40  	manager.Init()
    41  
    42  	require.DirExists(t, filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName))
    43  	require.FileExists(t, filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName, "cpuset.cpus"))
    44  	sharedCpusRaw, err := ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName, "cpuset.cpus"))
    45  	require.NoError(t, err)
    46  	sharedCpus, err := cpuset.Parse(string(sharedCpusRaw))
    47  	require.NoError(t, err)
    48  	require.Exactly(t, manager.parentCpuset.ToSlice(), sharedCpus.ToSlice())
    49  	require.DirExists(t, filepath.Join(manager.cgroupParentPath, ReservedCpusetCgroupName))
    50  }
    51  
    52  func TestCpusetManager_V1_AddAlloc_single(t *testing.T) {
    53  	testutil.CgroupsCompatibleV1(t)
    54  
    55  	manager, cleanup := tmpCpusetManagerV1(t)
    56  	defer cleanup()
    57  	manager.Init()
    58  
    59  	alloc := mock.Alloc()
    60  	// reserve just one core (the 0th core, which probably exists)
    61  	alloc.AllocatedResources.Tasks["web"].Cpu.ReservedCores = cpuset.New(0).ToSlice()
    62  	manager.AddAlloc(alloc)
    63  
    64  	// force reconcile
    65  	manager.reconcileCpusets()
    66  
    67  	// check that the 0th core is no longer available in the shared group
    68  	// actual contents of shared group depends on machine core count
    69  	require.DirExists(t, filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName))
    70  	require.FileExists(t, filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName, "cpuset.cpus"))
    71  	sharedCpusRaw, err := ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName, "cpuset.cpus"))
    72  	require.NoError(t, err)
    73  	sharedCpus, err := cpuset.Parse(string(sharedCpusRaw))
    74  	require.NoError(t, err)
    75  	require.NotEmpty(t, sharedCpus.ToSlice())
    76  	require.NotContains(t, sharedCpus.ToSlice(), uint16(0))
    77  
    78  	// check that the 0th core is allocated to reserved cgroup
    79  	require.DirExists(t, filepath.Join(manager.cgroupParentPath, ReservedCpusetCgroupName))
    80  	reservedCpusRaw, err := ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, ReservedCpusetCgroupName, "cpuset.cpus"))
    81  	require.NoError(t, err)
    82  	reservedCpus, err := cpuset.Parse(string(reservedCpusRaw))
    83  	require.NoError(t, err)
    84  	require.Exactly(t, alloc.AllocatedResources.Tasks["web"].Cpu.ReservedCores, reservedCpus.ToSlice())
    85  
    86  	// check that task cgroup exists and cpuset matches expected reserved cores
    87  	allocInfo, ok := manager.cgroupInfo[alloc.ID]
    88  	require.True(t, ok)
    89  	require.Len(t, allocInfo, 1)
    90  	taskInfo, ok := allocInfo["web"]
    91  	require.True(t, ok)
    92  
    93  	require.DirExists(t, taskInfo.CgroupPath)
    94  	taskCpusRaw, err := ioutil.ReadFile(filepath.Join(taskInfo.CgroupPath, "cpuset.cpus"))
    95  	require.NoError(t, err)
    96  	taskCpus, err := cpuset.Parse(string(taskCpusRaw))
    97  	require.NoError(t, err)
    98  	require.Exactly(t, alloc.AllocatedResources.Tasks["web"].Cpu.ReservedCores, taskCpus.ToSlice())
    99  }
   100  
   101  func TestCpusetManager_V1_RemoveAlloc(t *testing.T) {
   102  	testutil.CgroupsCompatibleV1(t)
   103  
   104  	// This case tests adding 2 allocations, reconciling then removing 1 alloc.
   105  	// It requires the system to have at least 3 cpu cores (one for each alloc),
   106  	// BUT plus another one because writing an empty cpuset causes the cgroup to
   107  	// inherit the parent.
   108  	testutil.MinimumCores(t, 3)
   109  
   110  	manager, cleanup := tmpCpusetManagerV1(t)
   111  	defer cleanup()
   112  	manager.Init()
   113  
   114  	alloc1 := mock.Alloc()
   115  	alloc1Cpuset := cpuset.New(manager.parentCpuset.ToSlice()[0])
   116  	alloc1.AllocatedResources.Tasks["web"].Cpu.ReservedCores = alloc1Cpuset.ToSlice()
   117  	manager.AddAlloc(alloc1)
   118  
   119  	alloc2 := mock.Alloc()
   120  	alloc2Cpuset := cpuset.New(manager.parentCpuset.ToSlice()[1])
   121  	alloc2.AllocatedResources.Tasks["web"].Cpu.ReservedCores = alloc2Cpuset.ToSlice()
   122  	manager.AddAlloc(alloc2)
   123  
   124  	//force reconcile
   125  	manager.reconcileCpusets()
   126  
   127  	// shared cpuset should not include any expected cores
   128  	sharedCpusRaw, err := ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName, "cpuset.cpus"))
   129  	require.NoError(t, err)
   130  	sharedCpus, err := cpuset.Parse(string(sharedCpusRaw))
   131  	require.NoError(t, err)
   132  	require.False(t, sharedCpus.ContainsAny(alloc1Cpuset.Union(alloc2Cpuset)))
   133  
   134  	// reserved cpuset should equal the expected cpus
   135  	reservedCpusRaw, err := ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, ReservedCpusetCgroupName, "cpuset.cpus"))
   136  	require.NoError(t, err)
   137  	reservedCpus, err := cpuset.Parse(string(reservedCpusRaw))
   138  	require.NoError(t, err)
   139  	require.True(t, reservedCpus.Equal(alloc1Cpuset.Union(alloc2Cpuset)))
   140  
   141  	// remove first allocation
   142  	alloc1TaskPath := manager.cgroupInfo[alloc1.ID]["web"].CgroupPath
   143  	manager.RemoveAlloc(alloc1.ID)
   144  	manager.reconcileCpusets()
   145  
   146  	// alloc1's task reserved cgroup should be removed
   147  	require.NoDirExists(t, alloc1TaskPath)
   148  
   149  	// shared cpuset should now include alloc1's cores
   150  	sharedCpusRaw, err = ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, SharedCpusetCgroupName, "cpuset.cpus"))
   151  	require.NoError(t, err)
   152  	sharedCpus, err = cpuset.Parse(string(sharedCpusRaw))
   153  	require.NoError(t, err)
   154  	require.False(t, sharedCpus.ContainsAny(alloc2Cpuset))
   155  	require.True(t, sharedCpus.IsSupersetOf(alloc1Cpuset))
   156  
   157  	// reserved cpuset should only include alloc2's cores
   158  	reservedCpusRaw, err = ioutil.ReadFile(filepath.Join(manager.cgroupParentPath, ReservedCpusetCgroupName, "cpuset.cpus"))
   159  	require.NoError(t, err)
   160  	reservedCpus, err = cpuset.Parse(string(reservedCpusRaw))
   161  	require.NoError(t, err)
   162  	require.True(t, reservedCpus.Equal(alloc2Cpuset))
   163  
   164  }