github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/client/pluginmanager/csimanager/volume_test.go (about) 1 package csimanager 2 3 import ( 4 "context" 5 "errors" 6 "io/ioutil" 7 "os" 8 "runtime" 9 "testing" 10 11 "github.com/hashicorp/nomad/helper/testlog" 12 "github.com/hashicorp/nomad/nomad/mock" 13 "github.com/hashicorp/nomad/nomad/structs" 14 "github.com/hashicorp/nomad/plugins/csi" 15 csifake "github.com/hashicorp/nomad/plugins/csi/fake" 16 "github.com/stretchr/testify/require" 17 ) 18 19 func tmpDir(t testing.TB) string { 20 t.Helper() 21 dir, err := ioutil.TempDir("", "nomad") 22 require.NoError(t, err) 23 return dir 24 } 25 26 func TestVolumeManager_ensureStagingDir(t *testing.T) { 27 t.Parallel() 28 29 cases := []struct { 30 Name string 31 Volume *structs.CSIVolume 32 UsageOptions *UsageOptions 33 CreateDirAheadOfTime bool 34 MountDirAheadOfTime bool 35 36 ExpectedErr error 37 ExpectedMountState bool 38 }{ 39 { 40 Name: "Creates a directory when one does not exist", 41 Volume: &structs.CSIVolume{ID: "foo"}, 42 UsageOptions: &UsageOptions{}, 43 }, 44 { 45 Name: "Does not fail because of a pre-existing directory", 46 Volume: &structs.CSIVolume{ID: "foo"}, 47 UsageOptions: &UsageOptions{}, 48 CreateDirAheadOfTime: true, 49 }, 50 { 51 Name: "Returns negative mount info", 52 UsageOptions: &UsageOptions{}, 53 Volume: &structs.CSIVolume{ID: "foo"}, 54 }, 55 { 56 Name: "Returns positive mount info", 57 Volume: &structs.CSIVolume{ID: "foo"}, 58 UsageOptions: &UsageOptions{}, 59 CreateDirAheadOfTime: true, 60 MountDirAheadOfTime: true, 61 ExpectedMountState: true, 62 }, 63 } 64 65 for _, tc := range cases { 66 t.Run(tc.Name, func(t *testing.T) { 67 // Step 1: Validate that the test case makes sense 68 if !tc.CreateDirAheadOfTime && tc.MountDirAheadOfTime { 69 require.Fail(t, "Cannot Mount without creating a dir") 70 } 71 72 if tc.MountDirAheadOfTime { 73 // We can enable these tests by either mounting a fake device on linux 74 // e.g shipping a small ext4 image file and using that as a loopback 75 // device, but there's no convenient way to implement this. 76 t.Skip("TODO: Skipped because we don't detect bind mounts") 77 } 78 79 // Step 2: Test Setup 80 tmpPath := tmpDir(t) 81 defer os.RemoveAll(tmpPath) 82 83 csiFake := &csifake.Client{} 84 eventer := func(e *structs.NodeEvent) {} 85 manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true) 86 expectedStagingPath := manager.stagingDirForVolume(tmpPath, tc.Volume.ID, tc.UsageOptions) 87 88 if tc.CreateDirAheadOfTime { 89 err := os.MkdirAll(expectedStagingPath, 0700) 90 require.NoError(t, err) 91 } 92 93 // Step 3: Now we can do some testing 94 95 path, detectedMount, testErr := manager.ensureStagingDir(tc.Volume, tc.UsageOptions) 96 if tc.ExpectedErr != nil { 97 require.EqualError(t, testErr, tc.ExpectedErr.Error()) 98 return // We don't perform extra validation if an error was detected. 99 } 100 101 require.NoError(t, testErr) 102 require.Equal(t, tc.ExpectedMountState, detectedMount) 103 104 // If the ensureStagingDir call had to create a directory itself, then here 105 // we validate that the directory exists and its permissions 106 if !tc.CreateDirAheadOfTime { 107 file, err := os.Lstat(path) 108 require.NoError(t, err) 109 require.True(t, file.IsDir()) 110 111 // TODO: Figure out a windows equivalent of this test 112 if runtime.GOOS != "windows" { 113 require.Equal(t, os.FileMode(0700), file.Mode().Perm()) 114 } 115 } 116 }) 117 } 118 } 119 120 func TestVolumeManager_stageVolume(t *testing.T) { 121 t.Parallel() 122 cases := []struct { 123 Name string 124 Volume *structs.CSIVolume 125 UsageOptions *UsageOptions 126 PluginErr error 127 ExpectedErr error 128 }{ 129 { 130 Name: "Returns an error when an invalid AttachmentMode is provided", 131 Volume: &structs.CSIVolume{ 132 ID: "foo", 133 AttachmentMode: "nonsense", 134 }, 135 UsageOptions: &UsageOptions{}, 136 ExpectedErr: errors.New("Unknown volume attachment mode: nonsense"), 137 }, 138 { 139 Name: "Returns an error when an invalid AccessMode is provided", 140 Volume: &structs.CSIVolume{ 141 ID: "foo", 142 AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, 143 AccessMode: "nonsense", 144 }, 145 UsageOptions: &UsageOptions{}, 146 ExpectedErr: errors.New("Unknown volume access mode: nonsense"), 147 }, 148 { 149 Name: "Returns an error when the plugin returns an error", 150 Volume: &structs.CSIVolume{ 151 ID: "foo", 152 AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, 153 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 154 }, 155 UsageOptions: &UsageOptions{}, 156 PluginErr: errors.New("Some Unknown Error"), 157 ExpectedErr: errors.New("Some Unknown Error"), 158 }, 159 { 160 Name: "Happy Path", 161 Volume: &structs.CSIVolume{ 162 ID: "foo", 163 AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, 164 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 165 }, 166 UsageOptions: &UsageOptions{}, 167 PluginErr: nil, 168 ExpectedErr: nil, 169 }, 170 } 171 172 for _, tc := range cases { 173 t.Run(tc.Name, func(t *testing.T) { 174 tmpPath := tmpDir(t) 175 defer os.RemoveAll(tmpPath) 176 177 csiFake := &csifake.Client{} 178 csiFake.NextNodeStageVolumeErr = tc.PluginErr 179 180 eventer := func(e *structs.NodeEvent) {} 181 manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true) 182 ctx := context.Background() 183 184 err := manager.stageVolume(ctx, tc.Volume, tc.UsageOptions, nil) 185 186 if tc.ExpectedErr != nil { 187 require.EqualError(t, err, tc.ExpectedErr.Error()) 188 } else { 189 require.NoError(t, err) 190 } 191 }) 192 } 193 } 194 195 func TestVolumeManager_unstageVolume(t *testing.T) { 196 t.Parallel() 197 cases := []struct { 198 Name string 199 Volume *structs.CSIVolume 200 UsageOptions *UsageOptions 201 PluginErr error 202 ExpectedErr error 203 ExpectedCSICallCount int64 204 }{ 205 { 206 Name: "Returns an error when the plugin returns an error", 207 Volume: &structs.CSIVolume{ 208 ID: "foo", 209 }, 210 UsageOptions: &UsageOptions{}, 211 PluginErr: errors.New("Some Unknown Error"), 212 ExpectedErr: errors.New("Some Unknown Error"), 213 ExpectedCSICallCount: 1, 214 }, 215 { 216 Name: "Happy Path", 217 Volume: &structs.CSIVolume{ 218 ID: "foo", 219 }, 220 UsageOptions: &UsageOptions{}, 221 PluginErr: nil, 222 ExpectedErr: nil, 223 ExpectedCSICallCount: 1, 224 }, 225 } 226 227 for _, tc := range cases { 228 t.Run(tc.Name, func(t *testing.T) { 229 tmpPath := tmpDir(t) 230 defer os.RemoveAll(tmpPath) 231 232 csiFake := &csifake.Client{} 233 csiFake.NextNodeUnstageVolumeErr = tc.PluginErr 234 235 eventer := func(e *structs.NodeEvent) {} 236 manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true) 237 ctx := context.Background() 238 239 err := manager.unstageVolume(ctx, 240 tc.Volume.ID, tc.Volume.RemoteID(), tc.UsageOptions) 241 242 if tc.ExpectedErr != nil { 243 require.EqualError(t, err, tc.ExpectedErr.Error()) 244 } else { 245 require.NoError(t, err) 246 } 247 248 require.Equal(t, tc.ExpectedCSICallCount, csiFake.NodeUnstageVolumeCallCount) 249 }) 250 } 251 } 252 253 func TestVolumeManager_publishVolume(t *testing.T) { 254 t.Parallel() 255 cases := []struct { 256 Name string 257 Allocation *structs.Allocation 258 Volume *structs.CSIVolume 259 UsageOptions *UsageOptions 260 PluginErr error 261 ExpectedErr error 262 ExpectedCSICallCount int64 263 ExpectedVolumeCapability *csi.VolumeCapability 264 }{ 265 { 266 Name: "Returns an error when the plugin returns an error", 267 Allocation: structs.MockAlloc(), 268 Volume: &structs.CSIVolume{ 269 ID: "foo", 270 AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, 271 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 272 }, 273 UsageOptions: &UsageOptions{}, 274 PluginErr: errors.New("Some Unknown Error"), 275 ExpectedErr: errors.New("Some Unknown Error"), 276 ExpectedCSICallCount: 1, 277 }, 278 { 279 Name: "Happy Path", 280 Allocation: structs.MockAlloc(), 281 Volume: &structs.CSIVolume{ 282 ID: "foo", 283 AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, 284 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 285 }, 286 UsageOptions: &UsageOptions{}, 287 PluginErr: nil, 288 ExpectedErr: nil, 289 ExpectedCSICallCount: 1, 290 }, 291 { 292 Name: "Mount options in the volume", 293 Allocation: structs.MockAlloc(), 294 Volume: &structs.CSIVolume{ 295 ID: "foo", 296 AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, 297 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 298 MountOptions: &structs.CSIMountOptions{ 299 MountFlags: []string{"ro"}, 300 }, 301 }, 302 UsageOptions: &UsageOptions{}, 303 PluginErr: nil, 304 ExpectedErr: nil, 305 ExpectedCSICallCount: 1, 306 ExpectedVolumeCapability: &csi.VolumeCapability{ 307 AccessType: csi.VolumeAccessTypeMount, 308 AccessMode: csi.VolumeAccessModeMultiNodeMultiWriter, 309 MountVolume: &structs.CSIMountOptions{ 310 MountFlags: []string{"ro"}, 311 }, 312 }, 313 }, 314 { 315 Name: "Mount options override in the request", 316 Allocation: structs.MockAlloc(), 317 Volume: &structs.CSIVolume{ 318 ID: "foo", 319 AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, 320 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 321 MountOptions: &structs.CSIMountOptions{ 322 MountFlags: []string{"ro"}, 323 }, 324 }, 325 UsageOptions: &UsageOptions{ 326 MountOptions: &structs.CSIMountOptions{ 327 MountFlags: []string{"rw"}, 328 }, 329 }, 330 PluginErr: nil, 331 ExpectedErr: nil, 332 ExpectedCSICallCount: 1, 333 ExpectedVolumeCapability: &csi.VolumeCapability{ 334 AccessType: csi.VolumeAccessTypeMount, 335 AccessMode: csi.VolumeAccessModeMultiNodeMultiWriter, 336 MountVolume: &structs.CSIMountOptions{ 337 MountFlags: []string{"rw"}, 338 }, 339 }, 340 }, 341 } 342 343 for _, tc := range cases { 344 t.Run(tc.Name, func(t *testing.T) { 345 tmpPath := tmpDir(t) 346 defer os.RemoveAll(tmpPath) 347 348 csiFake := &csifake.Client{} 349 csiFake.NextNodePublishVolumeErr = tc.PluginErr 350 351 eventer := func(e *structs.NodeEvent) {} 352 manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true) 353 ctx := context.Background() 354 355 _, err := manager.publishVolume(ctx, tc.Volume, tc.Allocation, tc.UsageOptions, nil) 356 357 if tc.ExpectedErr != nil { 358 require.EqualError(t, err, tc.ExpectedErr.Error()) 359 } else { 360 require.NoError(t, err) 361 } 362 363 require.Equal(t, tc.ExpectedCSICallCount, csiFake.NodePublishVolumeCallCount) 364 365 if tc.ExpectedVolumeCapability != nil { 366 require.Equal(t, tc.ExpectedVolumeCapability, csiFake.PrevVolumeCapability) 367 } 368 369 }) 370 } 371 } 372 373 func TestVolumeManager_unpublishVolume(t *testing.T) { 374 t.Parallel() 375 cases := []struct { 376 Name string 377 Allocation *structs.Allocation 378 Volume *structs.CSIVolume 379 UsageOptions *UsageOptions 380 PluginErr error 381 ExpectedErr error 382 ExpectedCSICallCount int64 383 }{ 384 { 385 Name: "Returns an error when the plugin returns an error", 386 Allocation: structs.MockAlloc(), 387 Volume: &structs.CSIVolume{ 388 ID: "foo", 389 }, 390 UsageOptions: &UsageOptions{}, 391 PluginErr: errors.New("Some Unknown Error"), 392 ExpectedErr: errors.New("Some Unknown Error"), 393 ExpectedCSICallCount: 1, 394 }, 395 { 396 Name: "Happy Path", 397 Allocation: structs.MockAlloc(), 398 Volume: &structs.CSIVolume{ 399 ID: "foo", 400 }, 401 UsageOptions: &UsageOptions{}, 402 PluginErr: nil, 403 ExpectedErr: nil, 404 ExpectedCSICallCount: 1, 405 }, 406 } 407 408 for _, tc := range cases { 409 t.Run(tc.Name, func(t *testing.T) { 410 tmpPath := tmpDir(t) 411 defer os.RemoveAll(tmpPath) 412 413 csiFake := &csifake.Client{} 414 csiFake.NextNodeUnpublishVolumeErr = tc.PluginErr 415 416 eventer := func(e *structs.NodeEvent) {} 417 manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true) 418 ctx := context.Background() 419 420 err := manager.unpublishVolume(ctx, 421 tc.Volume.ID, tc.Volume.RemoteID(), tc.Allocation.ID, tc.UsageOptions) 422 423 if tc.ExpectedErr != nil { 424 require.EqualError(t, err, tc.ExpectedErr.Error()) 425 } else { 426 require.NoError(t, err) 427 } 428 429 require.Equal(t, tc.ExpectedCSICallCount, csiFake.NodeUnpublishVolumeCallCount) 430 }) 431 } 432 } 433 434 func TestVolumeManager_MountVolumeEvents(t *testing.T) { 435 t.Parallel() 436 437 tmpPath := tmpDir(t) 438 defer os.RemoveAll(tmpPath) 439 440 csiFake := &csifake.Client{} 441 442 var events []*structs.NodeEvent 443 eventer := func(e *structs.NodeEvent) { 444 events = append(events, e) 445 } 446 447 manager := newVolumeManager(testlog.HCLogger(t), eventer, csiFake, tmpPath, tmpPath, true) 448 ctx := context.Background() 449 vol := &structs.CSIVolume{ 450 ID: "vol", 451 Namespace: "ns", 452 AccessMode: structs.CSIVolumeAccessModeMultiNodeMultiWriter, 453 } 454 alloc := mock.Alloc() 455 usage := &UsageOptions{} 456 pubCtx := map[string]string{} 457 458 _, err := manager.MountVolume(ctx, vol, alloc, usage, pubCtx) 459 require.Error(t, err, "Unknown volume attachment mode: ") 460 require.Equal(t, 1, len(events)) 461 e := events[0] 462 require.Equal(t, "Mount volume", e.Message) 463 require.Equal(t, "Storage", e.Subsystem) 464 require.Equal(t, "vol", e.Details["volume_id"]) 465 require.Equal(t, "false", e.Details["success"]) 466 require.Equal(t, "Unknown volume attachment mode: ", e.Details["error"]) 467 events = events[1:] 468 469 vol.AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem 470 _, err = manager.MountVolume(ctx, vol, alloc, usage, pubCtx) 471 require.NoError(t, err) 472 473 require.Equal(t, 1, len(events)) 474 e = events[0] 475 require.Equal(t, "Mount volume", e.Message) 476 require.Equal(t, "Storage", e.Subsystem) 477 require.Equal(t, "vol", e.Details["volume_id"]) 478 require.Equal(t, "true", e.Details["success"]) 479 events = events[1:] 480 481 err = manager.UnmountVolume(ctx, vol.ID, vol.RemoteID(), alloc.ID, usage) 482 require.NoError(t, err) 483 484 require.Equal(t, 1, len(events)) 485 e = events[0] 486 require.Equal(t, "Unmount volume", e.Message) 487 require.Equal(t, "Storage", e.Subsystem) 488 require.Equal(t, "vol", e.Details["volume_id"]) 489 require.Equal(t, "true", e.Details["success"]) 490 }