k8s.io/kubernetes@v1.29.3/test/e2e/storage/testsuites/volumemode.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package testsuites 18 19 import ( 20 "context" 21 "fmt" 22 "path/filepath" 23 "strings" 24 25 "github.com/onsi/ginkgo/v2" 26 "github.com/onsi/gomega" 27 28 v1 "k8s.io/api/core/v1" 29 storagev1 "k8s.io/api/storage/v1" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/fields" 32 "k8s.io/apimachinery/pkg/util/errors" 33 clientset "k8s.io/client-go/kubernetes" 34 volevents "k8s.io/kubernetes/pkg/controller/volume/events" 35 "k8s.io/kubernetes/pkg/kubelet/events" 36 "k8s.io/kubernetes/test/e2e/framework" 37 e2eevents "k8s.io/kubernetes/test/e2e/framework/events" 38 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 39 e2epv "k8s.io/kubernetes/test/e2e/framework/pv" 40 e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" 41 e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" 42 storageframework "k8s.io/kubernetes/test/e2e/storage/framework" 43 storageutils "k8s.io/kubernetes/test/e2e/storage/utils" 44 admissionapi "k8s.io/pod-security-admission/api" 45 ) 46 47 const ( 48 noProvisioner = "kubernetes.io/no-provisioner" 49 pvNamePrefix = "pv" 50 ) 51 52 type volumeModeTestSuite struct { 53 tsInfo storageframework.TestSuiteInfo 54 } 55 56 var _ storageframework.TestSuite = &volumeModeTestSuite{} 57 58 // InitCustomVolumeModeTestSuite returns volumeModeTestSuite that implements TestSuite interface 59 // using custom test patterns 60 func InitCustomVolumeModeTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite { 61 return &volumeModeTestSuite{ 62 tsInfo: storageframework.TestSuiteInfo{ 63 Name: "volumeMode", 64 TestPatterns: patterns, 65 SupportedSizeRange: e2evolume.SizeRange{ 66 Min: "1Mi", 67 }, 68 }, 69 } 70 } 71 72 // InitVolumeModeTestSuite returns volumeModeTestSuite that implements TestSuite interface 73 // using testsuite default patterns 74 func InitVolumeModeTestSuite() storageframework.TestSuite { 75 patterns := []storageframework.TestPattern{ 76 storageframework.FsVolModePreprovisionedPV, 77 storageframework.FsVolModeDynamicPV, 78 storageframework.BlockVolModePreprovisionedPV, 79 storageframework.BlockVolModeDynamicPV, 80 } 81 return InitCustomVolumeModeTestSuite(patterns) 82 } 83 84 func (t *volumeModeTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { 85 return t.tsInfo 86 } 87 88 func (t *volumeModeTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { 89 } 90 91 func (t *volumeModeTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { 92 type local struct { 93 config *storageframework.PerTestConfig 94 driverCleanup func() 95 96 cs clientset.Interface 97 ns *v1.Namespace 98 // VolumeResource contains pv, pvc, sc, etc., owns cleaning that up 99 storageframework.VolumeResource 100 101 migrationCheck *migrationOpCheck 102 } 103 var ( 104 dInfo = driver.GetDriverInfo() 105 l local 106 ) 107 108 // Beware that it also registers an AfterEach which renders f unusable. Any code using 109 // f must run inside an It or Context callback. 110 f := framework.NewFrameworkWithCustomTimeouts("volumemode", storageframework.GetDriverTimeouts(driver)) 111 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 112 113 init := func(ctx context.Context) { 114 l = local{} 115 l.ns = f.Namespace 116 l.cs = f.ClientSet 117 118 // Now do the more expensive test initialization. 119 l.config = driver.PrepareTest(ctx, f) 120 l.migrationCheck = newMigrationOpCheck(ctx, f.ClientSet, f.ClientConfig(), dInfo.InTreePluginName) 121 } 122 123 // manualInit initializes l.VolumeResource without creating the PV & PVC objects. 124 manualInit := func(ctx context.Context) { 125 init(ctx) 126 127 fsType := pattern.FsType 128 volBindMode := storagev1.VolumeBindingImmediate 129 130 var ( 131 scName string 132 pvSource *v1.PersistentVolumeSource 133 volumeNodeAffinity *v1.VolumeNodeAffinity 134 ) 135 136 l.VolumeResource = storageframework.VolumeResource{ 137 Config: l.config, 138 Pattern: pattern, 139 } 140 141 // Create volume for pre-provisioned volume tests 142 l.Volume = storageframework.CreateVolume(ctx, driver, l.config, pattern.VolType) 143 144 switch pattern.VolType { 145 case storageframework.PreprovisionedPV: 146 if pattern.VolMode == v1.PersistentVolumeBlock { 147 scName = fmt.Sprintf("%s-%s-sc-for-block", l.ns.Name, dInfo.Name) 148 } else if pattern.VolMode == v1.PersistentVolumeFilesystem { 149 scName = fmt.Sprintf("%s-%s-sc-for-file", l.ns.Name, dInfo.Name) 150 } 151 if pDriver, ok := driver.(storageframework.PreprovisionedPVTestDriver); ok { 152 pvSource, volumeNodeAffinity = pDriver.GetPersistentVolumeSource(false, fsType, l.Volume) 153 if pvSource == nil { 154 e2eskipper.Skipf("Driver %q does not define PersistentVolumeSource - skipping", dInfo.Name) 155 } 156 157 storageClass, pvConfig, pvcConfig := generateConfigsForPreprovisionedPVTest(scName, volBindMode, pattern.VolMode, *pvSource, volumeNodeAffinity) 158 l.Sc = storageClass 159 l.Pv = e2epv.MakePersistentVolume(pvConfig) 160 l.Pvc = e2epv.MakePersistentVolumeClaim(pvcConfig, l.ns.Name) 161 } 162 case storageframework.DynamicPV: 163 if dDriver, ok := driver.(storageframework.DynamicPVTestDriver); ok { 164 l.Sc = dDriver.GetDynamicProvisionStorageClass(ctx, l.config, fsType) 165 if l.Sc == nil { 166 e2eskipper.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name) 167 } 168 l.Sc.VolumeBindingMode = &volBindMode 169 testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange 170 driverVolumeSizeRange := dInfo.SupportedSizeRange 171 claimSize, err := storageutils.GetSizeRangesIntersection(testVolumeSizeRange, driverVolumeSizeRange) 172 framework.ExpectNoError(err, "determine intersection of test size range %+v and driver size range %+v", testVolumeSizeRange, driverVolumeSizeRange) 173 174 l.Pvc = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ 175 ClaimSize: claimSize, 176 StorageClassName: &(l.Sc.Name), 177 VolumeMode: &pattern.VolMode, 178 }, l.ns.Name) 179 } 180 default: 181 framework.Failf("Volume mode test doesn't support: %s", pattern.VolType) 182 } 183 } 184 185 cleanup := func(ctx context.Context) { 186 var errs []error 187 errs = append(errs, l.CleanupResource(ctx)) 188 errs = append(errs, storageutils.TryFunc(l.driverCleanup)) 189 l.driverCleanup = nil 190 framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource") 191 l.migrationCheck.validateMigrationVolumeOpCounts(ctx) 192 } 193 194 // We register different tests depending on the drive 195 isBlockSupported := dInfo.Capabilities[storageframework.CapBlock] 196 switch pattern.VolType { 197 case storageframework.PreprovisionedPV: 198 if pattern.VolMode == v1.PersistentVolumeBlock && !isBlockSupported { 199 f.It("should fail to create pod by failing to mount volume", f.WithSlow(), func(ctx context.Context) { 200 manualInit(ctx) 201 ginkgo.DeferCleanup(cleanup) 202 203 var err error 204 205 ginkgo.By("Creating sc") 206 l.Sc, err = l.cs.StorageV1().StorageClasses().Create(ctx, l.Sc, metav1.CreateOptions{}) 207 framework.ExpectNoError(err, "Failed to create sc") 208 209 ginkgo.By("Creating pv and pvc") 210 l.Pv, err = l.cs.CoreV1().PersistentVolumes().Create(ctx, l.Pv, metav1.CreateOptions{}) 211 framework.ExpectNoError(err, "Failed to create pv") 212 213 // Prebind pv 214 l.Pvc.Spec.VolumeName = l.Pv.Name 215 l.Pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(ctx, l.Pvc, metav1.CreateOptions{}) 216 framework.ExpectNoError(err, "Failed to create pvc") 217 218 framework.ExpectNoError(e2epv.WaitOnPVandPVC(ctx, l.cs, f.Timeouts, l.ns.Name, l.Pv, l.Pvc), "Failed to bind pv and pvc") 219 220 ginkgo.By("Creating pod") 221 podConfig := e2epod.Config{ 222 NS: l.ns.Name, 223 PVCs: []*v1.PersistentVolumeClaim{l.Pvc}, 224 SeLinuxLabel: e2epod.GetLinuxLabel(), 225 NodeSelection: l.config.ClientNodeSelection, 226 ImageID: e2epod.GetDefaultTestImageID(), 227 } 228 pod, err := e2epod.MakeSecPod(&podConfig) 229 framework.ExpectNoError(err, "Failed to create pod") 230 231 pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(ctx, pod, metav1.CreateOptions{}) 232 framework.ExpectNoError(err, "Failed to create pod") 233 defer func() { 234 framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, l.cs, pod), "Failed to delete pod") 235 }() 236 237 eventSelector := fields.Set{ 238 "involvedObject.kind": "Pod", 239 "involvedObject.name": pod.Name, 240 "involvedObject.namespace": l.ns.Name, 241 "reason": events.FailedMountVolume, 242 }.AsSelector().String() 243 msg := "Unable to attach or mount volumes" 244 245 err = e2eevents.WaitTimeoutForEvent(ctx, l.cs, l.ns.Name, eventSelector, msg, f.Timeouts.PodStart) 246 // Events are unreliable, don't depend on the event. It's used only to speed up the test. 247 if err != nil { 248 framework.Logf("Warning: did not get event about FailedMountVolume") 249 } 250 251 // Check the pod is still not running 252 p, err := l.cs.CoreV1().Pods(l.ns.Name).Get(ctx, pod.Name, metav1.GetOptions{}) 253 framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)") 254 gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodPending), "Pod phase isn't pending") 255 }) 256 } 257 258 case storageframework.DynamicPV: 259 if pattern.VolMode == v1.PersistentVolumeBlock && !isBlockSupported { 260 f.It("should fail in binding dynamic provisioned PV to PVC", f.WithSlow(), "[LinuxOnly]", func(ctx context.Context) { 261 manualInit(ctx) 262 ginkgo.DeferCleanup(cleanup) 263 264 var err error 265 266 ginkgo.By("Creating sc") 267 l.Sc, err = l.cs.StorageV1().StorageClasses().Create(ctx, l.Sc, metav1.CreateOptions{}) 268 framework.ExpectNoError(err, "Failed to create sc") 269 270 ginkgo.By("Creating pv and pvc") 271 l.Pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(ctx, l.Pvc, metav1.CreateOptions{}) 272 framework.ExpectNoError(err, "Failed to create pvc") 273 274 eventSelector := fields.Set{ 275 "involvedObject.kind": "PersistentVolumeClaim", 276 "involvedObject.name": l.Pvc.Name, 277 "involvedObject.namespace": l.ns.Name, 278 "reason": volevents.ProvisioningFailed, 279 }.AsSelector().String() 280 // The error message is different for each storage driver 281 msg := "" 282 283 err = e2eevents.WaitTimeoutForEvent(ctx, l.cs, l.ns.Name, eventSelector, msg, f.Timeouts.ClaimProvision) 284 // Events are unreliable, don't depend on the event. It's used only to speed up the test. 285 if err != nil { 286 framework.Logf("Warning: did not get event about provisioning failed") 287 } 288 289 // Check the pvc is still pending 290 pvc, err := l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Get(ctx, l.Pvc.Name, metav1.GetOptions{}) 291 framework.ExpectNoError(err, "Failed to re-read the pvc after event (or timeout)") 292 gomega.Expect(pvc.Status.Phase).To(gomega.Equal(v1.ClaimPending), "PVC phase isn't pending") 293 }) 294 } 295 default: 296 framework.Failf("Volume mode test doesn't support volType: %v", pattern.VolType) 297 } 298 299 f.It("should fail to use a volume in a pod with mismatched mode", f.WithSlow(), func(ctx context.Context) { 300 skipTestIfBlockNotSupported(driver) 301 init(ctx) 302 testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange 303 l.VolumeResource = *storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange) 304 ginkgo.DeferCleanup(cleanup) 305 306 ginkgo.By("Creating pod") 307 var err error 308 podConfig := e2epod.Config{ 309 NS: l.ns.Name, 310 PVCs: []*v1.PersistentVolumeClaim{l.Pvc}, 311 SeLinuxLabel: e2epod.GetLinuxLabel(), 312 ImageID: e2epod.GetDefaultTestImageID(), 313 } 314 pod, err := e2epod.MakeSecPod(&podConfig) 315 framework.ExpectNoError(err) 316 317 // Change volumeMounts to volumeDevices and the other way around 318 pod = swapVolumeMode(pod) 319 320 // Run the pod 321 pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(ctx, pod, metav1.CreateOptions{}) 322 framework.ExpectNoError(err, "Failed to create pod") 323 defer func() { 324 framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, l.cs, pod), "Failed to delete pod") 325 }() 326 327 ginkgo.By("Waiting for the pod to fail") 328 // Wait for an event that the pod is invalid. 329 eventSelector := fields.Set{ 330 "involvedObject.kind": "Pod", 331 "involvedObject.name": pod.Name, 332 "involvedObject.namespace": l.ns.Name, 333 "reason": events.FailedMountVolume, 334 }.AsSelector().String() 335 336 var msg string 337 if pattern.VolMode == v1.PersistentVolumeBlock { 338 msg = "has volumeMode Block, but is specified in volumeMounts" 339 } else { 340 msg = "has volumeMode Filesystem, but is specified in volumeDevices" 341 } 342 err = e2eevents.WaitTimeoutForEvent(ctx, l.cs, l.ns.Name, eventSelector, msg, f.Timeouts.PodStart) 343 // Events are unreliable, don't depend on them. They're used only to speed up the test. 344 if err != nil { 345 framework.Logf("Warning: did not get event about mismatched volume use") 346 } 347 348 // Check the pod is still not running 349 p, err := l.cs.CoreV1().Pods(l.ns.Name).Get(ctx, pod.Name, metav1.GetOptions{}) 350 framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)") 351 gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodPending), "Pod phase isn't pending") 352 }) 353 354 ginkgo.It("should not mount / map unused volumes in a pod [LinuxOnly]", func(ctx context.Context) { 355 if pattern.VolMode == v1.PersistentVolumeBlock { 356 skipTestIfBlockNotSupported(driver) 357 } 358 init(ctx) 359 testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange 360 l.VolumeResource = *storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange) 361 ginkgo.DeferCleanup(cleanup) 362 363 ginkgo.By("Creating pod") 364 var err error 365 podConfig := e2epod.Config{ 366 NS: l.ns.Name, 367 PVCs: []*v1.PersistentVolumeClaim{l.Pvc}, 368 SeLinuxLabel: e2epod.GetLinuxLabel(), 369 ImageID: e2epod.GetDefaultTestImageID(), 370 } 371 pod, err := e2epod.MakeSecPod(&podConfig) 372 framework.ExpectNoError(err) 373 374 for i := range pod.Spec.Containers { 375 pod.Spec.Containers[i].VolumeDevices = nil 376 pod.Spec.Containers[i].VolumeMounts = nil 377 } 378 379 // Run the pod 380 pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(ctx, pod, metav1.CreateOptions{}) 381 framework.ExpectNoError(err) 382 defer func() { 383 framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, l.cs, pod)) 384 }() 385 386 err = e2epod.WaitForPodNameRunningInNamespace(ctx, l.cs, pod.Name, pod.Namespace) 387 framework.ExpectNoError(err) 388 389 // Reload the pod to get its node 390 pod, err = l.cs.CoreV1().Pods(l.ns.Name).Get(ctx, pod.Name, metav1.GetOptions{}) 391 framework.ExpectNoError(err) 392 gomega.Expect(pod.Spec.NodeName).ToNot(gomega.BeEmpty(), "pod should be scheduled to a node") 393 node, err := l.cs.CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, metav1.GetOptions{}) 394 framework.ExpectNoError(err) 395 396 ginkgo.By("Listing mounted volumes in the pod") 397 hostExec := storageutils.NewHostExec(f) 398 ginkgo.DeferCleanup(hostExec.Cleanup) 399 volumePaths, devicePaths, err := listPodVolumePluginDirectory(ctx, hostExec, pod, node) 400 framework.ExpectNoError(err) 401 402 driverInfo := driver.GetDriverInfo() 403 volumePlugin := driverInfo.InTreePluginName 404 if len(volumePlugin) == 0 { 405 // TODO: check if it's a CSI volume first 406 volumePlugin = "kubernetes.io/csi" 407 } 408 ginkgo.By(fmt.Sprintf("Checking that volume plugin %s is not used in pod directory", volumePlugin)) 409 safeVolumePlugin := strings.ReplaceAll(volumePlugin, "/", "~") 410 for _, path := range volumePaths { 411 gomega.Expect(path).NotTo(gomega.ContainSubstring(safeVolumePlugin), fmt.Sprintf("no %s volume should be mounted into pod directory", volumePlugin)) 412 } 413 for _, path := range devicePaths { 414 gomega.Expect(path).NotTo(gomega.ContainSubstring(safeVolumePlugin), fmt.Sprintf("no %s volume should be symlinked into pod directory", volumePlugin)) 415 } 416 }) 417 } 418 419 func generateConfigsForPreprovisionedPVTest(scName string, volBindMode storagev1.VolumeBindingMode, 420 volMode v1.PersistentVolumeMode, pvSource v1.PersistentVolumeSource, volumeNodeAffinity *v1.VolumeNodeAffinity) (*storagev1.StorageClass, 421 e2epv.PersistentVolumeConfig, e2epv.PersistentVolumeClaimConfig) { 422 // StorageClass 423 scConfig := &storagev1.StorageClass{ 424 ObjectMeta: metav1.ObjectMeta{ 425 Name: scName, 426 }, 427 Provisioner: noProvisioner, 428 VolumeBindingMode: &volBindMode, 429 } 430 // PV 431 pvConfig := e2epv.PersistentVolumeConfig{ 432 PVSource: pvSource, 433 NodeAffinity: volumeNodeAffinity, 434 NamePrefix: pvNamePrefix, 435 StorageClassName: scName, 436 VolumeMode: &volMode, 437 } 438 // PVC 439 pvcConfig := e2epv.PersistentVolumeClaimConfig{ 440 AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, 441 StorageClassName: &scName, 442 VolumeMode: &volMode, 443 } 444 445 return scConfig, pvConfig, pvcConfig 446 } 447 448 // swapVolumeMode changes volumeMounts to volumeDevices and the other way around 449 func swapVolumeMode(podTemplate *v1.Pod) *v1.Pod { 450 pod := podTemplate.DeepCopy() 451 for c := range pod.Spec.Containers { 452 container := &pod.Spec.Containers[c] 453 container.VolumeDevices = []v1.VolumeDevice{} 454 container.VolumeMounts = []v1.VolumeMount{} 455 456 // Change VolumeMounts to VolumeDevices 457 for _, volumeMount := range podTemplate.Spec.Containers[c].VolumeMounts { 458 container.VolumeDevices = append(container.VolumeDevices, v1.VolumeDevice{ 459 Name: volumeMount.Name, 460 DevicePath: volumeMount.MountPath, 461 }) 462 } 463 // Change VolumeDevices to VolumeMounts 464 for _, volumeDevice := range podTemplate.Spec.Containers[c].VolumeDevices { 465 container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{ 466 Name: volumeDevice.Name, 467 MountPath: volumeDevice.DevicePath, 468 }) 469 } 470 } 471 return pod 472 } 473 474 // listPodVolumePluginDirectory returns all volumes in /var/lib/kubelet/pods/<pod UID>/volumes/* and 475 // /var/lib/kubelet/pods/<pod UID>/volumeDevices/* 476 // Sample output: 477 // 478 // /var/lib/kubelet/pods/a4717a30-000a-4081-a7a8-f51adf280036/volumes/kubernetes.io~secret/default-token-rphdt 479 // /var/lib/kubelet/pods/4475b7a3-4a55-4716-9119-fd0053d9d4a6/volumeDevices/kubernetes.io~aws-ebs/pvc-5f9f80f5-c90b-4586-9966-83f91711e1c0 480 func listPodVolumePluginDirectory(ctx context.Context, h storageutils.HostExec, pod *v1.Pod, node *v1.Node) (mounts []string, devices []string, err error) { 481 mountPath := filepath.Join("/var/lib/kubelet/pods/", string(pod.UID), "volumes") 482 devicePath := filepath.Join("/var/lib/kubelet/pods/", string(pod.UID), "volumeDevices") 483 484 mounts, err = listPodDirectory(ctx, h, mountPath, node) 485 if err != nil { 486 return nil, nil, err 487 } 488 devices, err = listPodDirectory(ctx, h, devicePath, node) 489 if err != nil { 490 return nil, nil, err 491 } 492 return mounts, devices, nil 493 } 494 495 func listPodDirectory(ctx context.Context, h storageutils.HostExec, path string, node *v1.Node) ([]string, error) { 496 // Return no error if the directory does not exist (e.g. there are no block volumes used) 497 _, err := h.IssueCommandWithResult(ctx, "test ! -d "+path, node) 498 if err == nil { 499 // The directory does not exist 500 return nil, nil 501 } 502 // The directory either exists or a real error happened (e.g. "access denied"). 503 // Ignore the error, "find" will hit the error again and we report it there. 504 505 // Inside /var/lib/kubelet/pods/<pod>/volumes, look for <volume_plugin>/<volume-name>, hence depth 2 506 cmd := fmt.Sprintf("find %s -mindepth 2 -maxdepth 2", path) 507 out, err := h.IssueCommandWithResult(ctx, cmd, node) 508 if err != nil { 509 return nil, fmt.Errorf("error checking directory %s on node %s: %w", path, node.Name, err) 510 } 511 return strings.Split(out, "\n"), nil 512 }