github.com/vmware/govmomi@v0.51.0/cns/client_test.go (about)

     1  // © Broadcom. All Rights Reserved.
     2  // The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
     3  // SPDX-License-Identifier: Apache-2.0
     4  
     5  package cns
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"os"
    11  	"strconv"
    12  	"strings"
    13  	"testing"
    14  
    15  	"github.com/dougm/pretty"
    16  
    17  	"github.com/vmware/govmomi/find"
    18  	"github.com/vmware/govmomi/object"
    19  	"github.com/vmware/govmomi/property"
    20  	"github.com/vmware/govmomi/vim25/debug"
    21  	"github.com/vmware/govmomi/vim25/mo"
    22  	"github.com/vmware/govmomi/vim25/soap"
    23  
    24  	"github.com/vmware/govmomi"
    25  	cnstypes "github.com/vmware/govmomi/cns/types"
    26  	vim25types "github.com/vmware/govmomi/vim25/types"
    27  	vsanfstypes "github.com/vmware/govmomi/vsan/vsanfs/types"
    28  )
    29  
    30  const VSphere70u3VersionInt = 703
    31  const VSphere80u3VersionInt = 803
    32  
    33  func TestClient(t *testing.T) {
    34  	// set CNS_DEBUG to true if you need to emit soap traces from these tests
    35  	// soap traces will be emitted in the govmomi/cns/.soap directory
    36  	// example export CNS_DEBUG='true'
    37  	enableDebug := os.Getenv("CNS_DEBUG")
    38  	soapTraceDirectory := ".soap"
    39  
    40  	url := os.Getenv("CNS_VC_URL") // example: export CNS_VC_URL='https://username:password@vc-ip/sdk'
    41  	datacenter := os.Getenv("CNS_DATACENTER")
    42  	datastore := os.Getenv("CNS_DATASTORE")
    43  
    44  	// set CNS_RUN_FILESHARE_TESTS environment to true, if your setup has vsanfileshare enabled.
    45  	// when CNS_RUN_FILESHARE_TESTS is not set to true, vsan file share related tests are skipped.
    46  	// example: export CNS_RUN_FILESHARE_TESTS='true'
    47  	run_fileshare_tests := os.Getenv("CNS_RUN_FILESHARE_TESTS")
    48  
    49  	// if backingDiskURLPath is not set, test for Creating Volume with setting BackingDiskUrlPath in the BackingObjectDetails of
    50  	// CnsVolumeCreateSpec will be skipped.
    51  	// example: export BACKING_DISK_URL_PATH='https://vc-ip/folder/vmdkfilePath.vmdk?dcPath=DataCenterPath&dsName=DataStoreName'
    52  	backingDiskURLPath := os.Getenv("BACKING_DISK_URL_PATH")
    53  
    54  	// set REMOTE_VC_URL, REMOTE_DATACENTER only if you want to test cross-VC CNS operations.
    55  	// For instance, testing cross-VC volume migration.
    56  	remoteVcUrl := os.Getenv("REMOTE_VC_URL")
    57  	remoteDatacenter := os.Getenv("REMOTE_DATACENTER")
    58  
    59  	// if datastoreForMigration is not set, test for CNS Relocate API of a volume to another datastore is skipped.
    60  	// input format is same as CNS_DATASTORE. Format eg. "vSANDirect_10.92.217.162_mpx.vmhba0:C0:T2:L0"/ "vsandatastore"
    61  	datastoreForMigration := os.Getenv("CNS_MIGRATION_DATASTORE")
    62  
    63  	// if spbmPolicyId4Reconfig is not set, test for CnsReconfigVolumePolicy API will be skipped
    64  	// example: export CNS_SPBM_POLICY_ID_4_RECONFIG=6f64d90e-2ad5-4c4d-8cbc-a3330ebc496c
    65  	spbmPolicyId4Reconfig := os.Getenv("CNS_SPBM_POLICY_ID_4_RECONFIG")
    66  
    67  	if url == "" || datacenter == "" || datastore == "" {
    68  		t.Skip("CNS_VC_URL or CNS_DATACENTER or CNS_DATASTORE is not set")
    69  	}
    70  	resourcePoolPath := os.Getenv("CNS_RESOURCE_POOL_PATH") // example "/datacenter-name/host/host-ip/Resources" or  /datacenter-name/host/cluster-name/Resources
    71  	u, err := soap.ParseURL(url)
    72  	if err != nil {
    73  		t.Fatal(err)
    74  	}
    75  
    76  	if enableDebug == "true" {
    77  		if _, err := os.Stat(soapTraceDirectory); os.IsNotExist(err) {
    78  			os.Mkdir(soapTraceDirectory, 0755)
    79  		}
    80  		p := debug.FileProvider{
    81  			Path: soapTraceDirectory,
    82  		}
    83  		debug.SetProvider(&p)
    84  	}
    85  
    86  	ctx := context.Background()
    87  	c, err := govmomi.NewClient(ctx, u, true)
    88  	if err != nil {
    89  		t.Fatal(err)
    90  	}
    91  	cnsClient, err := NewClient(ctx, c.Client)
    92  	if err != nil {
    93  		t.Fatal(err)
    94  	}
    95  	finder := find.NewFinder(cnsClient.vim25Client, false)
    96  	dc, err := finder.Datacenter(ctx, datacenter)
    97  	if err != nil {
    98  		t.Fatal(err)
    99  	}
   100  	finder.SetDatacenter(dc)
   101  	ds, err := finder.Datastore(ctx, datastore)
   102  	if err != nil {
   103  		t.Fatal(err)
   104  	}
   105  
   106  	props := []string{"info", "summary"}
   107  	pc := property.DefaultCollector(c.Client)
   108  	var dsSummaries []mo.Datastore
   109  	err = pc.Retrieve(ctx, []vim25types.ManagedObjectReference{ds.Reference()}, props, &dsSummaries)
   110  	if err != nil {
   111  		t.Fatal(err)
   112  	}
   113  	dsUrl := dsSummaries[0].Summary.Url
   114  
   115  	var dsList []vim25types.ManagedObjectReference
   116  	dsList = append(dsList, ds.Reference())
   117  
   118  	var containerClusterArray []cnstypes.CnsContainerCluster
   119  	containerCluster := cnstypes.CnsContainerCluster{
   120  		ClusterType:         string(cnstypes.CnsClusterTypeKubernetes),
   121  		ClusterId:           "demo-cluster-id",
   122  		VSphereUser:         "Administrator@vsphere.local",
   123  		ClusterFlavor:       string(cnstypes.CnsClusterFlavorVanilla),
   124  		ClusterDistribution: "OpenShift",
   125  	}
   126  	containerClusterArray = append(containerClusterArray, containerCluster)
   127  
   128  	// Test CreateVolume API
   129  	var cnsVolumeCreateSpecList []cnstypes.CnsVolumeCreateSpec
   130  	cnsVolumeCreateSpec := cnstypes.CnsVolumeCreateSpec{
   131  		Name:       "pvc-901e87eb-c2bd-11e9-806f-005056a0c9a0",
   132  		VolumeType: string(cnstypes.CnsVolumeTypeBlock),
   133  		Datastores: dsList,
   134  		Metadata: cnstypes.CnsVolumeMetadata{
   135  			ContainerCluster: containerCluster,
   136  		},
   137  		BackingObjectDetails: &cnstypes.CnsBlockBackingDetails{
   138  			CnsBackingObjectDetails: cnstypes.CnsBackingObjectDetails{
   139  				CapacityInMb: 5120,
   140  			},
   141  		},
   142  	}
   143  	cnsVolumeCreateSpecList = append(cnsVolumeCreateSpecList, cnsVolumeCreateSpec)
   144  	t.Logf("Creating volume using the spec: %+v", pretty.Sprint(cnsVolumeCreateSpec))
   145  	createTask, err := cnsClient.CreateVolume(ctx, cnsVolumeCreateSpecList)
   146  	if err != nil {
   147  		t.Errorf("Failed to create volume. Error: %+v \n", err)
   148  		t.Fatal(err)
   149  	}
   150  	createTaskInfo, err := GetTaskInfo(ctx, createTask)
   151  	if err != nil {
   152  		t.Errorf("Failed to create volume. Error: %+v \n", err)
   153  		t.Fatal(err)
   154  	}
   155  	createTaskResult, err := GetTaskResult(ctx, createTaskInfo)
   156  	if err != nil {
   157  		t.Errorf("Failed to create volume. Error: %+v \n", err)
   158  		t.Fatal(err)
   159  	}
   160  	if createTaskResult == nil {
   161  		t.Fatalf("Empty create task results")
   162  		t.FailNow()
   163  	}
   164  	createVolumeOperationRes := createTaskResult.GetCnsVolumeOperationResult()
   165  	if createVolumeOperationRes.Fault != nil {
   166  		t.Fatalf("Failed to create volume: fault=%+v", createVolumeOperationRes.Fault)
   167  	}
   168  	volumeId := createVolumeOperationRes.VolumeId.Id
   169  	volumeCreateResult := (createTaskResult).(*cnstypes.CnsVolumeCreateResult)
   170  	t.Logf("volumeCreateResult %+v", volumeCreateResult)
   171  	t.Logf("Volume created sucessfully. volumeId: %s", volumeId)
   172  
   173  	if cnsClient.Version != ReleaseVSAN67u3 {
   174  		// Test creating static volume using existing CNS volume should fail
   175  		var staticCnsVolumeCreateSpecList []cnstypes.CnsVolumeCreateSpec
   176  		staticCnsVolumeCreateSpec := cnstypes.CnsVolumeCreateSpec{
   177  			Name:       "pvc-901e87eb-c2bd-11e9-806f-005056a0c9a0",
   178  			VolumeType: string(cnstypes.CnsVolumeTypeBlock),
   179  			Metadata: cnstypes.CnsVolumeMetadata{
   180  				ContainerCluster: containerCluster,
   181  			},
   182  			BackingObjectDetails: &cnstypes.CnsBlockBackingDetails{
   183  				CnsBackingObjectDetails: cnstypes.CnsBackingObjectDetails{
   184  					CapacityInMb: 5120,
   185  				},
   186  				BackingDiskId: volumeId,
   187  			},
   188  		}
   189  
   190  		staticCnsVolumeCreateSpecList = append(staticCnsVolumeCreateSpecList, staticCnsVolumeCreateSpec)
   191  		t.Logf("Creating volume using the spec: %+v", pretty.Sprint(staticCnsVolumeCreateSpec))
   192  		recreateTask, err := cnsClient.CreateVolume(ctx, staticCnsVolumeCreateSpecList)
   193  		if err != nil {
   194  			t.Errorf("Failed to create volume. Error: %+v \n", err)
   195  			t.Fatal(err)
   196  		}
   197  		reCreateTaskInfo, err := GetTaskInfo(ctx, recreateTask)
   198  		if err != nil {
   199  			t.Errorf("Failed to create volume. Error: %+v \n", err)
   200  			t.Fatal(err)
   201  		}
   202  		reCreateTaskResult, err := GetTaskResult(ctx, reCreateTaskInfo)
   203  		if err != nil {
   204  			t.Errorf("Failed to create volume. Error: %+v \n", err)
   205  			t.Fatal(err)
   206  		}
   207  		if reCreateTaskResult == nil {
   208  			t.Fatalf("Empty create task results")
   209  			t.FailNow()
   210  		}
   211  		reCreateVolumeOperationRes := reCreateTaskResult.GetCnsVolumeOperationResult()
   212  		t.Logf("reCreateVolumeOperationRes.: %+v", pretty.Sprint(reCreateVolumeOperationRes))
   213  		if reCreateVolumeOperationRes.Fault != nil {
   214  			t.Logf("reCreateVolumeOperationRes.Fault: %+v", pretty.Sprint(reCreateVolumeOperationRes.Fault))
   215  			_, ok := reCreateVolumeOperationRes.Fault.Fault.(cnstypes.CnsAlreadyRegisteredFault)
   216  			if !ok {
   217  				t.Fatalf("Fault is not a CnsAlreadyRegisteredFault")
   218  			}
   219  		} else {
   220  			t.Fatalf("re-create same volume should fail with CnsAlreadyRegisteredFault")
   221  		}
   222  	}
   223  
   224  	// Test QueryVolume API
   225  	var queryFilter cnstypes.CnsQueryFilter
   226  	var volumeIDList []cnstypes.CnsVolumeId
   227  	volumeIDList = append(volumeIDList, cnstypes.CnsVolumeId{Id: volumeId})
   228  	queryFilter.VolumeIds = volumeIDList
   229  	t.Logf("Calling QueryVolume using queryFilter: %+v", pretty.Sprint(queryFilter))
   230  	queryResult, err := cnsClient.QueryVolume(ctx, queryFilter)
   231  	if err != nil {
   232  		t.Errorf("Failed to query volume. Error: %+v \n", err)
   233  		t.Fatal(err)
   234  	}
   235  	t.Logf("Successfully Queried Volumes. queryResult: %+v", pretty.Sprint(queryResult))
   236  
   237  	// Test QueryVolumeInfo API
   238  	// QueryVolumeInfo is not supported on ReleaseVSAN67u3 and ReleaseVSAN70
   239  	// This API is available on vSphere 7.0u1 onward
   240  	if cnsClient.Version != ReleaseVSAN67u3 && cnsClient.Version != ReleaseVSAN70 {
   241  		t.Logf("Calling QueryVolumeInfo using: %+v", pretty.Sprint(volumeIDList))
   242  		queryVolumeInfoTask, err := cnsClient.QueryVolumeInfo(ctx, volumeIDList)
   243  		if err != nil {
   244  			t.Errorf("Failed to query volumes with QueryVolumeInfo. Error: %+v \n", err)
   245  			t.Fatal(err)
   246  		}
   247  		queryVolumeInfoTaskInfo, err := GetTaskInfo(ctx, queryVolumeInfoTask)
   248  		if err != nil {
   249  			t.Errorf("Failed to query volumes with QueryVolumeInfo. Error: %+v \n", err)
   250  			t.Fatal(err)
   251  		}
   252  		queryVolumeInfoTaskResults, err := GetTaskResultArray(ctx, queryVolumeInfoTaskInfo)
   253  		if err != nil {
   254  			t.Errorf("Failed to query volumes with QueryVolumeInfo. Error: %+v \n", err)
   255  			t.Fatal(err)
   256  		}
   257  		if queryVolumeInfoTaskResults == nil {
   258  			t.Fatalf("Empty queryVolumeInfoTaskResult")
   259  			t.FailNow()
   260  		}
   261  		for _, queryVolumeInfoTaskResult := range queryVolumeInfoTaskResults {
   262  			queryVolumeInfoOperationRes := queryVolumeInfoTaskResult.GetCnsVolumeOperationResult()
   263  			if queryVolumeInfoOperationRes.Fault != nil {
   264  				t.Fatalf("Failed to query volumes with QueryVolumeInfo. fault=%+v", queryVolumeInfoOperationRes.Fault)
   265  			}
   266  			t.Logf("Successfully Queried Volumes. queryVolumeInfoTaskResult: %+v", pretty.Sprint(queryVolumeInfoTaskResult))
   267  		}
   268  	}
   269  
   270  	// Test BackingDiskObjectId field only for vVol or vSAN volume type
   271  	var queryFilterBackingDiskObjectIdTest cnstypes.CnsQueryFilter
   272  	var volumeIDListBackingDiskObjectIdTest []cnstypes.CnsVolumeId
   273  	volumeIDListBackingDiskObjectIdTest = append(volumeIDListBackingDiskObjectIdTest, cnstypes.CnsVolumeId{Id: volumeId})
   274  	queryFilterBackingDiskObjectIdTest.VolumeIds = volumeIDListBackingDiskObjectIdTest
   275  	t.Logf("Calling QueryVolume using queryFilter: %+v", pretty.Sprint(queryFilterBackingDiskObjectIdTest))
   276  	queryResultBackingDiskObjectIdTest, err := cnsClient.QueryVolume(ctx, queryFilterBackingDiskObjectIdTest)
   277  	if err != nil {
   278  		t.Errorf("Failed to query all volumes. Error: %+v \n", err)
   279  		t.Fatal(err)
   280  	}
   281  	t.Logf("Successfully Queried Volumes. queryResultBackingDiskObjectIdTest: %+v", pretty.Sprint(queryResultBackingDiskObjectIdTest))
   282  	t.Log("Checking backingDiskObjectId retieved")
   283  	datastoreType, err := ds.Type(ctx)
   284  	if err != nil {
   285  		t.Errorf("Failed to get datastore type. Error: %+v \n", err)
   286  		t.Fatal(err)
   287  	}
   288  	for _, vol := range queryResultBackingDiskObjectIdTest.Volumes {
   289  		// BackingDiskObjectId is only for vsan/vvol, for other type this field is empty but test should not fail
   290  		backingDiskObjectId := vol.BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).BackingDiskObjectId
   291  		if backingDiskObjectId == "" {
   292  			if datastoreType == vim25types.HostFileSystemVolumeFileSystemTypeVsan || datastoreType == vim25types.HostFileSystemVolumeFileSystemTypeVVOL {
   293  				t.Errorf("Failed to get BackingDiskObjectId")
   294  				t.FailNow()
   295  			}
   296  		}
   297  	}
   298  
   299  	// Test BackingDiskPath field
   300  	var queryFilterBackingDiskPathTest cnstypes.CnsQueryFilter
   301  	var volumeIDListBackingDiskPathTest []cnstypes.CnsVolumeId
   302  	volumeIDListBackingDiskPathTest = append(volumeIDListBackingDiskPathTest, cnstypes.CnsVolumeId{Id: volumeId})
   303  	queryFilterBackingDiskPathTest.VolumeIds = volumeIDListBackingDiskPathTest
   304  	t.Logf("Calling QueryVolume using queryFilter: %+v", pretty.Sprint(queryFilterBackingDiskPathTest))
   305  	queryResultBackingDiskPathTest, err := cnsClient.QueryVolume(ctx, queryFilterBackingDiskPathTest)
   306  	if err != nil {
   307  		t.Errorf("Failed to query all volumes. Error: %+v \n", err)
   308  		t.Fatal(err)
   309  	}
   310  	t.Logf("Successfully Queried Volumes. queryResultBackingDiskPathTest: %+v", pretty.Sprint(queryResultBackingDiskPathTest))
   311  	t.Log("Checking backingDiskPath retrieved")
   312  	for _, vol := range queryResultBackingDiskPathTest.Volumes {
   313  		backingDiskPath := vol.BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).BackingDiskPath
   314  		if backingDiskPath == "" {
   315  			t.Errorf("Failed to get BackingDiskPath")
   316  			t.FailNow()
   317  		}
   318  	}
   319  
   320  	// Test QuerySnapshots API on 7.0 U3 or above
   321  	var snapshotQueryFilter cnstypes.CnsSnapshotQueryFilter
   322  	var querySnapshotsTaskResult *cnstypes.CnsSnapshotQueryResult
   323  	var QuerySnapshotsFunc func(snapshotQueryFilter cnstypes.CnsSnapshotQueryFilter) *cnstypes.CnsSnapshotQueryResult
   324  
   325  	if isvSphereVersion70U3orAbove(ctx, c.ServiceContent.About) {
   326  		// Construct the CNS SnapshotQueryFilter and the function handler of QuerySnapshots
   327  		QuerySnapshotsFunc = func(snapshotQueryFilter cnstypes.CnsSnapshotQueryFilter) *cnstypes.CnsSnapshotQueryResult {
   328  			querySnapshotsTask, err := cnsClient.QuerySnapshots(ctx, snapshotQueryFilter)
   329  			if err != nil {
   330  				t.Fatalf("Failed to get the task of QuerySnapshots. Error: %+v \n", err)
   331  			}
   332  			querySnapshotsTaskInfo, err := GetTaskInfo(ctx, querySnapshotsTask)
   333  			if err != nil {
   334  				t.Fatalf("Failed to get the task info of QuerySnapshots. Error: %+v \n", err)
   335  			}
   336  			querySnapshotsTaskResult, err := GetQuerySnapshotsTaskResult(ctx, querySnapshotsTaskInfo)
   337  			if err != nil {
   338  				t.Fatalf("Failed to get the task result of QuerySnapshots. Error: %+v \n", err)
   339  			}
   340  			return querySnapshotsTaskResult
   341  		}
   342  
   343  		// Calls QuerySnapshots before CreateSnapshots
   344  		snapshotQueryFilter = cnstypes.CnsSnapshotQueryFilter{
   345  			SnapshotQuerySpecs: []cnstypes.CnsSnapshotQuerySpec{
   346  				{
   347  					VolumeId: cnstypes.CnsVolumeId{Id: volumeId},
   348  				},
   349  			},
   350  		}
   351  		t.Logf("QuerySnapshots before CreateSnapshots, snapshotQueryFilter %+v", snapshotQueryFilter)
   352  		querySnapshotsTaskResult = QuerySnapshotsFunc(snapshotQueryFilter)
   353  		t.Logf("snapshotQueryResult %+v", querySnapshotsTaskResult)
   354  	}
   355  
   356  	// Test CreateSnapshot API
   357  	// Construct the CNS SnapshotCreateSpec list
   358  	desc := "example-vanilla-block-snapshot"
   359  	var cnsSnapshotCreateSpecList []cnstypes.CnsSnapshotCreateSpec
   360  	cnsSnapshotCreateSpec := cnstypes.CnsSnapshotCreateSpec{
   361  		VolumeId: cnstypes.CnsVolumeId{
   362  			Id: volumeId,
   363  		},
   364  		Description: desc,
   365  	}
   366  	cnsSnapshotCreateSpecList = append(cnsSnapshotCreateSpecList, cnsSnapshotCreateSpec)
   367  	t.Logf("Creating snapshot using the spec: %+v", pretty.Sprint(cnsSnapshotCreateSpecList))
   368  	createSnapshotsTask, err := cnsClient.CreateSnapshots(ctx, cnsSnapshotCreateSpecList)
   369  	if err != nil {
   370  		t.Errorf("Failed to get the task of CreateSnapshots. Error: %+v \n", err)
   371  		t.Fatal(err)
   372  	}
   373  	createSnapshotsTaskInfo, err := GetTaskInfo(ctx, createSnapshotsTask)
   374  	if err != nil {
   375  		t.Errorf("Failed to get the task info of CreateSnapshots. Error: %+v \n", err)
   376  		t.Fatal(err)
   377  	}
   378  	createSnapshotsTaskResult, err := GetTaskResult(ctx, createSnapshotsTaskInfo)
   379  	if err != nil {
   380  		t.Errorf("Failed to get the task result of CreateSnapshots. Error: %+v \n", err)
   381  		t.Fatal(err)
   382  	}
   383  	createSnapshotsOperationRes := createSnapshotsTaskResult.GetCnsVolumeOperationResult()
   384  	if createSnapshotsOperationRes.Fault != nil {
   385  		t.Fatalf("Failed to create snapshots: fault=%+v", createSnapshotsOperationRes.Fault)
   386  	}
   387  
   388  	snapshotCreateResult := any(createSnapshotsTaskResult).(*cnstypes.CnsSnapshotCreateResult)
   389  	snapshotId := snapshotCreateResult.Snapshot.SnapshotId.Id
   390  	snapshotCreateTime := snapshotCreateResult.Snapshot.CreateTime
   391  	t.Logf("snapshotCreateResult: %+v", pretty.Sprint(snapshotCreateResult))
   392  	t.Logf("CreateSnapshots: Snapshot created successfully. volumeId: %q, snapshot id %q, time stamp %+v, opId: %q", volumeId, snapshotId, snapshotCreateTime, createSnapshotsTaskInfo.ActivationId)
   393  
   394  	// Test QuerySnapshots API on 7.0 U3 or above
   395  	if isvSphereVersion70U3orAbove(ctx, c.ServiceContent.About) {
   396  		// Calls QuerySnapshots after CreateSnapshots
   397  		snapshotQueryFilter = cnstypes.CnsSnapshotQueryFilter{
   398  			SnapshotQuerySpecs: []cnstypes.CnsSnapshotQuerySpec{
   399  				{
   400  					VolumeId:   cnstypes.CnsVolumeId{Id: volumeId},
   401  					SnapshotId: &cnstypes.CnsSnapshotId{Id: snapshotId},
   402  				},
   403  			},
   404  		}
   405  		t.Logf("QuerySnapshots after CreateSnapshots, snapshotQueryFilter %+v", snapshotQueryFilter)
   406  		querySnapshotsTaskResult = QuerySnapshotsFunc(snapshotQueryFilter)
   407  		t.Logf("snapshotQueryResult %+v", querySnapshotsTaskResult)
   408  	}
   409  
   410  	// Test CreateVolumeFromSnapshot functionality by calling CreateVolume with VolumeSource set
   411  	// Query Volume for capacity
   412  	var queryVolumeIDList []cnstypes.CnsVolumeId
   413  	queryVolumeIDList = append(queryVolumeIDList, cnstypes.CnsVolumeId{Id: volumeId})
   414  	queryFilter.VolumeIds = queryVolumeIDList
   415  	t.Logf("CreateVolumeFromSnapshot: calling QueryVolume using queryFilter: %+v", pretty.Sprint(queryFilter))
   416  	queryResult, err = cnsClient.QueryVolume(ctx, queryFilter)
   417  	if err != nil {
   418  		t.Errorf("Failed to query volume. Error: %+v \n", err)
   419  		t.Fatal(err)
   420  	}
   421  	var snapshotSize int64
   422  	if len(queryResult.Volumes) > 0 {
   423  		snapshotSize = queryResult.Volumes[0].BackingObjectDetails.GetCnsBackingObjectDetails().CapacityInMb
   424  	} else {
   425  		msg := fmt.Sprintf("failed to get the snapshot size by querying volume: %q", volumeId)
   426  		t.Fatal(msg)
   427  	}
   428  	t.Logf("CreateVolumeFromSnapshot: Successfully Queried Volumes. queryResult: %+v", pretty.Sprint(queryResult))
   429  
   430  	// Construct the CNS VolumeCreateSpec list
   431  	cnsCreateVolumeFromSnapshotCreateSpec := cnstypes.CnsVolumeCreateSpec{
   432  		Name:       "pvc-901e87eb-c2bd-11e9-806f-005056a0c9a0-create-from-snapshot",
   433  		VolumeType: string(cnstypes.CnsVolumeTypeBlock),
   434  		Datastores: dsList,
   435  		Metadata: cnstypes.CnsVolumeMetadata{
   436  			ContainerCluster: containerCluster,
   437  		},
   438  		BackingObjectDetails: &cnstypes.CnsBlockBackingDetails{
   439  			CnsBackingObjectDetails: cnstypes.CnsBackingObjectDetails{
   440  				CapacityInMb: snapshotSize,
   441  			},
   442  		},
   443  		VolumeSource: &cnstypes.CnsSnapshotVolumeSource{
   444  			VolumeId: cnstypes.CnsVolumeId{
   445  				Id: volumeId,
   446  			},
   447  			SnapshotId: cnstypes.CnsSnapshotId{
   448  				Id: snapshotId,
   449  			},
   450  		},
   451  	}
   452  	var cnsCreateVolumeFromSnapshotCreateSpecList []cnstypes.CnsVolumeCreateSpec
   453  	cnsCreateVolumeFromSnapshotCreateSpecList = append(cnsCreateVolumeFromSnapshotCreateSpecList, cnsCreateVolumeFromSnapshotCreateSpec)
   454  	t.Logf("Creating volume from snapshot using the spec: %+v", pretty.Sprint(cnsCreateVolumeFromSnapshotCreateSpec))
   455  	createVolumeFromSnapshotTask, err := cnsClient.CreateVolume(ctx, cnsCreateVolumeFromSnapshotCreateSpecList)
   456  	if err != nil {
   457  		t.Errorf("Failed to create volume from snapshot. Error: %+v \n", err)
   458  		t.Fatal(err)
   459  	}
   460  	createVolumeFromSnapshotTaskInfo, err := GetTaskInfo(ctx, createVolumeFromSnapshotTask)
   461  	if err != nil {
   462  		t.Errorf("Failed to create volume from snapshot. Error: %+v \n", err)
   463  		t.Fatal(err)
   464  	}
   465  	createVolumeFromSnapshotTaskResult, err := GetTaskResult(ctx, createVolumeFromSnapshotTaskInfo)
   466  	if err != nil {
   467  		t.Errorf("Failed to create volume from snapshot. Error: %+v \n", err)
   468  		t.Fatal(err)
   469  	}
   470  	if createVolumeFromSnapshotTaskResult == nil {
   471  		t.Fatalf("Empty create task results")
   472  		t.FailNow()
   473  	}
   474  	createVolumeFromSnapshotOperationRes := createVolumeFromSnapshotTaskResult.GetCnsVolumeOperationResult()
   475  	if createVolumeFromSnapshotOperationRes.Fault != nil {
   476  		t.Fatalf("Failed to create volume from snapshot: fault=%+v", createVolumeFromSnapshotOperationRes.Fault)
   477  	}
   478  	createVolumeFromSnapshotVolumeId := createVolumeFromSnapshotOperationRes.VolumeId.Id
   479  	createVolumeFromSnapshotResult := (createVolumeFromSnapshotTaskResult).(*cnstypes.CnsVolumeCreateResult)
   480  	t.Logf("createVolumeFromSnapshotResult %+v", createVolumeFromSnapshotResult)
   481  	t.Logf("Volume created from snapshot %s sucessfully. volumeId: %s", snapshotId, createVolumeFromSnapshotVolumeId)
   482  
   483  	//  Clean up volume created from snapshot above
   484  	var deleteVolumeFromSnapshotVolumeIDList []cnstypes.CnsVolumeId
   485  	deleteVolumeFromSnapshotVolumeIDList = append(deleteVolumeFromSnapshotVolumeIDList, cnstypes.CnsVolumeId{Id: createVolumeFromSnapshotVolumeId})
   486  	t.Logf("Deleting volume: %+v", deleteVolumeFromSnapshotVolumeIDList)
   487  	deleteVolumeFromSnapshotTask, err := cnsClient.DeleteVolume(ctx, deleteVolumeFromSnapshotVolumeIDList, true)
   488  	if err != nil {
   489  		t.Errorf("Failed to delete volume. Error: %+v \n", err)
   490  		t.Fatal(err)
   491  	}
   492  	deleteVolumeFromSnapshotTaskInfo, err := GetTaskInfo(ctx, deleteVolumeFromSnapshotTask)
   493  	if err != nil {
   494  		t.Errorf("Failed to delete volume. Error: %+v \n", err)
   495  		t.Fatal(err)
   496  	}
   497  	deleteVolumeFromSnapshotTaskResult, err := GetTaskResult(ctx, deleteVolumeFromSnapshotTaskInfo)
   498  	if err != nil {
   499  		t.Errorf("Failed to delete volume. Error: %+v \n", err)
   500  		t.Fatal(err)
   501  	}
   502  	if deleteVolumeFromSnapshotTaskResult == nil {
   503  		t.Fatalf("Empty delete task results")
   504  		t.FailNow()
   505  	}
   506  	deleteVolumeFromSnapshotOperationRes := deleteVolumeFromSnapshotTaskResult.GetCnsVolumeOperationResult()
   507  	if deleteVolumeFromSnapshotOperationRes.Fault != nil {
   508  		t.Fatalf("Failed to delete volume: fault=%+v", deleteVolumeFromSnapshotOperationRes.Fault)
   509  	}
   510  	t.Logf("Volume: %q deleted sucessfully", createVolumeFromSnapshotVolumeId)
   511  
   512  	// Test DeleteSnapshot API
   513  	// Construct the CNS SnapshotDeleteSpec list
   514  	var cnsSnapshotDeleteSpecList []cnstypes.CnsSnapshotDeleteSpec
   515  	cnsSnapshotDeleteSpec := cnstypes.CnsSnapshotDeleteSpec{
   516  		VolumeId: cnstypes.CnsVolumeId{
   517  			Id: volumeId,
   518  		},
   519  		SnapshotId: cnstypes.CnsSnapshotId{
   520  			Id: snapshotId,
   521  		},
   522  	}
   523  	cnsSnapshotDeleteSpecList = append(cnsSnapshotDeleteSpecList, cnsSnapshotDeleteSpec)
   524  	t.Logf("Deleting snapshot using the spec: %+v", pretty.Sprint(cnsSnapshotDeleteSpecList))
   525  	deleteSnapshotsTask, err := cnsClient.DeleteSnapshots(ctx, cnsSnapshotDeleteSpecList)
   526  	if err != nil {
   527  		t.Errorf("Failed to get the task of DeleteSnapshots. Error: %+v \n", err)
   528  		t.Fatal(err)
   529  	}
   530  	deleteSnapshotsTaskInfo, err := GetTaskInfo(ctx, deleteSnapshotsTask)
   531  	if err != nil {
   532  		t.Errorf("Failed to get the task info of DeleteSnapshots. Error: %+v \n", err)
   533  		t.Fatal(err)
   534  	}
   535  
   536  	deleteSnapshotsTaskResult, err := GetTaskResult(ctx, deleteSnapshotsTaskInfo)
   537  	if err != nil {
   538  		t.Errorf("Failed to get the task result of DeleteSnapshots. Error: %+v \n", err)
   539  		t.Fatal(err)
   540  	}
   541  
   542  	deleteSnapshotsOperationRes := deleteSnapshotsTaskResult.GetCnsVolumeOperationResult()
   543  	if deleteSnapshotsOperationRes.Fault != nil {
   544  		t.Fatalf("Failed to delete snapshots: fault=%+v", deleteSnapshotsOperationRes.Fault)
   545  	}
   546  
   547  	snapshotDeleteResult := any(deleteSnapshotsTaskResult).(*cnstypes.CnsSnapshotDeleteResult)
   548  	t.Logf("snapshotDeleteResult: %+v", pretty.Sprint(snapshotCreateResult))
   549  	t.Logf("DeleteSnapshots: Snapshot deleted successfully. volumeId: %q, snapshot id %q, opId: %q", volumeId, snapshotDeleteResult.SnapshotId, deleteSnapshotsTaskInfo.ActivationId)
   550  
   551  	// Test Relocate API
   552  	// Relocate API is not supported on ReleaseVSAN67u3 and ReleaseVSAN70
   553  	// This API is available on vSphere 7.0u1 onward
   554  	if cnsClient.Version != ReleaseVSAN67u3 && cnsClient.Version != ReleaseVSAN70 &&
   555  		datastoreForMigration != "" {
   556  
   557  		var migrationDS *object.Datastore
   558  		var serviceLocatorInstance *vim25types.ServiceLocator = nil
   559  
   560  		// Cross-VC migration.
   561  		// This is only supported on 8.0u3 onwards.
   562  		if remoteVcUrl != "" && isvSphereVersion80U3orAbove(ctx, c.ServiceContent.About) {
   563  			remoteUrl, err := soap.ParseURL(remoteVcUrl)
   564  			if err != nil {
   565  				t.Fatal(err)
   566  			}
   567  			remoteVcClient, err := govmomi.NewClient(ctx, remoteUrl, true)
   568  			if err != nil {
   569  				t.Fatal(err)
   570  			}
   571  			remoteCnsClient, err := NewClient(ctx, remoteVcClient.Client)
   572  			if err != nil {
   573  				t.Fatal(err)
   574  			}
   575  			remoteFinder := find.NewFinder(remoteCnsClient.vim25Client, false)
   576  			remoteDc, err := remoteFinder.Datacenter(ctx, remoteDatacenter)
   577  			if err != nil {
   578  				t.Fatal(err)
   579  			}
   580  			remoteFinder.SetDatacenter(remoteDc)
   581  
   582  			migrationDS, err = remoteFinder.Datastore(ctx, datastoreForMigration)
   583  			if err != nil {
   584  				t.Fatal(err)
   585  			}
   586  
   587  			// Get ServiceLocator instance for remote VC.
   588  			userName := remoteUrl.User.Username()
   589  			password, _ := remoteUrl.User.Password()
   590  			serviceLocatorInstance, err = GetServiceLocatorInstance(ctx, userName, password, remoteVcClient)
   591  			if err != nil {
   592  				t.Fatal(err)
   593  			}
   594  
   595  		} else {
   596  			// Same VC migration
   597  			migrationDS, err = finder.Datastore(ctx, datastoreForMigration)
   598  			if err != nil {
   599  				t.Fatal(err)
   600  			}
   601  		}
   602  
   603  		blockVolRelocateSpec := cnstypes.CnsBlockVolumeRelocateSpec{
   604  			CnsVolumeRelocateSpec: cnstypes.CnsVolumeRelocateSpec{
   605  				VolumeId: cnstypes.CnsVolumeId{
   606  					Id: volumeId,
   607  				},
   608  				Datastore: migrationDS.Reference(),
   609  			},
   610  		}
   611  		if serviceLocatorInstance != nil {
   612  			blockVolRelocateSpec.ServiceLocator = serviceLocatorInstance
   613  		}
   614  
   615  		t.Logf("Relocating volume using the spec: %+v", pretty.Sprint(blockVolRelocateSpec))
   616  
   617  		relocateTask, err := cnsClient.RelocateVolume(ctx, blockVolRelocateSpec)
   618  		if err != nil {
   619  			t.Errorf("Failed to migrate volume with Relocate API. Error: %+v \n", err)
   620  			t.Fatal(err)
   621  		}
   622  		relocateTaskInfo, err := GetTaskInfo(ctx, relocateTask)
   623  		if err != nil {
   624  			t.Errorf("Failed to get info of task returned by Relocate API. Error: %+v \n", err)
   625  			t.Fatal(err)
   626  		}
   627  		taskResults, err := GetTaskResultArray(ctx, relocateTaskInfo)
   628  		if err != nil {
   629  			t.Fatal(err)
   630  		}
   631  		for _, taskResult := range taskResults {
   632  			res := taskResult.GetCnsVolumeOperationResult()
   633  			if res.Fault != nil {
   634  				t.Fatalf("Relocation failed due to fault: %+v", res.Fault)
   635  			}
   636  			t.Logf("Successfully Relocated volume. Relocate task info result: %+v", pretty.Sprint(taskResult))
   637  		}
   638  	}
   639  
   640  	// Test ExtendVolume API
   641  	var newCapacityInMb int64 = 10240
   642  	var cnsVolumeExtendSpecList []cnstypes.CnsVolumeExtendSpec
   643  	cnsVolumeExtendSpec := cnstypes.CnsVolumeExtendSpec{
   644  		VolumeId: cnstypes.CnsVolumeId{
   645  			Id: volumeId,
   646  		},
   647  		CapacityInMb: newCapacityInMb,
   648  	}
   649  	cnsVolumeExtendSpecList = append(cnsVolumeExtendSpecList, cnsVolumeExtendSpec)
   650  	t.Logf("Extending volume using the spec: %+v", pretty.Sprint(cnsVolumeExtendSpecList))
   651  	extendTask, err := cnsClient.ExtendVolume(ctx, cnsVolumeExtendSpecList)
   652  	if err != nil {
   653  		t.Errorf("Failed to extend volume. Error: %+v \n", err)
   654  		t.Fatal(err)
   655  	}
   656  	extendTaskInfo, err := GetTaskInfo(ctx, extendTask)
   657  	if err != nil {
   658  		t.Errorf("Failed to extend volume. Error: %+v \n", err)
   659  		t.Fatal(err)
   660  	}
   661  	extendTaskResult, err := GetTaskResult(ctx, extendTaskInfo)
   662  	if err != nil {
   663  		t.Errorf("Failed to extend volume. Error: %+v \n", err)
   664  		t.Fatal(err)
   665  	}
   666  	if extendTaskResult == nil {
   667  		t.Fatalf("Empty extend task results")
   668  		t.FailNow()
   669  	}
   670  	extendVolumeOperationRes := extendTaskResult.GetCnsVolumeOperationResult()
   671  	if extendVolumeOperationRes.Fault != nil {
   672  		t.Fatalf("Failed to extend volume: fault=%+v", extendVolumeOperationRes.Fault)
   673  	}
   674  	extendVolumeId := extendVolumeOperationRes.VolumeId.Id
   675  	t.Logf("Volume extended sucessfully. Volume ID: %s", extendVolumeId)
   676  
   677  	// Verify volume is extended to the specified size
   678  	t.Logf("Calling QueryVolume after ExtendVolume using queryFilter: %+v", queryFilter)
   679  	queryResult, err = cnsClient.QueryVolume(ctx, queryFilter)
   680  	if err != nil {
   681  		t.Errorf("Failed to query volume. Error: %+v \n", err)
   682  		t.Fatal(err)
   683  	}
   684  	t.Logf("Successfully Queried Volumes after ExtendVolume. queryResult: %+v", pretty.Sprint(queryResult))
   685  	queryCapacity := queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb
   686  	if newCapacityInMb != queryCapacity {
   687  		t.Errorf("After extend volume %s, expected new volume size is %d, but actual volume size is %d.", extendVolumeId, newCapacityInMb, queryCapacity)
   688  	} else {
   689  		t.Logf("Volume extended sucessfully to the new size. Volume ID: %s New Size: %d", extendVolumeId, newCapacityInMb)
   690  	}
   691  
   692  	// Test UpdateVolumeMetadata
   693  	var updateSpecList []cnstypes.CnsVolumeMetadataUpdateSpec
   694  
   695  	var metadataList []cnstypes.BaseCnsEntityMetadata
   696  	newLabels := []vim25types.KeyValue{
   697  		{
   698  			Key:   "testLabel",
   699  			Value: "testValue",
   700  		},
   701  	}
   702  	pvmetadata := &cnstypes.CnsKubernetesEntityMetadata{
   703  		CnsEntityMetadata: cnstypes.CnsEntityMetadata{
   704  			DynamicData: vim25types.DynamicData{},
   705  			EntityName:  "pvc-53465372-5c12-4818-96f8-0ace4f4fd116",
   706  			Labels:      newLabels,
   707  			Delete:      false,
   708  			ClusterID:   "demo-cluster-id",
   709  		},
   710  		EntityType: string(cnstypes.CnsKubernetesEntityTypePV),
   711  		Namespace:  "",
   712  	}
   713  	metadataList = append(metadataList, cnstypes.BaseCnsEntityMetadata(pvmetadata))
   714  
   715  	pvcmetadata := &cnstypes.CnsKubernetesEntityMetadata{
   716  		CnsEntityMetadata: cnstypes.CnsEntityMetadata{
   717  			DynamicData: vim25types.DynamicData{},
   718  			EntityName:  "example-vanilla-block-pvc",
   719  			Labels:      newLabels,
   720  			Delete:      false,
   721  			ClusterID:   "demo-cluster-id",
   722  		},
   723  		EntityType: string(cnstypes.CnsKubernetesEntityTypePVC),
   724  		Namespace:  "default",
   725  		ReferredEntity: []cnstypes.CnsKubernetesEntityReference{
   726  			{
   727  				EntityType: string(cnstypes.CnsKubernetesEntityTypePV),
   728  				EntityName: "pvc-53465372-5c12-4818-96f8-0ace4f4fd116",
   729  				Namespace:  "",
   730  				ClusterID:  "demo-cluster-id",
   731  			},
   732  		},
   733  	}
   734  	metadataList = append(metadataList, cnstypes.BaseCnsEntityMetadata(pvcmetadata))
   735  
   736  	podmetadata := &cnstypes.CnsKubernetesEntityMetadata{
   737  		CnsEntityMetadata: cnstypes.CnsEntityMetadata{
   738  			DynamicData: vim25types.DynamicData{},
   739  			EntityName:  "example-pod",
   740  			Delete:      false,
   741  			ClusterID:   "demo-cluster-id",
   742  		},
   743  		EntityType: string(cnstypes.CnsKubernetesEntityTypePOD),
   744  		Namespace:  "default",
   745  		ReferredEntity: []cnstypes.CnsKubernetesEntityReference{
   746  			{
   747  				EntityType: string(cnstypes.CnsKubernetesEntityTypePVC),
   748  				EntityName: "example-vanilla-block-pvc",
   749  				Namespace:  "default",
   750  				ClusterID:  "demo-cluster-id",
   751  			},
   752  		},
   753  	}
   754  	metadataList = append(metadataList, cnstypes.BaseCnsEntityMetadata(podmetadata))
   755  
   756  	cnsVolumeMetadataUpdateSpec := cnstypes.CnsVolumeMetadataUpdateSpec{
   757  		VolumeId: cnstypes.CnsVolumeId{Id: volumeId},
   758  		Metadata: cnstypes.CnsVolumeMetadata{
   759  			DynamicData:           vim25types.DynamicData{},
   760  			ContainerCluster:      containerCluster,
   761  			EntityMetadata:        metadataList,
   762  			ContainerClusterArray: containerClusterArray,
   763  		},
   764  	}
   765  	t.Logf("Updating volume using the spec: %+v", cnsVolumeMetadataUpdateSpec)
   766  	updateSpecList = append(updateSpecList, cnsVolumeMetadataUpdateSpec)
   767  	updateTask, err := cnsClient.UpdateVolumeMetadata(ctx, updateSpecList)
   768  	if err != nil {
   769  		t.Errorf("Failed to update volume metadata. Error: %+v \n", err)
   770  		t.Fatal(err)
   771  	}
   772  	updateTaskInfo, err := GetTaskInfo(ctx, updateTask)
   773  	if err != nil {
   774  		t.Errorf("Failed to update volume metadata. Error: %+v \n", err)
   775  		t.Fatal(err)
   776  	}
   777  	updateTaskResult, err := GetTaskResult(ctx, updateTaskInfo)
   778  	if err != nil {
   779  		t.Errorf("Failed to update volume metadata. Error: %+v \n", err)
   780  		t.Fatal(err)
   781  	}
   782  	if updateTaskResult == nil {
   783  		t.Fatalf("Empty update task results")
   784  		t.FailNow()
   785  	}
   786  	updateVolumeOperationRes := updateTaskResult.GetCnsVolumeOperationResult()
   787  	if updateVolumeOperationRes.Fault != nil {
   788  		t.Fatalf("Failed to update volume metadata: fault=%+v", updateVolumeOperationRes.Fault)
   789  	} else {
   790  		t.Logf("Successfully updated volume metadata")
   791  	}
   792  
   793  	t.Logf("Calling QueryVolume using queryFilter: %+v", pretty.Sprint(queryFilter))
   794  	queryResult, err = cnsClient.QueryVolume(ctx, queryFilter)
   795  	if err != nil {
   796  		t.Errorf("Failed to query volume. Error: %+v \n", err)
   797  		t.Fatal(err)
   798  	}
   799  	t.Logf("Successfully Queried Volumes. queryResult: %+v", pretty.Sprint(queryResult))
   800  
   801  	// Test QueryAll
   802  	querySelection := cnstypes.CnsQuerySelection{
   803  		Names: []string{
   804  			string(cnstypes.CnsQuerySelectionName_VOLUME_NAME),
   805  			string(cnstypes.CnsQuerySelectionName_VOLUME_TYPE),
   806  			string(cnstypes.CnsQuerySelectionName_DATASTORE_URL),
   807  			string(cnstypes.CnsQuerySelectionName_POLICY_ID),
   808  			string(cnstypes.CnsQuerySelectionName_HEALTH_STATUS),
   809  			string(cnstypes.CnsQuerySelectionName_BACKING_OBJECT_DETAILS),
   810  			string(cnstypes.CnsQuerySelectionName_COMPLIANCE_STATUS),
   811  			string(cnstypes.CnsQuerySelectionName_DATASTORE_ACCESSIBILITY_STATUS),
   812  		},
   813  	}
   814  	queryResult, err = cnsClient.QueryAllVolume(ctx, cnstypes.CnsQueryFilter{}, querySelection)
   815  	if err != nil {
   816  		t.Errorf("Failed to query all volumes. Error: %+v \n", err)
   817  		t.Fatal(err)
   818  	}
   819  	t.Logf("Successfully Queried all Volumes. queryResult: %+v", pretty.Sprint(queryResult))
   820  
   821  	// Create a VM to test Attach Volume API.
   822  	virtualMachineConfigSpec := vim25types.VirtualMachineConfigSpec{
   823  		Name: "test-node-vm",
   824  		Files: &vim25types.VirtualMachineFileInfo{
   825  			VmPathName: "[" + datastore + "]",
   826  		},
   827  		NumCPUs:  1,
   828  		MemoryMB: 4,
   829  		DeviceChange: []vim25types.BaseVirtualDeviceConfigSpec{
   830  			&vim25types.VirtualDeviceConfigSpec{
   831  				Operation: vim25types.VirtualDeviceConfigSpecOperationAdd,
   832  				Device: &vim25types.ParaVirtualSCSIController{
   833  					VirtualSCSIController: vim25types.VirtualSCSIController{
   834  						SharedBus: vim25types.VirtualSCSISharingNoSharing,
   835  						VirtualController: vim25types.VirtualController{
   836  							BusNumber: 0,
   837  							VirtualDevice: vim25types.VirtualDevice{
   838  								Key: 1000,
   839  							},
   840  						},
   841  					},
   842  				},
   843  			},
   844  		},
   845  	}
   846  	defaultFolder, err := finder.DefaultFolder(ctx)
   847  	if err != nil {
   848  		t.Fatal(err)
   849  	}
   850  	var resourcePool *object.ResourcePool
   851  	if resourcePoolPath == "" {
   852  		resourcePool, err = finder.DefaultResourcePool(ctx)
   853  	} else {
   854  		resourcePool, err = finder.ResourcePool(ctx, resourcePoolPath)
   855  	}
   856  	if err != nil {
   857  		t.Errorf("Error occurred while getting DefaultResourcePool. err: %+v", err)
   858  		t.Fatal(err)
   859  	}
   860  	task, err := defaultFolder.CreateVM(ctx, virtualMachineConfigSpec, resourcePool, nil)
   861  	if err != nil {
   862  		t.Errorf("Failed to create VM. Error: %+v \n", err)
   863  		t.Fatal(err)
   864  	}
   865  
   866  	vmTaskInfo, err := task.WaitForResult(ctx, nil)
   867  	if err != nil {
   868  		t.Errorf("Error occurred while waiting for create VM task result. err: %+v", err)
   869  		t.Fatal(err)
   870  	}
   871  
   872  	vmRef := vmTaskInfo.Result.(object.Reference)
   873  	t.Logf("Node VM created sucessfully. vmRef: %+v", vmRef.Reference())
   874  
   875  	nodeVM := object.NewVirtualMachine(cnsClient.vim25Client, vmRef.Reference())
   876  	defer nodeVM.Destroy(ctx)
   877  
   878  	// Test AttachVolume API
   879  	var cnsVolumeAttachSpecList []cnstypes.CnsVolumeAttachDetachSpec
   880  	cnsVolumeAttachSpec := cnstypes.CnsVolumeAttachDetachSpec{
   881  		VolumeId: cnstypes.CnsVolumeId{
   882  			Id: volumeId,
   883  		},
   884  		Vm: nodeVM.Reference(),
   885  	}
   886  	cnsVolumeAttachSpecList = append(cnsVolumeAttachSpecList, cnsVolumeAttachSpec)
   887  	t.Logf("Attaching volume using the spec: %+v", cnsVolumeAttachSpec)
   888  	attachTask, err := cnsClient.AttachVolume(ctx, cnsVolumeAttachSpecList)
   889  	if err != nil {
   890  		t.Errorf("Failed to attach volume. Error: %+v \n", err)
   891  		t.Fatal(err)
   892  	}
   893  	attachTaskInfo, err := GetTaskInfo(ctx, attachTask)
   894  	if err != nil {
   895  		t.Errorf("Failed to attach volume. Error: %+v \n", err)
   896  		t.Fatal(err)
   897  	}
   898  	attachTaskResult, err := GetTaskResult(ctx, attachTaskInfo)
   899  	if err != nil {
   900  		t.Errorf("Failed to attach volume. Error: %+v \n", err)
   901  		t.Fatal(err)
   902  	}
   903  	if attachTaskResult == nil {
   904  		t.Fatalf("Empty attach task results")
   905  		t.FailNow()
   906  	}
   907  	attachVolumeOperationRes := attachTaskResult.GetCnsVolumeOperationResult()
   908  	if attachVolumeOperationRes.Fault != nil {
   909  		t.Fatalf("Failed to attach volume: fault=%+v", attachVolumeOperationRes.Fault)
   910  	}
   911  	diskUUID := any(attachTaskResult).(*cnstypes.CnsVolumeAttachResult).DiskUUID
   912  	t.Logf("Volume attached sucessfully. Disk UUID: %s", diskUUID)
   913  
   914  	// Re-Attach same volume to the same node and expect ResourceInUse fault
   915  	t.Logf("Re-Attaching volume using the spec: %+v", cnsVolumeAttachSpec)
   916  	attachTask, err = cnsClient.AttachVolume(ctx, cnsVolumeAttachSpecList)
   917  	if err != nil {
   918  		t.Errorf("Failed to attach volume. Error: %+v \n", err)
   919  		t.Fatal(err)
   920  	}
   921  	attachTaskInfo, err = GetTaskInfo(ctx, attachTask)
   922  	if err != nil {
   923  		t.Errorf("Failed to attach volume. Error: %+v \n", err)
   924  		t.Fatal(err)
   925  	}
   926  	attachTaskResult, err = GetTaskResult(ctx, attachTaskInfo)
   927  	if err != nil {
   928  		t.Errorf("Failed to attach volume. Error: %+v \n", err)
   929  		t.Fatal(err)
   930  	}
   931  	if attachTaskResult == nil {
   932  		t.Fatalf("Empty attach task results")
   933  		t.FailNow()
   934  	}
   935  	reAttachVolumeOperationRes := attachTaskResult.GetCnsVolumeOperationResult()
   936  	if reAttachVolumeOperationRes.Fault != nil {
   937  		t.Logf("reAttachVolumeOperationRes.Fault: %+v", pretty.Sprint(reAttachVolumeOperationRes.Fault))
   938  		_, ok := reAttachVolumeOperationRes.Fault.Fault.(*vim25types.ResourceInUse)
   939  		if !ok {
   940  			t.Fatalf("Fault is not ResourceInUse")
   941  		}
   942  	} else {
   943  		t.Fatalf("re-attach same volume should fail with ResourceInUse fault")
   944  	}
   945  
   946  	// Test DetachVolume API
   947  	var cnsVolumeDetachSpecList []cnstypes.CnsVolumeAttachDetachSpec
   948  	cnsVolumeDetachSpec := cnstypes.CnsVolumeAttachDetachSpec{
   949  		VolumeId: cnstypes.CnsVolumeId{
   950  			Id: volumeId,
   951  		},
   952  		Vm: nodeVM.Reference(),
   953  	}
   954  	cnsVolumeDetachSpecList = append(cnsVolumeDetachSpecList, cnsVolumeDetachSpec)
   955  	t.Logf("Detaching volume using the spec: %+v", cnsVolumeDetachSpec)
   956  	detachTask, err := cnsClient.DetachVolume(ctx, cnsVolumeDetachSpecList)
   957  	if err != nil {
   958  		t.Errorf("Failed to detach volume. Error: %+v \n", err)
   959  		t.Fatal(err)
   960  	}
   961  	detachTaskInfo, err := GetTaskInfo(ctx, detachTask)
   962  	if err != nil {
   963  		t.Errorf("Failed to detach volume. Error: %+v \n", err)
   964  		t.Fatal(err)
   965  	}
   966  	detachTaskResult, err := GetTaskResult(ctx, detachTaskInfo)
   967  	if err != nil {
   968  		t.Errorf("Failed to detach volume. Error: %+v \n", err)
   969  		t.Fatal(err)
   970  	}
   971  	if detachTaskResult == nil {
   972  		t.Fatalf("Empty detach task results")
   973  		t.FailNow()
   974  	}
   975  	detachVolumeOperationRes := detachTaskResult.GetCnsVolumeOperationResult()
   976  	if detachVolumeOperationRes.Fault != nil {
   977  		t.Fatalf("Failed to detach volume: fault=%+v", detachVolumeOperationRes.Fault)
   978  	}
   979  	t.Logf("Volume detached sucessfully")
   980  
   981  	// Test QueryVolumeAsync API only for vSphere version 7.0.3 onwards
   982  	if isvSphereVersion70U3orAbove(ctx, c.ServiceContent.About) {
   983  		queryVolumeAsyncTask, err := cnsClient.QueryVolumeAsync(ctx, queryFilter, nil)
   984  		if err != nil {
   985  			t.Errorf("Failed to query volumes with QueryVolumeAsync. Error: %+v \n", err)
   986  		}
   987  		queryVolumeAsyncTaskInfo, err := GetTaskInfo(ctx, queryVolumeAsyncTask)
   988  		if err != nil {
   989  			t.Errorf("Failed to query volumes with QueryVolumeAsync. Error: %+v \n", err)
   990  		}
   991  		queryVolumeAsyncTaskResults, err := GetTaskResultArray(ctx, queryVolumeAsyncTaskInfo)
   992  		if err != nil {
   993  			t.Errorf("Failed to query volumes with QueryVolumeAsync. Error: %+v \n", err)
   994  		}
   995  		for _, queryVolumeAsyncTaskResult := range queryVolumeAsyncTaskResults {
   996  			queryVolumeAsyncOperationRes := queryVolumeAsyncTaskResult.GetCnsVolumeOperationResult()
   997  			if queryVolumeAsyncOperationRes.Fault != nil {
   998  				t.Fatalf("Failed to query volumes with QueryVolumeAsync. fault=%+v", queryVolumeAsyncOperationRes.Fault)
   999  			}
  1000  			t.Logf("Successfully queried Volume using queryAsync API. queryVolumeAsyncTaskResult: %+v", pretty.Sprint(queryVolumeAsyncTaskResult))
  1001  		}
  1002  	}
  1003  
  1004  	// Test DeleteVolume API
  1005  	t.Logf("Deleting volume: %+v", volumeIDList)
  1006  	deleteTask, err := cnsClient.DeleteVolume(ctx, volumeIDList, true)
  1007  	if err != nil {
  1008  		t.Errorf("Failed to delete volume. Error: %+v \n", err)
  1009  		t.Fatal(err)
  1010  	}
  1011  	deleteTaskInfo, err := GetTaskInfo(ctx, deleteTask)
  1012  	if err != nil {
  1013  		t.Errorf("Failed to delete volume. Error: %+v \n", err)
  1014  		t.Fatal(err)
  1015  	}
  1016  	deleteTaskResult, err := GetTaskResult(ctx, deleteTaskInfo)
  1017  	if err != nil {
  1018  		t.Errorf("Failed to delete volume. Error: %+v \n", err)
  1019  		t.Fatal(err)
  1020  	}
  1021  	if deleteTaskResult == nil {
  1022  		t.Fatalf("Empty delete task results")
  1023  		t.FailNow()
  1024  	}
  1025  	deleteVolumeOperationRes := deleteTaskResult.GetCnsVolumeOperationResult()
  1026  	if deleteVolumeOperationRes.Fault != nil {
  1027  		t.Fatalf("Failed to delete volume: fault=%+v", deleteVolumeOperationRes.Fault)
  1028  	}
  1029  	t.Logf("Volume: %q deleted sucessfully", volumeId)
  1030  
  1031  	if run_fileshare_tests == "true" && cnsClient.Version != ReleaseVSAN67u3 {
  1032  		// Test creating vSAN file-share Volume
  1033  		var cnsFileVolumeCreateSpecList []cnstypes.CnsVolumeCreateSpec
  1034  		vSANFileCreateSpec := &cnstypes.CnsVSANFileCreateSpec{
  1035  			SoftQuotaInMb: 5120,
  1036  			Permission: []vsanfstypes.VsanFileShareNetPermission{
  1037  				{
  1038  					Ips:         "*",
  1039  					Permissions: vsanfstypes.VsanFileShareAccessTypeREAD_WRITE,
  1040  					AllowRoot:   true,
  1041  				},
  1042  			},
  1043  		}
  1044  
  1045  		cnsFileVolumeCreateSpec := cnstypes.CnsVolumeCreateSpec{
  1046  			Name:       "pvc-file-share-volume",
  1047  			VolumeType: string(cnstypes.CnsVolumeTypeFile),
  1048  			Datastores: dsList,
  1049  			Metadata: cnstypes.CnsVolumeMetadata{
  1050  				ContainerCluster:      containerCluster,
  1051  				ContainerClusterArray: containerClusterArray,
  1052  			},
  1053  			BackingObjectDetails: &cnstypes.CnsVsanFileShareBackingDetails{
  1054  				CnsFileBackingDetails: cnstypes.CnsFileBackingDetails{
  1055  					CnsBackingObjectDetails: cnstypes.CnsBackingObjectDetails{
  1056  						CapacityInMb: 5120,
  1057  					},
  1058  				},
  1059  			},
  1060  			CreateSpec: vSANFileCreateSpec,
  1061  		}
  1062  		cnsFileVolumeCreateSpecList = append(cnsFileVolumeCreateSpecList, cnsFileVolumeCreateSpec)
  1063  		t.Logf("Creating CNS file volume using the spec: %+v", cnsFileVolumeCreateSpec)
  1064  		createTask, err = cnsClient.CreateVolume(ctx, cnsFileVolumeCreateSpecList)
  1065  		if err != nil {
  1066  			t.Errorf("Failed to create vsan fileshare volume. Error: %+v \n", err)
  1067  			t.Fatal(err)
  1068  		}
  1069  		createTaskInfo, err = GetTaskInfo(ctx, createTask)
  1070  		if err != nil {
  1071  			t.Errorf("Failed to create Fileshare volume. Error: %+v \n", err)
  1072  			t.Fatal(err)
  1073  		}
  1074  		createTaskResult, err = GetTaskResult(ctx, createTaskInfo)
  1075  		if err != nil {
  1076  			t.Errorf("Failed to create Fileshare volume. Error: %+v \n", err)
  1077  			t.Fatal(err)
  1078  		}
  1079  		if createTaskResult == nil {
  1080  			t.Fatalf("Empty create task results")
  1081  			t.FailNow()
  1082  		}
  1083  		createVolumeOperationRes = createTaskResult.GetCnsVolumeOperationResult()
  1084  		if createVolumeOperationRes.Fault != nil {
  1085  			t.Fatalf("Failed to create Fileshare volume: fault=%+v", createVolumeOperationRes.Fault)
  1086  		}
  1087  		filevolumeId := createVolumeOperationRes.VolumeId.Id
  1088  		t.Logf("Fileshare volume created sucessfully. filevolumeId: %s", filevolumeId)
  1089  
  1090  		// Test QueryVolume API
  1091  		volumeIDList = []cnstypes.CnsVolumeId{{Id: filevolumeId}}
  1092  		queryFilter.VolumeIds = volumeIDList
  1093  		t.Logf("Calling QueryVolume using queryFilter: %+v", queryFilter)
  1094  		queryResult, err = cnsClient.QueryVolume(ctx, queryFilter)
  1095  		if err != nil {
  1096  			t.Errorf("Failed to query volume. Error: %+v \n", err)
  1097  			t.Fatal(err)
  1098  		}
  1099  		t.Logf("Successfully Queried Volumes. queryResult: %+v", queryResult)
  1100  		fileBackingInfo := queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsVsanFileShareBackingDetails)
  1101  		t.Logf("File Share Name: %s with accessPoints: %+v", fileBackingInfo.Name, fileBackingInfo.AccessPoints)
  1102  
  1103  		// Test add read-only permissions using Configure ACLs
  1104  		netPerms := make([]vsanfstypes.VsanFileShareNetPermission, 0)
  1105  		netPerms = append(netPerms, vsanfstypes.VsanFileShareNetPermission{
  1106  			Ips:         "192.168.124.2",
  1107  			Permissions: "READ_ONLY",
  1108  		})
  1109  
  1110  		vSanNFSACLEntry := make([]cnstypes.CnsNFSAccessControlSpec, 0)
  1111  		vSanNFSACLEntry = append(vSanNFSACLEntry, cnstypes.CnsNFSAccessControlSpec{
  1112  			Permission: netPerms,
  1113  		})
  1114  
  1115  		volumeID := cnstypes.CnsVolumeId{
  1116  			Id: filevolumeId,
  1117  		}
  1118  		aclSpec := cnstypes.CnsVolumeACLConfigureSpec{
  1119  			VolumeId:              volumeID,
  1120  			AccessControlSpecList: vSanNFSACLEntry,
  1121  		}
  1122  		t.Logf("Invoking ConfigureVolumeACLs using the spec: %+v", pretty.Sprint(aclSpec))
  1123  		aclTask, err := cnsClient.ConfigureVolumeACLs(ctx, aclSpec)
  1124  		if err != nil {
  1125  			t.Errorf("Failed to configure VolumeACLs. Error: %+v", err)
  1126  			t.Fatal(err)
  1127  		}
  1128  		aclTaskInfo, err := GetTaskInfo(ctx, aclTask)
  1129  		if err != nil {
  1130  			t.Errorf("Failed to configure VolumeACLs. Error: %+v", err)
  1131  			t.Fatal(err)
  1132  		}
  1133  		aclTaskResult, err := GetTaskResult(ctx, aclTaskInfo)
  1134  		if err != nil {
  1135  			t.Errorf("Failed to configure VolumeACLs. Error: %+v", err)
  1136  			t.Fatal(err)
  1137  		}
  1138  		if aclTaskResult == nil {
  1139  			t.Fatalf("Empty configure VolumeACLs task results")
  1140  			t.FailNow()
  1141  		}
  1142  
  1143  		// Test to revoke all permissions using Configure ACLs
  1144  		netPerms = make([]vsanfstypes.VsanFileShareNetPermission, 0)
  1145  		netPerms = append(netPerms, vsanfstypes.VsanFileShareNetPermission{
  1146  			Ips:         "192.168.124.2",
  1147  			Permissions: "READ_ONLY",
  1148  		})
  1149  
  1150  		vSanNFSACLEntry = make([]cnstypes.CnsNFSAccessControlSpec, 0)
  1151  		vSanNFSACLEntry = append(vSanNFSACLEntry, cnstypes.CnsNFSAccessControlSpec{
  1152  			Permission: netPerms,
  1153  			Delete:     true,
  1154  		})
  1155  
  1156  		aclSpec = cnstypes.CnsVolumeACLConfigureSpec{
  1157  			VolumeId:              volumeID,
  1158  			AccessControlSpecList: vSanNFSACLEntry,
  1159  		}
  1160  		t.Logf("Invoking ConfigureVolumeACLs using the spec: %+v", pretty.Sprint(aclSpec))
  1161  		aclTask, err = cnsClient.ConfigureVolumeACLs(ctx, aclSpec)
  1162  		if err != nil {
  1163  			t.Errorf("Failed to configure VolumeACLs. Error: %+v", err)
  1164  			t.Fatal(err)
  1165  		}
  1166  		aclTaskInfo, err = GetTaskInfo(ctx, aclTask)
  1167  		if err != nil {
  1168  			t.Errorf("Failed to configure VolumeACLs. Error: %+v", err)
  1169  			t.Fatal(err)
  1170  		}
  1171  		aclTaskResult, err = GetTaskResult(ctx, aclTaskInfo)
  1172  		if err != nil {
  1173  			t.Errorf("Failed to configure VolumeACLs. Error: %+v", err)
  1174  			t.Fatal(err)
  1175  		}
  1176  		if aclTaskResult == nil {
  1177  			t.Fatalf("Empty configure VolumeACLs task results")
  1178  			t.FailNow()
  1179  		}
  1180  
  1181  		// Test Deleting vSAN file-share Volume
  1182  		var fileVolumeIDList []cnstypes.CnsVolumeId
  1183  		fileVolumeIDList = append(fileVolumeIDList, cnstypes.CnsVolumeId{Id: filevolumeId})
  1184  		t.Logf("Deleting fileshare volume: %+v", fileVolumeIDList)
  1185  		deleteTask, err = cnsClient.DeleteVolume(ctx, fileVolumeIDList, true)
  1186  		if err != nil {
  1187  			t.Errorf("Failed to delete fileshare volume. Error: %+v \n", err)
  1188  			t.Fatal(err)
  1189  		}
  1190  		deleteTaskInfo, err = GetTaskInfo(ctx, deleteTask)
  1191  		if err != nil {
  1192  			t.Errorf("Failed to delete fileshare volume. Error: %+v \n", err)
  1193  			t.Fatal(err)
  1194  		}
  1195  		deleteTaskResult, err = GetTaskResult(ctx, deleteTaskInfo)
  1196  		if err != nil {
  1197  			t.Errorf("Failed to delete fileshare volume. Error: %+v \n", err)
  1198  			t.Fatal(err)
  1199  		}
  1200  		if deleteTaskResult == nil {
  1201  			t.Fatalf("Empty delete task results")
  1202  			t.FailNow()
  1203  		}
  1204  		deleteVolumeOperationRes = deleteTaskResult.GetCnsVolumeOperationResult()
  1205  		if deleteVolumeOperationRes.Fault != nil {
  1206  			t.Fatalf("Failed to delete fileshare volume: fault=%+v", deleteVolumeOperationRes.Fault)
  1207  		}
  1208  		t.Logf("fileshare volume:%q deleted sucessfully", filevolumeId)
  1209  	}
  1210  	if backingDiskURLPath != "" && cnsClient.Version != ReleaseVSAN67u3 && cnsClient.Version != ReleaseVSAN70 {
  1211  		// Test CreateVolume API with existing VMDK
  1212  		var cnsVolumeCreateSpecList []cnstypes.CnsVolumeCreateSpec
  1213  		cnsVolumeCreateSpec := cnstypes.CnsVolumeCreateSpec{
  1214  			Name:       "pvc-901e87eb-c2bd-11e9-806f-005056a0c9a0",
  1215  			VolumeType: string(cnstypes.CnsVolumeTypeBlock),
  1216  			Metadata: cnstypes.CnsVolumeMetadata{
  1217  				ContainerCluster: containerCluster,
  1218  			},
  1219  			BackingObjectDetails: &cnstypes.CnsBlockBackingDetails{
  1220  				BackingDiskUrlPath: backingDiskURLPath,
  1221  			},
  1222  		}
  1223  		cnsVolumeCreateSpecList = append(cnsVolumeCreateSpecList, cnsVolumeCreateSpec)
  1224  		t.Logf("Creating volume using the spec: %+v", pretty.Sprint(cnsVolumeCreateSpec))
  1225  		createTask, err := cnsClient.CreateVolume(ctx, cnsVolumeCreateSpecList)
  1226  		if err != nil {
  1227  			t.Errorf("Failed to create volume. Error: %+v \n", err)
  1228  			t.Fatal(err)
  1229  		}
  1230  		createTaskInfo, err := GetTaskInfo(ctx, createTask)
  1231  		if err != nil {
  1232  			t.Errorf("Failed to create volume. Error: %+v \n", err)
  1233  			t.Fatal(err)
  1234  		}
  1235  		createTaskResult, err := GetTaskResult(ctx, createTaskInfo)
  1236  		if err != nil {
  1237  			t.Errorf("Failed to create volume. Error: %+v \n", err)
  1238  			t.Fatal(err)
  1239  		}
  1240  		if createTaskResult == nil {
  1241  			t.Fatalf("Empty create task results")
  1242  			t.FailNow()
  1243  		}
  1244  		createVolumeOperationRes := createTaskResult.GetCnsVolumeOperationResult()
  1245  		var volumeID string
  1246  		if createVolumeOperationRes.Fault != nil {
  1247  			t.Logf("Failed to create volume: fault=%+v", createVolumeOperationRes.Fault)
  1248  			fault, ok := createVolumeOperationRes.Fault.Fault.(cnstypes.CnsAlreadyRegisteredFault)
  1249  			if !ok {
  1250  				t.Fatalf("Fault is not CnsAlreadyRegisteredFault")
  1251  			} else {
  1252  				t.Logf("Fault is CnsAlreadyRegisteredFault. backingDiskURLPath: %s is already registered", backingDiskURLPath)
  1253  				volumeID = fault.VolumeId.Id
  1254  			}
  1255  		} else {
  1256  			volumeID = createVolumeOperationRes.VolumeId.Id
  1257  			t.Logf("Volume created sucessfully with backingDiskURLPath: %s. volumeId: %s", backingDiskURLPath, volumeID)
  1258  
  1259  			// Test re creating volume using BACKING_DISK_URL_PATH
  1260  			var reCreateCnsVolumeCreateSpecList []cnstypes.CnsVolumeCreateSpec
  1261  			reCreateCnsVolumeCreateSpec := cnstypes.CnsVolumeCreateSpec{
  1262  				Name:       "pvc-901e87eb-c2bd-11e9-806f-005056a0c9a0",
  1263  				VolumeType: string(cnstypes.CnsVolumeTypeBlock),
  1264  				Metadata: cnstypes.CnsVolumeMetadata{
  1265  					ContainerCluster: containerCluster,
  1266  				},
  1267  				BackingObjectDetails: &cnstypes.CnsBlockBackingDetails{
  1268  					BackingDiskUrlPath: backingDiskURLPath,
  1269  				},
  1270  			}
  1271  
  1272  			reCreateCnsVolumeCreateSpecList = append(reCreateCnsVolumeCreateSpecList, reCreateCnsVolumeCreateSpec)
  1273  			t.Logf("Creating volume using the spec: %+v", pretty.Sprint(reCreateCnsVolumeCreateSpec))
  1274  			recreateTask, err := cnsClient.CreateVolume(ctx, reCreateCnsVolumeCreateSpecList)
  1275  			if err != nil {
  1276  				t.Errorf("Failed to create volume. Error: %+v \n", err)
  1277  				t.Fatal(err)
  1278  			}
  1279  			reCreateTaskInfo, err := GetTaskInfo(ctx, recreateTask)
  1280  			if err != nil {
  1281  				t.Errorf("Failed to create volume. Error: %+v \n", err)
  1282  				t.Fatal(err)
  1283  			}
  1284  			reCreateTaskResult, err := GetTaskResult(ctx, reCreateTaskInfo)
  1285  			if err != nil {
  1286  				t.Errorf("Failed to create volume. Error: %+v \n", err)
  1287  				t.Fatal(err)
  1288  			}
  1289  			if reCreateTaskResult == nil {
  1290  				t.Fatalf("Empty create task results")
  1291  				t.FailNow()
  1292  			}
  1293  			reCreateVolumeOperationRes := reCreateTaskResult.GetCnsVolumeOperationResult()
  1294  			t.Logf("reCreateVolumeOperationRes.: %+v", pretty.Sprint(reCreateVolumeOperationRes))
  1295  			if reCreateVolumeOperationRes.Fault != nil {
  1296  				t.Logf("Failed to create volume: fault=%+v", reCreateVolumeOperationRes.Fault)
  1297  				_, ok := reCreateVolumeOperationRes.Fault.Fault.(cnstypes.CnsAlreadyRegisteredFault)
  1298  				if !ok {
  1299  					t.Fatalf("Fault is not CnsAlreadyRegisteredFault")
  1300  				} else {
  1301  					t.Logf("Fault is CnsAlreadyRegisteredFault. backingDiskURLPath: %q is already registered", backingDiskURLPath)
  1302  				}
  1303  			}
  1304  		}
  1305  
  1306  		// Test QueryVolume API
  1307  		var queryFilter cnstypes.CnsQueryFilter
  1308  		var volumeIDList []cnstypes.CnsVolumeId
  1309  		volumeIDList = append(volumeIDList, cnstypes.CnsVolumeId{Id: volumeID})
  1310  		queryFilter.VolumeIds = volumeIDList
  1311  		t.Logf("Calling QueryVolume using queryFilter: %+v", pretty.Sprint(queryFilter))
  1312  		queryResult, err := cnsClient.QueryVolume(ctx, queryFilter)
  1313  		if err != nil {
  1314  			t.Errorf("Failed to query volume. Error: %+v \n", err)
  1315  			t.Fatal(err)
  1316  		}
  1317  		t.Logf("Successfully Queried Volumes. queryResult: %+v", pretty.Sprint(queryResult))
  1318  
  1319  		t.Logf("Deleting CNS volume created above using BACKING_DISK_URL_PATH: %s with volume: %+v", backingDiskURLPath, volumeIDList)
  1320  		deleteTask, err = cnsClient.DeleteVolume(ctx, volumeIDList, true)
  1321  		if err != nil {
  1322  			t.Errorf("Failed to delete volume. Error: %+v \n", err)
  1323  			t.Fatal(err)
  1324  		}
  1325  		deleteTaskInfo, err = GetTaskInfo(ctx, deleteTask)
  1326  		if err != nil {
  1327  			t.Errorf("Failed to delete volume. Error: %+v \n", err)
  1328  			t.Fatal(err)
  1329  		}
  1330  		deleteTaskResult, err = GetTaskResult(ctx, deleteTaskInfo)
  1331  		if err != nil {
  1332  			t.Errorf("Failed to delete volume. Error: %+v \n", err)
  1333  			t.Fatal(err)
  1334  		}
  1335  		if deleteTaskResult == nil {
  1336  			t.Fatalf("Empty delete task results")
  1337  			t.FailNow()
  1338  		}
  1339  		deleteVolumeOperationRes = deleteTaskResult.GetCnsVolumeOperationResult()
  1340  		if deleteVolumeOperationRes.Fault != nil {
  1341  			t.Fatalf("Failed to delete volume: fault=%+v", deleteVolumeOperationRes.Fault)
  1342  		}
  1343  		t.Logf("volume:%q deleted sucessfully", volumeID)
  1344  	}
  1345  
  1346  	// Test CnsReconfigVolumePolicy API
  1347  	if spbmPolicyId4Reconfig != "" {
  1348  		var cnsVolumeCreateSpecList []cnstypes.CnsVolumeCreateSpec
  1349  		cnsVolumeCreateSpec := cnstypes.CnsVolumeCreateSpec{
  1350  			Name:       "pvc-901e87eb-c2bd-11e9-806f-005056a0c9a0-1",
  1351  			VolumeType: string(cnstypes.CnsVolumeTypeBlock),
  1352  			Datastores: dsList,
  1353  			Metadata: cnstypes.CnsVolumeMetadata{
  1354  				ContainerCluster: containerCluster,
  1355  			},
  1356  			BackingObjectDetails: &cnstypes.CnsBlockBackingDetails{
  1357  				CnsBackingObjectDetails: cnstypes.CnsBackingObjectDetails{
  1358  					CapacityInMb: 5120,
  1359  				},
  1360  			},
  1361  		}
  1362  		cnsVolumeCreateSpecList = append(cnsVolumeCreateSpecList, cnsVolumeCreateSpec)
  1363  		t.Logf("Creating volume using the spec: %+v", pretty.Sprint(cnsVolumeCreateSpecList))
  1364  		createTask, err = cnsClient.CreateVolume(ctx, cnsVolumeCreateSpecList)
  1365  		if err != nil {
  1366  			t.Errorf("Failed to create volume. Error: %+v \n", err)
  1367  			t.Fatal(err)
  1368  		}
  1369  		createTaskInfo, err = GetTaskInfo(ctx, createTask)
  1370  		if err != nil {
  1371  			t.Errorf("Failed to create volume. Error: %+v \n", err)
  1372  			t.Fatal(err)
  1373  		}
  1374  		createTaskResult, err = GetTaskResult(ctx, createTaskInfo)
  1375  		if err != nil {
  1376  			t.Errorf("Failed to create volume. Error: %+v \n", err)
  1377  			t.Fatal(err)
  1378  		}
  1379  		if createTaskResult == nil {
  1380  			t.Fatalf("Empty create task results")
  1381  			t.FailNow()
  1382  		}
  1383  		createVolumeOperationRes = createTaskResult.GetCnsVolumeOperationResult()
  1384  		if createVolumeOperationRes.Fault != nil {
  1385  			t.Fatalf("Failed to create volume: fault=%+v", createVolumeOperationRes.Fault)
  1386  		}
  1387  		volumeId = createVolumeOperationRes.VolumeId.Id
  1388  		volumeCreateResult = (createTaskResult).(*cnstypes.CnsVolumeCreateResult)
  1389  		t.Logf("volumeCreateResult %+v", volumeCreateResult)
  1390  		t.Logf("Volume created sucessfully. volumeId: %s", volumeId)
  1391  
  1392  		t.Logf("Calling reconfigpolicy on volume %v with policy %+v \n", volumeId, spbmPolicyId4Reconfig)
  1393  		reconfigSpecs := []cnstypes.CnsVolumePolicyReconfigSpec{
  1394  			{
  1395  				VolumeId: createVolumeOperationRes.VolumeId,
  1396  				Profile: []vim25types.BaseVirtualMachineProfileSpec{
  1397  					&vim25types.VirtualMachineDefinedProfileSpec{
  1398  						ProfileId: spbmPolicyId4Reconfig,
  1399  					},
  1400  				},
  1401  			},
  1402  		}
  1403  		reconfigTask, err := cnsClient.ReconfigVolumePolicy(ctx, reconfigSpecs)
  1404  		if err != nil {
  1405  			t.Errorf("Failed to reconfig policy %v on volume %v. Error: %+v \n", spbmPolicyId4Reconfig, volumeId, err)
  1406  			t.Fatal(err)
  1407  		}
  1408  		reconfigTaskInfo, err := GetTaskInfo(ctx, reconfigTask)
  1409  		if err != nil {
  1410  			t.Errorf("Failed to reconfig volume. Error: %+v \n", err)
  1411  			t.Fatal(err)
  1412  		}
  1413  		reconfigTaskResult, err := GetTaskResult(ctx, reconfigTaskInfo)
  1414  		if err != nil {
  1415  			t.Errorf("Failed to reconfig volume. Error: %+v \n", err)
  1416  			t.Fatal(err)
  1417  		}
  1418  		if reconfigTaskResult == nil {
  1419  			t.Fatalf("Empty reconfig task results")
  1420  			t.FailNow()
  1421  		}
  1422  		reconfigVolumeOperationRes := reconfigTaskResult.GetCnsVolumeOperationResult()
  1423  		if reconfigVolumeOperationRes.Fault != nil {
  1424  			t.Fatalf("Failed to reconfig volume %v with policy %v: fault=%+v",
  1425  				volumeId, spbmPolicyId4Reconfig, reconfigVolumeOperationRes.Fault)
  1426  		}
  1427  		t.Logf("reconfigpolicy on volume %v with policy %+v successful\n", volumeId, spbmPolicyId4Reconfig)
  1428  	}
  1429  
  1430  	// Test CnsSyncDatastore API
  1431  	t.Logf("Calling syncDatastore on %v ...\n", dsUrl)
  1432  	syncDatastoreTask, err := cnsClient.SyncDatastore(ctx, dsUrl, false)
  1433  	if err != nil {
  1434  		t.Errorf("Failed to sync datastore %v. Error: %+v \n", dsUrl, err)
  1435  		t.Fatal(err)
  1436  	}
  1437  	syncDatastoreTaskInfo, err := GetTaskInfo(ctx, syncDatastoreTask)
  1438  	if err != nil {
  1439  		t.Errorf("Failed to get sync datastore taskInfo. Error: %+v \n", err)
  1440  		t.Fatal(err)
  1441  	}
  1442  	if syncDatastoreTaskInfo.State != vim25types.TaskInfoStateSuccess {
  1443  		t.Errorf("Failed to sync datastore. Error: %+v \n", syncDatastoreTaskInfo.Error)
  1444  		t.Fatalf("%+v", syncDatastoreTaskInfo.Error)
  1445  	}
  1446  	t.Logf("syncDatastore on %v successful\n", dsUrl)
  1447  
  1448  	t.Logf("Calling syncDatastore on %v with fullsync...\n", dsUrl)
  1449  	syncDatastoreTask, err = cnsClient.SyncDatastore(ctx, dsUrl, true)
  1450  	if err != nil {
  1451  		t.Errorf("Failed to sync datastore %v with full sync. Error: %+v \n", dsUrl, err)
  1452  		t.Fatal(err)
  1453  	}
  1454  	syncDatastoreTaskInfo, err = GetTaskInfo(ctx, syncDatastoreTask)
  1455  	if err != nil {
  1456  		t.Errorf("Failed to get sync datastore taskInfo with full sync. Error: %+v \n", err)
  1457  		t.Fatal(err)
  1458  	}
  1459  	if syncDatastoreTaskInfo.State != vim25types.TaskInfoStateSuccess {
  1460  		t.Errorf("Failed to sync datastore with full sync. Error: %+v \n", syncDatastoreTaskInfo.Error)
  1461  		t.Fatalf("%+v", syncDatastoreTaskInfo.Error)
  1462  	}
  1463  	t.Logf("syncDatastore on %v with full sync successful\n", dsUrl)
  1464  }
  1465  
  1466  // isvSphereVersion70U3orAbove checks if specified version is 7.0 Update 3 or higher
  1467  // The method takes aboutInfo{} as input which contains details about
  1468  // VC version, build number and so on.
  1469  // If the version is 7.0 Update 3 or higher, the method returns true, else returns false
  1470  // along with appropriate errors during failure cases
  1471  func isvSphereVersion70U3orAbove(ctx context.Context, aboutInfo vim25types.AboutInfo) bool {
  1472  	items := strings.Split(aboutInfo.Version, ".")
  1473  	version := strings.Join(items[:], "")
  1474  	// Convert version string to string, Ex: "7.0.3" becomes 703, "7.0.3.1" becomes 703
  1475  	if len(version) >= 3 {
  1476  		vSphereVersionInt, err := strconv.Atoi(version[0:3])
  1477  		if err != nil {
  1478  			return false
  1479  		}
  1480  		// Check if the current vSphere version is 7.0.3 or higher
  1481  		if vSphereVersionInt >= VSphere70u3VersionInt {
  1482  			return true
  1483  		}
  1484  	}
  1485  	// For all other versions
  1486  	return false
  1487  }
  1488  
  1489  // isvSphereVersion80U3orAbove checks if specified version is 8.0 Update 3 or higher
  1490  // The method takes aboutInfo{} as input which contains details about
  1491  // VC version, build number and so on.
  1492  // If the version is 8.0 Update 3 or higher, the method returns true, else returns false
  1493  // along with appropriate errors during failure cases
  1494  func isvSphereVersion80U3orAbove(ctx context.Context, aboutInfo vim25types.AboutInfo) bool {
  1495  	items := strings.Split(aboutInfo.Version, ".")
  1496  	version := strings.Join(items[:], "")
  1497  	// Convert version string to string, Ex: "8.0.3" becomes 803, "8.0.3.1" becomes 703
  1498  	if len(version) >= 3 {
  1499  		vSphereVersionInt, err := strconv.Atoi(version[0:3])
  1500  		if err != nil {
  1501  			return false
  1502  		}
  1503  		// Check if the current vSphere version is 8.0.3 or higher
  1504  		if vSphereVersionInt >= VSphere80u3VersionInt {
  1505  			return true
  1506  		}
  1507  	}
  1508  	// For all other versions
  1509  	return false
  1510  }