github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/plugins/csi/client_test.go (about)

     1  package csi
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"path/filepath"
     8  	"testing"
     9  	"time"
    10  
    11  	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"
    12  	"github.com/golang/protobuf/ptypes/wrappers"
    13  	"github.com/hashicorp/nomad/ci"
    14  	"github.com/hashicorp/nomad/nomad/structs"
    15  	fake "github.com/hashicorp/nomad/plugins/csi/testing"
    16  	"github.com/stretchr/testify/require"
    17  	"google.golang.org/grpc"
    18  	"google.golang.org/grpc/codes"
    19  	"google.golang.org/grpc/status"
    20  	"google.golang.org/protobuf/types/known/timestamppb"
    21  )
    22  
    23  func newTestClient(t *testing.T) (*fake.IdentityClient, *fake.ControllerClient, *fake.NodeClient, CSIPlugin) {
    24  	ic := fake.NewIdentityClient()
    25  	cc := fake.NewControllerClient()
    26  	nc := fake.NewNodeClient()
    27  
    28  	// we've set this as non-blocking so it won't connect to the
    29  	// socket unless a RPC is invoked
    30  	conn, err := grpc.DialContext(context.Background(),
    31  		filepath.Join(t.TempDir(), "csi.sock"), grpc.WithInsecure())
    32  	if err != nil {
    33  		t.Errorf("failed: %v", err)
    34  	}
    35  
    36  	client := &client{
    37  		conn:             conn,
    38  		identityClient:   ic,
    39  		controllerClient: cc,
    40  		nodeClient:       nc,
    41  	}
    42  
    43  	return ic, cc, nc, client
    44  }
    45  
    46  func TestClient_RPC_PluginProbe(t *testing.T) {
    47  	ci.Parallel(t)
    48  
    49  	cases := []struct {
    50  		Name             string
    51  		ResponseErr      error
    52  		ProbeResponse    *csipbv1.ProbeResponse
    53  		ExpectedResponse bool
    54  		ExpectedErr      error
    55  	}{
    56  		{
    57  			Name:        "handles underlying grpc errors",
    58  			ResponseErr: fmt.Errorf("some grpc error"),
    59  			ExpectedErr: fmt.Errorf("some grpc error"),
    60  		},
    61  		{
    62  			Name: "returns false for ready when the provider returns false",
    63  			ProbeResponse: &csipbv1.ProbeResponse{
    64  				Ready: &wrappers.BoolValue{Value: false},
    65  			},
    66  			ExpectedResponse: false,
    67  		},
    68  		{
    69  			Name: "returns true for ready when the provider returns true",
    70  			ProbeResponse: &csipbv1.ProbeResponse{
    71  				Ready: &wrappers.BoolValue{Value: true},
    72  			},
    73  			ExpectedResponse: true,
    74  		},
    75  		{
    76  			/* When a SP does not return a ready value, a CO MAY treat this as ready.
    77  			   We do so because example plugins rely on this behaviour. We may
    78  				 re-evaluate this decision in the future. */
    79  			Name: "returns true for ready when the provider returns a nil wrapper",
    80  			ProbeResponse: &csipbv1.ProbeResponse{
    81  				Ready: nil,
    82  			},
    83  			ExpectedResponse: true,
    84  		},
    85  	}
    86  
    87  	for _, tc := range cases {
    88  		t.Run(tc.Name, func(t *testing.T) {
    89  			ic, _, _, client := newTestClient(t)
    90  			defer client.Close()
    91  
    92  			ic.NextErr = tc.ResponseErr
    93  			ic.NextPluginProbe = tc.ProbeResponse
    94  
    95  			resp, err := client.PluginProbe(context.TODO())
    96  			if tc.ExpectedErr != nil {
    97  				require.EqualError(t, err, tc.ExpectedErr.Error())
    98  			}
    99  
   100  			require.Equal(t, tc.ExpectedResponse, resp)
   101  		})
   102  	}
   103  
   104  }
   105  
   106  func TestClient_RPC_PluginInfo(t *testing.T) {
   107  	ci.Parallel(t)
   108  
   109  	cases := []struct {
   110  		Name                    string
   111  		ResponseErr             error
   112  		InfoResponse            *csipbv1.GetPluginInfoResponse
   113  		ExpectedResponseName    string
   114  		ExpectedResponseVersion string
   115  		ExpectedErr             error
   116  	}{
   117  		{
   118  			Name:        "handles underlying grpc errors",
   119  			ResponseErr: fmt.Errorf("some grpc error"),
   120  			ExpectedErr: fmt.Errorf("some grpc error"),
   121  		},
   122  		{
   123  			Name: "returns an error if we receive an empty `name`",
   124  			InfoResponse: &csipbv1.GetPluginInfoResponse{
   125  				Name:          "",
   126  				VendorVersion: "",
   127  			},
   128  			ExpectedErr: fmt.Errorf("PluginGetInfo: plugin returned empty name field"),
   129  		},
   130  		{
   131  			Name: "returns the name when successfully retrieved and not empty",
   132  			InfoResponse: &csipbv1.GetPluginInfoResponse{
   133  				Name:          "com.hashicorp.storage",
   134  				VendorVersion: "1.0.1",
   135  			},
   136  			ExpectedResponseName:    "com.hashicorp.storage",
   137  			ExpectedResponseVersion: "1.0.1",
   138  		},
   139  	}
   140  
   141  	for _, tc := range cases {
   142  		t.Run(tc.Name, func(t *testing.T) {
   143  			ic, _, _, client := newTestClient(t)
   144  			defer client.Close()
   145  
   146  			ic.NextErr = tc.ResponseErr
   147  			ic.NextPluginInfo = tc.InfoResponse
   148  
   149  			name, version, err := client.PluginGetInfo(context.TODO())
   150  			if tc.ExpectedErr != nil {
   151  				require.EqualError(t, err, tc.ExpectedErr.Error())
   152  			}
   153  
   154  			require.Equal(t, tc.ExpectedResponseName, name)
   155  			require.Equal(t, tc.ExpectedResponseVersion, version)
   156  		})
   157  	}
   158  
   159  }
   160  
   161  func TestClient_RPC_PluginGetCapabilities(t *testing.T) {
   162  	ci.Parallel(t)
   163  
   164  	cases := []struct {
   165  		Name             string
   166  		ResponseErr      error
   167  		Response         *csipbv1.GetPluginCapabilitiesResponse
   168  		ExpectedResponse *PluginCapabilitySet
   169  		ExpectedErr      error
   170  	}{
   171  		{
   172  			Name:        "handles underlying grpc errors",
   173  			ResponseErr: fmt.Errorf("some grpc error"),
   174  			ExpectedErr: fmt.Errorf("some grpc error"),
   175  		},
   176  		{
   177  			Name: "HasControllerService is true when it's part of the response",
   178  			Response: &csipbv1.GetPluginCapabilitiesResponse{
   179  				Capabilities: []*csipbv1.PluginCapability{
   180  					{
   181  						Type: &csipbv1.PluginCapability_Service_{
   182  							Service: &csipbv1.PluginCapability_Service{
   183  								Type: csipbv1.PluginCapability_Service_CONTROLLER_SERVICE,
   184  							},
   185  						},
   186  					},
   187  				},
   188  			},
   189  			ExpectedResponse: &PluginCapabilitySet{hasControllerService: true},
   190  		},
   191  		{
   192  			Name: "HasTopologies is true when it's part of the response",
   193  			Response: &csipbv1.GetPluginCapabilitiesResponse{
   194  				Capabilities: []*csipbv1.PluginCapability{
   195  					{
   196  						Type: &csipbv1.PluginCapability_Service_{
   197  							Service: &csipbv1.PluginCapability_Service{
   198  								Type: csipbv1.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS,
   199  							},
   200  						},
   201  					},
   202  				},
   203  			},
   204  			ExpectedResponse: &PluginCapabilitySet{hasTopologies: true},
   205  		},
   206  	}
   207  
   208  	for _, tc := range cases {
   209  		t.Run(tc.Name, func(t *testing.T) {
   210  			ic, _, _, client := newTestClient(t)
   211  			defer client.Close()
   212  
   213  			ic.NextErr = tc.ResponseErr
   214  			ic.NextPluginCapabilities = tc.Response
   215  
   216  			resp, err := client.PluginGetCapabilities(context.TODO())
   217  			if tc.ExpectedErr != nil {
   218  				require.EqualError(t, err, tc.ExpectedErr.Error())
   219  			}
   220  
   221  			require.Equal(t, tc.ExpectedResponse, resp)
   222  		})
   223  	}
   224  }
   225  
   226  func TestClient_RPC_ControllerGetCapabilities(t *testing.T) {
   227  	ci.Parallel(t)
   228  
   229  	cases := []struct {
   230  		Name             string
   231  		ResponseErr      error
   232  		Response         *csipbv1.ControllerGetCapabilitiesResponse
   233  		ExpectedResponse *ControllerCapabilitySet
   234  		ExpectedErr      error
   235  	}{
   236  		{
   237  			Name:        "handles underlying grpc errors",
   238  			ResponseErr: fmt.Errorf("some grpc error"),
   239  			ExpectedErr: fmt.Errorf("some grpc error"),
   240  		},
   241  		{
   242  			Name: "ignores unknown capabilities",
   243  			Response: &csipbv1.ControllerGetCapabilitiesResponse{
   244  				Capabilities: []*csipbv1.ControllerServiceCapability{
   245  					{
   246  						Type: &csipbv1.ControllerServiceCapability_Rpc{
   247  							Rpc: &csipbv1.ControllerServiceCapability_RPC{
   248  								Type: csipbv1.ControllerServiceCapability_RPC_UNKNOWN,
   249  							},
   250  						},
   251  					},
   252  				},
   253  			},
   254  			ExpectedResponse: &ControllerCapabilitySet{},
   255  		},
   256  		{
   257  			Name: "detects list volumes capabilities",
   258  			Response: &csipbv1.ControllerGetCapabilitiesResponse{
   259  				Capabilities: []*csipbv1.ControllerServiceCapability{
   260  					{
   261  						Type: &csipbv1.ControllerServiceCapability_Rpc{
   262  							Rpc: &csipbv1.ControllerServiceCapability_RPC{
   263  								Type: csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES,
   264  							},
   265  						},
   266  					},
   267  					{
   268  						Type: &csipbv1.ControllerServiceCapability_Rpc{
   269  							Rpc: &csipbv1.ControllerServiceCapability_RPC{
   270  								Type: csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES,
   271  							},
   272  						},
   273  					},
   274  				},
   275  			},
   276  			ExpectedResponse: &ControllerCapabilitySet{
   277  				HasListVolumes:               true,
   278  				HasListVolumesPublishedNodes: true,
   279  			},
   280  		},
   281  		{
   282  			Name: "detects publish capabilities",
   283  			Response: &csipbv1.ControllerGetCapabilitiesResponse{
   284  				Capabilities: []*csipbv1.ControllerServiceCapability{
   285  					{
   286  						Type: &csipbv1.ControllerServiceCapability_Rpc{
   287  							Rpc: &csipbv1.ControllerServiceCapability_RPC{
   288  								Type: csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY,
   289  							},
   290  						},
   291  					},
   292  					{
   293  						Type: &csipbv1.ControllerServiceCapability_Rpc{
   294  							Rpc: &csipbv1.ControllerServiceCapability_RPC{
   295  								Type: csipbv1.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
   296  							},
   297  						},
   298  					},
   299  				},
   300  			},
   301  			ExpectedResponse: &ControllerCapabilitySet{
   302  				HasPublishUnpublishVolume: true,
   303  				HasPublishReadonly:        true,
   304  			},
   305  		},
   306  	}
   307  
   308  	for _, tc := range cases {
   309  		t.Run(tc.Name, func(t *testing.T) {
   310  			_, cc, _, client := newTestClient(t)
   311  			defer client.Close()
   312  
   313  			cc.NextErr = tc.ResponseErr
   314  			cc.NextCapabilitiesResponse = tc.Response
   315  
   316  			resp, err := client.ControllerGetCapabilities(context.TODO())
   317  			if tc.ExpectedErr != nil {
   318  				require.EqualError(t, err, tc.ExpectedErr.Error())
   319  			}
   320  
   321  			require.Equal(t, tc.ExpectedResponse, resp)
   322  		})
   323  	}
   324  }
   325  
   326  func TestClient_RPC_NodeGetCapabilities(t *testing.T) {
   327  	ci.Parallel(t)
   328  
   329  	cases := []struct {
   330  		Name             string
   331  		ResponseErr      error
   332  		Response         *csipbv1.NodeGetCapabilitiesResponse
   333  		ExpectedResponse *NodeCapabilitySet
   334  		ExpectedErr      error
   335  	}{
   336  		{
   337  			Name:        "handles underlying grpc errors",
   338  			ResponseErr: fmt.Errorf("some grpc error"),
   339  			ExpectedErr: fmt.Errorf("some grpc error"),
   340  		},
   341  		{
   342  			Name: "detects multiple capabilities",
   343  			Response: &csipbv1.NodeGetCapabilitiesResponse{
   344  				Capabilities: []*csipbv1.NodeServiceCapability{
   345  					{
   346  						Type: &csipbv1.NodeServiceCapability_Rpc{
   347  							Rpc: &csipbv1.NodeServiceCapability_RPC{
   348  								Type: csipbv1.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
   349  							},
   350  						},
   351  					},
   352  					{
   353  						Type: &csipbv1.NodeServiceCapability_Rpc{
   354  							Rpc: &csipbv1.NodeServiceCapability_RPC{
   355  								Type: csipbv1.NodeServiceCapability_RPC_EXPAND_VOLUME,
   356  							},
   357  						},
   358  					},
   359  				},
   360  			},
   361  			ExpectedResponse: &NodeCapabilitySet{
   362  				HasStageUnstageVolume: true,
   363  				HasExpandVolume:       true,
   364  			},
   365  		},
   366  	}
   367  
   368  	for _, tc := range cases {
   369  		t.Run(tc.Name, func(t *testing.T) {
   370  			_, _, nc, client := newTestClient(t)
   371  			defer client.Close()
   372  
   373  			nc.NextErr = tc.ResponseErr
   374  			nc.NextCapabilitiesResponse = tc.Response
   375  
   376  			resp, err := client.NodeGetCapabilities(context.TODO())
   377  			if tc.ExpectedErr != nil {
   378  				require.EqualError(t, err, tc.ExpectedErr.Error())
   379  			}
   380  
   381  			require.Equal(t, tc.ExpectedResponse, resp)
   382  		})
   383  	}
   384  }
   385  
   386  func TestClient_RPC_ControllerPublishVolume(t *testing.T) {
   387  	ci.Parallel(t)
   388  
   389  	cases := []struct {
   390  		Name             string
   391  		Request          *ControllerPublishVolumeRequest
   392  		ResponseErr      error
   393  		Response         *csipbv1.ControllerPublishVolumeResponse
   394  		ExpectedResponse *ControllerPublishVolumeResponse
   395  		ExpectedErr      error
   396  	}{
   397  		{
   398  			Name:        "handles underlying grpc errors",
   399  			Request:     &ControllerPublishVolumeRequest{ExternalID: "vol", NodeID: "node"},
   400  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
   401  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
   402  		},
   403  		{
   404  			Name:        "handles missing NodeID",
   405  			Request:     &ControllerPublishVolumeRequest{ExternalID: "vol"},
   406  			Response:    &csipbv1.ControllerPublishVolumeResponse{},
   407  			ExpectedErr: fmt.Errorf("missing NodeID"),
   408  		},
   409  
   410  		{
   411  			Name: "handles PublishContext == nil",
   412  			Request: &ControllerPublishVolumeRequest{
   413  				ExternalID: "vol", NodeID: "node"},
   414  			Response:         &csipbv1.ControllerPublishVolumeResponse{},
   415  			ExpectedResponse: &ControllerPublishVolumeResponse{},
   416  		},
   417  		{
   418  			Name:    "handles PublishContext != nil",
   419  			Request: &ControllerPublishVolumeRequest{ExternalID: "vol", NodeID: "node"},
   420  			Response: &csipbv1.ControllerPublishVolumeResponse{
   421  				PublishContext: map[string]string{
   422  					"com.hashicorp/nomad-node-id": "foobar",
   423  					"com.plugin/device":           "/dev/sdc1",
   424  				},
   425  			},
   426  			ExpectedResponse: &ControllerPublishVolumeResponse{
   427  				PublishContext: map[string]string{
   428  					"com.hashicorp/nomad-node-id": "foobar",
   429  					"com.plugin/device":           "/dev/sdc1",
   430  				},
   431  			},
   432  		},
   433  	}
   434  
   435  	for _, tc := range cases {
   436  		t.Run(tc.Name, func(t *testing.T) {
   437  			_, cc, _, client := newTestClient(t)
   438  			defer client.Close()
   439  
   440  			cc.NextErr = tc.ResponseErr
   441  			cc.NextPublishVolumeResponse = tc.Response
   442  
   443  			resp, err := client.ControllerPublishVolume(context.TODO(), tc.Request)
   444  			if tc.ExpectedErr != nil {
   445  				require.EqualError(t, err, tc.ExpectedErr.Error())
   446  			}
   447  
   448  			require.Equal(t, tc.ExpectedResponse, resp)
   449  		})
   450  	}
   451  }
   452  
   453  func TestClient_RPC_ControllerUnpublishVolume(t *testing.T) {
   454  	ci.Parallel(t)
   455  
   456  	cases := []struct {
   457  		Name             string
   458  		Request          *ControllerUnpublishVolumeRequest
   459  		ResponseErr      error
   460  		Response         *csipbv1.ControllerUnpublishVolumeResponse
   461  		ExpectedResponse *ControllerUnpublishVolumeResponse
   462  		ExpectedErr      error
   463  	}{
   464  		{
   465  			Name:        "handles underlying grpc errors",
   466  			Request:     &ControllerUnpublishVolumeRequest{ExternalID: "vol", NodeID: "node"},
   467  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
   468  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
   469  		},
   470  		{
   471  			Name:             "handles missing NodeID",
   472  			Request:          &ControllerUnpublishVolumeRequest{ExternalID: "vol"},
   473  			ExpectedErr:      fmt.Errorf("missing NodeID"),
   474  			ExpectedResponse: nil,
   475  		},
   476  		{
   477  			Name:             "handles successful response",
   478  			Request:          &ControllerUnpublishVolumeRequest{ExternalID: "vol", NodeID: "node"},
   479  			ExpectedResponse: &ControllerUnpublishVolumeResponse{},
   480  		},
   481  	}
   482  
   483  	for _, tc := range cases {
   484  		t.Run(tc.Name, func(t *testing.T) {
   485  			_, cc, _, client := newTestClient(t)
   486  			defer client.Close()
   487  
   488  			cc.NextErr = tc.ResponseErr
   489  			cc.NextUnpublishVolumeResponse = tc.Response
   490  
   491  			resp, err := client.ControllerUnpublishVolume(context.TODO(), tc.Request)
   492  			if tc.ExpectedErr != nil {
   493  				require.EqualError(t, err, tc.ExpectedErr.Error())
   494  			}
   495  
   496  			require.Equal(t, tc.ExpectedResponse, resp)
   497  		})
   498  	}
   499  }
   500  
   501  func TestClient_RPC_ControllerValidateVolume(t *testing.T) {
   502  	ci.Parallel(t)
   503  
   504  	cases := []struct {
   505  		Name        string
   506  		AccessType  VolumeAccessType
   507  		AccessMode  VolumeAccessMode
   508  		ResponseErr error
   509  		Response    *csipbv1.ValidateVolumeCapabilitiesResponse
   510  		ExpectedErr error
   511  	}{
   512  		{
   513  			Name:        "handles underlying grpc errors",
   514  			AccessType:  VolumeAccessTypeMount,
   515  			AccessMode:  VolumeAccessModeMultiNodeMultiWriter,
   516  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
   517  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
   518  		},
   519  		{
   520  			Name:        "handles success empty capabilities",
   521  			AccessType:  VolumeAccessTypeMount,
   522  			AccessMode:  VolumeAccessModeMultiNodeMultiWriter,
   523  			Response:    &csipbv1.ValidateVolumeCapabilitiesResponse{},
   524  			ResponseErr: nil,
   525  			ExpectedErr: nil,
   526  		},
   527  		{
   528  			Name:       "handles success exact match MountVolume",
   529  			AccessType: VolumeAccessTypeMount,
   530  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   531  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   532  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   533  					VolumeContext: map[string]string{},
   534  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   535  						{
   536  							AccessType: &csipbv1.VolumeCapability_Mount{
   537  								Mount: &csipbv1.VolumeCapability_MountVolume{
   538  									FsType:     "ext4",
   539  									MountFlags: []string{"errors=remount-ro", "noatime"},
   540  								},
   541  							},
   542  							AccessMode: &csipbv1.VolumeCapability_AccessMode{
   543  								Mode: csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
   544  							},
   545  						},
   546  					},
   547  				},
   548  			},
   549  			ResponseErr: nil,
   550  			ExpectedErr: nil,
   551  		},
   552  
   553  		{
   554  			Name:       "handles success exact match BlockVolume",
   555  			AccessType: VolumeAccessTypeBlock,
   556  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   557  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   558  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   559  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   560  						{
   561  							AccessType: &csipbv1.VolumeCapability_Block{
   562  								Block: &csipbv1.VolumeCapability_BlockVolume{},
   563  							},
   564  
   565  							AccessMode: &csipbv1.VolumeCapability_AccessMode{
   566  								Mode: csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
   567  							},
   568  						},
   569  					},
   570  				},
   571  			},
   572  			ResponseErr: nil,
   573  			ExpectedErr: nil,
   574  		},
   575  
   576  		{
   577  			Name:       "handles failure AccessMode mismatch",
   578  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   579  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   580  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   581  					VolumeContext: map[string]string{},
   582  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   583  						{
   584  							AccessType: &csipbv1.VolumeCapability_Block{
   585  								Block: &csipbv1.VolumeCapability_BlockVolume{},
   586  							},
   587  							AccessMode: &csipbv1.VolumeCapability_AccessMode{
   588  								Mode: csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
   589  							},
   590  						},
   591  					},
   592  				},
   593  			},
   594  			ResponseErr: nil,
   595  			// this is a multierror
   596  			ExpectedErr: fmt.Errorf("volume capability validation failed: 1 error occurred:\n\t* requested access mode MULTI_NODE_MULTI_WRITER, got SINGLE_NODE_WRITER\n\n"),
   597  		},
   598  
   599  		{
   600  			Name:       "handles failure MountFlags mismatch",
   601  			AccessType: VolumeAccessTypeMount,
   602  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   603  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   604  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   605  					VolumeContext: map[string]string{},
   606  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   607  						{
   608  							AccessType: &csipbv1.VolumeCapability_Mount{
   609  								Mount: &csipbv1.VolumeCapability_MountVolume{
   610  									FsType:     "ext4",
   611  									MountFlags: []string{},
   612  								},
   613  							},
   614  							AccessMode: &csipbv1.VolumeCapability_AccessMode{
   615  								Mode: csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
   616  							},
   617  						},
   618  					},
   619  				},
   620  			},
   621  			ResponseErr: nil,
   622  			// this is a multierror
   623  			ExpectedErr: fmt.Errorf("volume capability validation failed: 1 error occurred:\n\t* requested mount flags did not match available capabilities\n\n"),
   624  		},
   625  
   626  		{
   627  			Name:       "handles failure MountFlags with Block",
   628  			AccessType: VolumeAccessTypeBlock,
   629  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   630  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   631  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   632  					VolumeContext: map[string]string{},
   633  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   634  						{
   635  							AccessType: &csipbv1.VolumeCapability_Mount{
   636  								Mount: &csipbv1.VolumeCapability_MountVolume{
   637  									FsType:     "ext4",
   638  									MountFlags: []string{},
   639  								},
   640  							},
   641  							AccessMode: &csipbv1.VolumeCapability_AccessMode{
   642  								Mode: csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
   643  							},
   644  						},
   645  					},
   646  				},
   647  			},
   648  			ResponseErr: nil,
   649  			// this is a multierror
   650  			ExpectedErr: fmt.Errorf("volume capability validation failed: 1 error occurred:\n\t* 'file-system' access type was not requested but was validated by the controller\n\n"),
   651  		},
   652  
   653  		{
   654  			Name:       "handles success incomplete no AccessType",
   655  			AccessType: VolumeAccessTypeMount,
   656  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   657  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   658  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   659  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   660  						{
   661  							AccessMode: &csipbv1.VolumeCapability_AccessMode{
   662  								Mode: csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
   663  							},
   664  						},
   665  					},
   666  				},
   667  			},
   668  			ResponseErr: nil,
   669  			ExpectedErr: nil,
   670  		},
   671  
   672  		{
   673  			Name:       "handles success incomplete no AccessMode",
   674  			AccessType: VolumeAccessTypeBlock,
   675  			AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   676  			Response: &csipbv1.ValidateVolumeCapabilitiesResponse{
   677  				Confirmed: &csipbv1.ValidateVolumeCapabilitiesResponse_Confirmed{
   678  					VolumeCapabilities: []*csipbv1.VolumeCapability{
   679  						{
   680  							AccessType: &csipbv1.VolumeCapability_Block{
   681  								Block: &csipbv1.VolumeCapability_BlockVolume{},
   682  							},
   683  						},
   684  					},
   685  				},
   686  			},
   687  			ResponseErr: nil,
   688  			ExpectedErr: nil,
   689  		},
   690  	}
   691  
   692  	for _, tc := range cases {
   693  		t.Run(tc.Name, func(t *testing.T) {
   694  			_, cc, _, client := newTestClient(t)
   695  			defer client.Close()
   696  
   697  			requestedCaps := []*VolumeCapability{{
   698  				AccessType: tc.AccessType,
   699  				AccessMode: tc.AccessMode,
   700  				MountVolume: &structs.CSIMountOptions{ // should be ignored
   701  					FSType:     "ext4",
   702  					MountFlags: []string{"noatime", "errors=remount-ro"},
   703  				},
   704  			}}
   705  			req := &ControllerValidateVolumeRequest{
   706  				ExternalID:   "volumeID",
   707  				Secrets:      structs.CSISecrets{},
   708  				Capabilities: requestedCaps,
   709  				Parameters:   map[string]string{},
   710  				Context:      map[string]string{},
   711  			}
   712  
   713  			cc.NextValidateVolumeCapabilitiesResponse = tc.Response
   714  			cc.NextErr = tc.ResponseErr
   715  
   716  			err := client.ControllerValidateCapabilities(context.TODO(), req)
   717  			if tc.ExpectedErr != nil {
   718  				require.EqualError(t, err, tc.ExpectedErr.Error())
   719  			} else {
   720  				require.NoError(t, err, tc.Name)
   721  			}
   722  		})
   723  	}
   724  }
   725  
   726  func TestClient_RPC_ControllerCreateVolume(t *testing.T) {
   727  	ci.Parallel(t)
   728  
   729  	cases := []struct {
   730  		Name          string
   731  		CapacityRange *CapacityRange
   732  		ContentSource *VolumeContentSource
   733  		ResponseErr   error
   734  		Response      *csipbv1.CreateVolumeResponse
   735  		ExpectedErr   error
   736  	}{
   737  		{
   738  			Name:        "handles underlying grpc errors",
   739  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
   740  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
   741  		},
   742  
   743  		{
   744  			Name: "handles error invalid capacity range",
   745  			CapacityRange: &CapacityRange{
   746  				RequiredBytes: 1000,
   747  				LimitBytes:    500,
   748  			},
   749  			ExpectedErr: errors.New("LimitBytes cannot be less than RequiredBytes"),
   750  		},
   751  
   752  		{
   753  			Name: "handles error invalid content source",
   754  			ContentSource: &VolumeContentSource{
   755  				SnapshotID: "snap-12345",
   756  				CloneID:    "vol-12345",
   757  			},
   758  			ExpectedErr: errors.New(
   759  				"one of SnapshotID or CloneID must be set if ContentSource is set"),
   760  		},
   761  
   762  		{
   763  			Name:     "handles success missing source and range",
   764  			Response: &csipbv1.CreateVolumeResponse{},
   765  		},
   766  
   767  		{
   768  			Name: "handles success with capacity range, source, and topology",
   769  			CapacityRange: &CapacityRange{
   770  				RequiredBytes: 500,
   771  				LimitBytes:    1000,
   772  			},
   773  			ContentSource: &VolumeContentSource{
   774  				SnapshotID: "snap-12345",
   775  			},
   776  			Response: &csipbv1.CreateVolumeResponse{
   777  				Volume: &csipbv1.Volume{
   778  					CapacityBytes: 1000,
   779  					ContentSource: &csipbv1.VolumeContentSource{
   780  						Type: &csipbv1.VolumeContentSource_Snapshot{
   781  							Snapshot: &csipbv1.VolumeContentSource_SnapshotSource{
   782  								SnapshotId: "snap-12345",
   783  							},
   784  						},
   785  					},
   786  					AccessibleTopology: []*csipbv1.Topology{
   787  						{Segments: map[string]string{"rack": "R1"}},
   788  					},
   789  				},
   790  			},
   791  		},
   792  	}
   793  	for _, tc := range cases {
   794  		t.Run(tc.Name, func(t *testing.T) {
   795  			_, cc, _, client := newTestClient(t)
   796  			defer client.Close()
   797  
   798  			req := &ControllerCreateVolumeRequest{
   799  				Name:          "vol-123456",
   800  				CapacityRange: tc.CapacityRange,
   801  				VolumeCapabilities: []*VolumeCapability{
   802  					{
   803  						AccessType: VolumeAccessTypeMount,
   804  						AccessMode: VolumeAccessModeMultiNodeMultiWriter,
   805  					},
   806  				},
   807  				Parameters:    map[string]string{},
   808  				Secrets:       structs.CSISecrets{},
   809  				ContentSource: tc.ContentSource,
   810  				AccessibilityRequirements: &TopologyRequirement{
   811  					Requisite: []*Topology{
   812  						{
   813  							Segments: map[string]string{"rack": "R1"},
   814  						},
   815  						{
   816  							Segments: map[string]string{"rack": "R2"},
   817  						},
   818  					},
   819  				},
   820  			}
   821  
   822  			cc.NextCreateVolumeResponse = tc.Response
   823  			cc.NextErr = tc.ResponseErr
   824  
   825  			resp, err := client.ControllerCreateVolume(context.TODO(), req)
   826  			if tc.ExpectedErr != nil {
   827  				require.EqualError(t, err, tc.ExpectedErr.Error())
   828  				return
   829  			}
   830  			require.NoError(t, err, tc.Name)
   831  			if tc.Response == nil {
   832  				require.Nil(t, resp)
   833  				return
   834  			}
   835  			if tc.CapacityRange != nil {
   836  				require.Greater(t, resp.Volume.CapacityBytes, int64(0))
   837  			}
   838  			if tc.ContentSource != nil {
   839  				require.Equal(t, tc.ContentSource.CloneID, resp.Volume.ContentSource.CloneID)
   840  				require.Equal(t, tc.ContentSource.SnapshotID, resp.Volume.ContentSource.SnapshotID)
   841  			}
   842  			if tc.Response != nil && tc.Response.Volume != nil {
   843  				require.Len(t, resp.Volume.AccessibleTopology, 1)
   844  				require.Equal(t,
   845  					req.AccessibilityRequirements.Requisite[0].Segments,
   846  					resp.Volume.AccessibleTopology[0].Segments,
   847  				)
   848  			}
   849  
   850  		})
   851  	}
   852  }
   853  
   854  func TestClient_RPC_ControllerDeleteVolume(t *testing.T) {
   855  	ci.Parallel(t)
   856  
   857  	cases := []struct {
   858  		Name        string
   859  		Request     *ControllerDeleteVolumeRequest
   860  		ResponseErr error
   861  		ExpectedErr error
   862  	}{
   863  		{
   864  			Name:        "handles underlying grpc errors",
   865  			Request:     &ControllerDeleteVolumeRequest{ExternalVolumeID: "vol-12345"},
   866  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
   867  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
   868  		},
   869  
   870  		{
   871  			Name:        "handles error missing volume ID",
   872  			Request:     &ControllerDeleteVolumeRequest{},
   873  			ExpectedErr: errors.New("missing ExternalVolumeID"),
   874  		},
   875  
   876  		{
   877  			Name:    "handles success",
   878  			Request: &ControllerDeleteVolumeRequest{ExternalVolumeID: "vol-12345"},
   879  		},
   880  	}
   881  	for _, tc := range cases {
   882  		t.Run(tc.Name, func(t *testing.T) {
   883  			_, cc, _, client := newTestClient(t)
   884  			defer client.Close()
   885  
   886  			cc.NextErr = tc.ResponseErr
   887  			err := client.ControllerDeleteVolume(context.TODO(), tc.Request)
   888  			if tc.ExpectedErr != nil {
   889  				require.EqualError(t, err, tc.ExpectedErr.Error())
   890  				return
   891  			}
   892  			require.NoError(t, err, tc.Name)
   893  		})
   894  	}
   895  }
   896  
   897  func TestClient_RPC_ControllerListVolume(t *testing.T) {
   898  	ci.Parallel(t)
   899  
   900  	cases := []struct {
   901  		Name        string
   902  		Request     *ControllerListVolumesRequest
   903  		ResponseErr error
   904  		ExpectedErr error
   905  	}{
   906  		{
   907  			Name:        "handles underlying grpc errors",
   908  			Request:     &ControllerListVolumesRequest{},
   909  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
   910  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
   911  		},
   912  
   913  		{
   914  			Name:        "handles error invalid max entries",
   915  			Request:     &ControllerListVolumesRequest{MaxEntries: -1},
   916  			ExpectedErr: errors.New("MaxEntries cannot be negative"),
   917  		},
   918  
   919  		{
   920  			Name:    "handles success",
   921  			Request: &ControllerListVolumesRequest{},
   922  		},
   923  	}
   924  
   925  	for _, tc := range cases {
   926  		t.Run(tc.Name, func(t *testing.T) {
   927  			_, cc, _, client := newTestClient(t)
   928  			defer client.Close()
   929  
   930  			cc.NextErr = tc.ResponseErr
   931  			if tc.ResponseErr != nil {
   932  				// note: there's nothing interesting to assert here other than
   933  				// that we don't throw a NPE during transformation from
   934  				// protobuf to our struct
   935  				cc.NextListVolumesResponse = &csipbv1.ListVolumesResponse{
   936  					Entries: []*csipbv1.ListVolumesResponse_Entry{
   937  						{
   938  							Volume: &csipbv1.Volume{
   939  								CapacityBytes: 1000000,
   940  								VolumeId:      "vol-0",
   941  								VolumeContext: map[string]string{"foo": "bar"},
   942  
   943  								ContentSource: &csipbv1.VolumeContentSource{},
   944  								AccessibleTopology: []*csipbv1.Topology{
   945  									{
   946  										Segments: map[string]string{"rack": "A"},
   947  									},
   948  								},
   949  							},
   950  						},
   951  
   952  						{
   953  							Volume: &csipbv1.Volume{
   954  								VolumeId: "vol-1",
   955  								AccessibleTopology: []*csipbv1.Topology{
   956  									{
   957  										Segments: map[string]string{"rack": "A"},
   958  									},
   959  								},
   960  							},
   961  						},
   962  
   963  						{
   964  							Volume: &csipbv1.Volume{
   965  								VolumeId: "vol-3",
   966  								ContentSource: &csipbv1.VolumeContentSource{
   967  									Type: &csipbv1.VolumeContentSource_Snapshot{
   968  										Snapshot: &csipbv1.VolumeContentSource_SnapshotSource{
   969  											SnapshotId: "snap-12345",
   970  										},
   971  									},
   972  								},
   973  							},
   974  						},
   975  					},
   976  					NextToken: "abcdef",
   977  				}
   978  			}
   979  
   980  			resp, err := client.ControllerListVolumes(context.TODO(), tc.Request)
   981  			if tc.ExpectedErr != nil {
   982  				require.EqualError(t, err, tc.ExpectedErr.Error())
   983  				return
   984  			}
   985  			require.NoError(t, err, tc.Name)
   986  			require.NotNil(t, resp)
   987  
   988  		})
   989  	}
   990  }
   991  
   992  func TestClient_RPC_ControllerCreateSnapshot(t *testing.T) {
   993  	ci.Parallel(t)
   994  
   995  	now := time.Now()
   996  
   997  	cases := []struct {
   998  		Name        string
   999  		Request     *ControllerCreateSnapshotRequest
  1000  		Response    *csipbv1.CreateSnapshotResponse
  1001  		ResponseErr error
  1002  		ExpectedErr error
  1003  	}{
  1004  		{
  1005  			Name: "handles underlying grpc errors",
  1006  			Request: &ControllerCreateSnapshotRequest{
  1007  				VolumeID: "vol-12345",
  1008  				Name:     "snap-12345",
  1009  			},
  1010  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
  1011  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
  1012  		},
  1013  
  1014  		{
  1015  			Name:        "handles error missing volume ID",
  1016  			Request:     &ControllerCreateSnapshotRequest{},
  1017  			ExpectedErr: errors.New("missing VolumeID"),
  1018  		},
  1019  
  1020  		{
  1021  			Name: "handles success",
  1022  			Request: &ControllerCreateSnapshotRequest{
  1023  				VolumeID: "vol-12345",
  1024  				Name:     "snap-12345",
  1025  			},
  1026  			Response: &csipbv1.CreateSnapshotResponse{
  1027  				Snapshot: &csipbv1.Snapshot{
  1028  					SizeBytes:      100000,
  1029  					SnapshotId:     "snap-12345",
  1030  					SourceVolumeId: "vol-12345",
  1031  					CreationTime:   timestamppb.New(now),
  1032  					ReadyToUse:     true,
  1033  				},
  1034  			},
  1035  		},
  1036  	}
  1037  	for _, tc := range cases {
  1038  		t.Run(tc.Name, func(t *testing.T) {
  1039  			_, cc, _, client := newTestClient(t)
  1040  			defer client.Close()
  1041  
  1042  			cc.NextErr = tc.ResponseErr
  1043  			cc.NextCreateSnapshotResponse = tc.Response
  1044  			// note: there's nothing interesting to assert about the response
  1045  			// here other than that we don't throw a NPE during transformation
  1046  			// from protobuf to our struct
  1047  			resp, err := client.ControllerCreateSnapshot(context.TODO(), tc.Request)
  1048  			if tc.ExpectedErr != nil {
  1049  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1050  			} else {
  1051  				require.NoError(t, err, tc.Name)
  1052  				require.NotZero(t, resp.Snapshot.CreateTime)
  1053  				require.Equal(t, now.Second(), time.Unix(resp.Snapshot.CreateTime, 0).Second())
  1054  			}
  1055  		})
  1056  	}
  1057  }
  1058  
  1059  func TestClient_RPC_ControllerDeleteSnapshot(t *testing.T) {
  1060  	ci.Parallel(t)
  1061  
  1062  	cases := []struct {
  1063  		Name        string
  1064  		Request     *ControllerDeleteSnapshotRequest
  1065  		ResponseErr error
  1066  		ExpectedErr error
  1067  	}{
  1068  		{
  1069  			Name:        "handles underlying grpc errors",
  1070  			Request:     &ControllerDeleteSnapshotRequest{SnapshotID: "vol-12345"},
  1071  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
  1072  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
  1073  		},
  1074  
  1075  		{
  1076  			Name:        "handles error missing volume ID",
  1077  			Request:     &ControllerDeleteSnapshotRequest{},
  1078  			ExpectedErr: errors.New("missing SnapshotID"),
  1079  		},
  1080  
  1081  		{
  1082  			Name:    "handles success",
  1083  			Request: &ControllerDeleteSnapshotRequest{SnapshotID: "vol-12345"},
  1084  		},
  1085  	}
  1086  	for _, tc := range cases {
  1087  		t.Run(tc.Name, func(t *testing.T) {
  1088  			_, cc, _, client := newTestClient(t)
  1089  			defer client.Close()
  1090  
  1091  			cc.NextErr = tc.ResponseErr
  1092  			err := client.ControllerDeleteSnapshot(context.TODO(), tc.Request)
  1093  			if tc.ExpectedErr != nil {
  1094  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1095  				return
  1096  			}
  1097  			require.NoError(t, err, tc.Name)
  1098  		})
  1099  	}
  1100  }
  1101  
  1102  func TestClient_RPC_ControllerListSnapshots(t *testing.T) {
  1103  	ci.Parallel(t)
  1104  
  1105  	cases := []struct {
  1106  		Name        string
  1107  		Request     *ControllerListSnapshotsRequest
  1108  		ResponseErr error
  1109  		ExpectedErr error
  1110  	}{
  1111  		{
  1112  			Name:        "handles underlying grpc errors",
  1113  			Request:     &ControllerListSnapshotsRequest{},
  1114  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
  1115  			ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
  1116  		},
  1117  
  1118  		{
  1119  			Name:        "handles error invalid max entries",
  1120  			Request:     &ControllerListSnapshotsRequest{MaxEntries: -1},
  1121  			ExpectedErr: errors.New("MaxEntries cannot be negative"),
  1122  		},
  1123  
  1124  		{
  1125  			Name:    "handles success",
  1126  			Request: &ControllerListSnapshotsRequest{},
  1127  		},
  1128  	}
  1129  
  1130  	now := time.Now()
  1131  
  1132  	for _, tc := range cases {
  1133  		t.Run(tc.Name, func(t *testing.T) {
  1134  			_, cc, _, client := newTestClient(t)
  1135  			defer client.Close()
  1136  
  1137  			cc.NextErr = tc.ResponseErr
  1138  			if tc.ResponseErr == nil {
  1139  				cc.NextListSnapshotsResponse = &csipbv1.ListSnapshotsResponse{
  1140  					Entries: []*csipbv1.ListSnapshotsResponse_Entry{
  1141  						{
  1142  							Snapshot: &csipbv1.Snapshot{
  1143  								SizeBytes:      1000000,
  1144  								SnapshotId:     "snap-12345",
  1145  								SourceVolumeId: "vol-12345",
  1146  								ReadyToUse:     true,
  1147  								CreationTime:   timestamppb.New(now),
  1148  							},
  1149  						},
  1150  					},
  1151  					NextToken: "abcdef",
  1152  				}
  1153  			}
  1154  
  1155  			resp, err := client.ControllerListSnapshots(context.TODO(), tc.Request)
  1156  			if tc.ExpectedErr != nil {
  1157  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1158  				return
  1159  			}
  1160  			require.NoError(t, err, tc.Name)
  1161  			require.NotNil(t, resp)
  1162  			require.Len(t, resp.Entries, 1)
  1163  			require.NotZero(t, resp.Entries[0].Snapshot.CreateTime)
  1164  			require.Equal(t, now.Second(),
  1165  				time.Unix(resp.Entries[0].Snapshot.CreateTime, 0).Second())
  1166  		})
  1167  	}
  1168  }
  1169  
  1170  func TestClient_RPC_NodeStageVolume(t *testing.T) {
  1171  	ci.Parallel(t)
  1172  
  1173  	cases := []struct {
  1174  		Name        string
  1175  		ResponseErr error
  1176  		Response    *csipbv1.NodeStageVolumeResponse
  1177  		ExpectedErr error
  1178  	}{
  1179  		{
  1180  			Name:        "handles underlying grpc errors",
  1181  			ResponseErr: status.Errorf(codes.AlreadyExists, "some grpc error"),
  1182  			ExpectedErr: fmt.Errorf("volume \"foo\" is already staged to \"/path\" but with incompatible capabilities for this request: rpc error: code = AlreadyExists desc = some grpc error"),
  1183  		},
  1184  		{
  1185  			Name:        "handles success",
  1186  			ResponseErr: nil,
  1187  			ExpectedErr: nil,
  1188  		},
  1189  	}
  1190  
  1191  	for _, tc := range cases {
  1192  		t.Run(tc.Name, func(t *testing.T) {
  1193  			_, _, nc, client := newTestClient(t)
  1194  			defer client.Close()
  1195  
  1196  			nc.NextErr = tc.ResponseErr
  1197  			nc.NextStageVolumeResponse = tc.Response
  1198  
  1199  			err := client.NodeStageVolume(context.TODO(), &NodeStageVolumeRequest{
  1200  				ExternalID:        "foo",
  1201  				StagingTargetPath: "/path",
  1202  				VolumeCapability:  &VolumeCapability{},
  1203  			})
  1204  			if tc.ExpectedErr != nil {
  1205  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1206  			} else {
  1207  				require.Nil(t, err)
  1208  			}
  1209  		})
  1210  	}
  1211  }
  1212  
  1213  func TestClient_RPC_NodeUnstageVolume(t *testing.T) {
  1214  	ci.Parallel(t)
  1215  
  1216  	cases := []struct {
  1217  		Name        string
  1218  		ResponseErr error
  1219  		Response    *csipbv1.NodeUnstageVolumeResponse
  1220  		ExpectedErr error
  1221  	}{
  1222  		{
  1223  			Name:        "handles underlying grpc errors",
  1224  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
  1225  			ExpectedErr: fmt.Errorf("node plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
  1226  		},
  1227  		{
  1228  			Name:        "handles success",
  1229  			ResponseErr: nil,
  1230  			ExpectedErr: nil,
  1231  		},
  1232  	}
  1233  
  1234  	for _, tc := range cases {
  1235  		t.Run(tc.Name, func(t *testing.T) {
  1236  			_, _, nc, client := newTestClient(t)
  1237  			defer client.Close()
  1238  
  1239  			nc.NextErr = tc.ResponseErr
  1240  			nc.NextUnstageVolumeResponse = tc.Response
  1241  
  1242  			err := client.NodeUnstageVolume(context.TODO(), "foo", "/foo")
  1243  			if tc.ExpectedErr != nil {
  1244  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1245  			} else {
  1246  				require.Nil(t, err)
  1247  			}
  1248  		})
  1249  	}
  1250  }
  1251  
  1252  func TestClient_RPC_NodePublishVolume(t *testing.T) {
  1253  	ci.Parallel(t)
  1254  
  1255  	cases := []struct {
  1256  		Name        string
  1257  		Request     *NodePublishVolumeRequest
  1258  		ResponseErr error
  1259  		Response    *csipbv1.NodePublishVolumeResponse
  1260  		ExpectedErr error
  1261  	}{
  1262  		{
  1263  			Name: "handles underlying grpc errors",
  1264  			Request: &NodePublishVolumeRequest{
  1265  				ExternalID:       "foo",
  1266  				TargetPath:       "/dev/null",
  1267  				VolumeCapability: &VolumeCapability{},
  1268  			},
  1269  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
  1270  			ExpectedErr: fmt.Errorf("node plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
  1271  		},
  1272  		{
  1273  			Name: "handles success",
  1274  			Request: &NodePublishVolumeRequest{
  1275  				ExternalID:       "foo",
  1276  				TargetPath:       "/dev/null",
  1277  				VolumeCapability: &VolumeCapability{},
  1278  			},
  1279  			ResponseErr: nil,
  1280  			ExpectedErr: nil,
  1281  		},
  1282  		{
  1283  			Name: "Performs validation of the publish volume request",
  1284  			Request: &NodePublishVolumeRequest{
  1285  				ExternalID: "",
  1286  			},
  1287  			ResponseErr: nil,
  1288  			ExpectedErr: errors.New("validation error: missing volume ID"),
  1289  		},
  1290  	}
  1291  
  1292  	for _, tc := range cases {
  1293  		t.Run(tc.Name, func(t *testing.T) {
  1294  			_, _, nc, client := newTestClient(t)
  1295  			defer client.Close()
  1296  
  1297  			nc.NextErr = tc.ResponseErr
  1298  			nc.NextPublishVolumeResponse = tc.Response
  1299  
  1300  			err := client.NodePublishVolume(context.TODO(), tc.Request)
  1301  			if tc.ExpectedErr != nil {
  1302  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1303  			} else {
  1304  				require.Nil(t, err)
  1305  			}
  1306  		})
  1307  	}
  1308  }
  1309  func TestClient_RPC_NodeUnpublishVolume(t *testing.T) {
  1310  	ci.Parallel(t)
  1311  
  1312  	cases := []struct {
  1313  		Name        string
  1314  		ExternalID  string
  1315  		TargetPath  string
  1316  		ResponseErr error
  1317  		Response    *csipbv1.NodeUnpublishVolumeResponse
  1318  		ExpectedErr error
  1319  	}{
  1320  		{
  1321  			Name:        "handles underlying grpc errors",
  1322  			ExternalID:  "foo",
  1323  			TargetPath:  "/dev/null",
  1324  			ResponseErr: status.Errorf(codes.Internal, "some grpc error"),
  1325  			ExpectedErr: fmt.Errorf("node plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"),
  1326  		},
  1327  		{
  1328  			Name:        "handles success",
  1329  			ExternalID:  "foo",
  1330  			TargetPath:  "/dev/null",
  1331  			ResponseErr: nil,
  1332  			ExpectedErr: nil,
  1333  		},
  1334  		{
  1335  			Name:        "Performs validation of the request args - ExternalID",
  1336  			ResponseErr: nil,
  1337  			ExpectedErr: errors.New("missing volumeID"),
  1338  		},
  1339  		{
  1340  			Name:        "Performs validation of the request args - TargetPath",
  1341  			ExternalID:  "foo",
  1342  			ResponseErr: nil,
  1343  			ExpectedErr: errors.New("missing targetPath"),
  1344  		},
  1345  	}
  1346  
  1347  	for _, tc := range cases {
  1348  		t.Run(tc.Name, func(t *testing.T) {
  1349  			_, _, nc, client := newTestClient(t)
  1350  			defer client.Close()
  1351  
  1352  			nc.NextErr = tc.ResponseErr
  1353  			nc.NextUnpublishVolumeResponse = tc.Response
  1354  
  1355  			err := client.NodeUnpublishVolume(context.TODO(), tc.ExternalID, tc.TargetPath)
  1356  			if tc.ExpectedErr != nil {
  1357  				require.EqualError(t, err, tc.ExpectedErr.Error())
  1358  			} else {
  1359  				require.Nil(t, err)
  1360  			}
  1361  		})
  1362  	}
  1363  }