github.com/aminovpavel/nomad@v0.11.8/nomad/client_csi_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "testing" 6 7 memdb "github.com/hashicorp/go-memdb" 8 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 9 "github.com/hashicorp/nomad/client" 10 "github.com/hashicorp/nomad/client/config" 11 cstructs "github.com/hashicorp/nomad/client/structs" 12 "github.com/hashicorp/nomad/helper/uuid" 13 "github.com/hashicorp/nomad/nomad/mock" 14 "github.com/hashicorp/nomad/nomad/structs" 15 "github.com/hashicorp/nomad/testutil" 16 "github.com/stretchr/testify/require" 17 ) 18 19 func TestClientCSIController_AttachVolume_Local(t *testing.T) { 20 t.Parallel() 21 require := require.New(t) 22 23 // Start a server and client 24 s, cleanupS := TestServer(t, nil) 25 defer cleanupS() 26 codec := rpcClient(t, s) 27 testutil.WaitForLeader(t, s.RPC) 28 29 c, cleanupC := client.TestClient(t, func(c *config.Config) { 30 c.Servers = []string{s.config.RPCAddr.String()} 31 }) 32 defer cleanupC() 33 34 waitForNodes(t, s, 1) 35 36 req := &cstructs.ClientCSIControllerAttachVolumeRequest{ 37 CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()}, 38 } 39 40 // Fetch the response 41 var resp structs.GenericResponse 42 err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp) 43 require.NotNil(err) 44 // Should recieve an error from the client endpoint 45 require.Contains(err.Error(), "must specify plugin name to dispense") 46 } 47 48 func TestClientCSIController_AttachVolume_Forwarded(t *testing.T) { 49 t.Parallel() 50 require := require.New(t) 51 52 // Start a server and client 53 s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) 54 defer cleanupS1() 55 s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) 56 defer cleanupS2() 57 TestJoin(t, s1, s2) 58 testutil.WaitForLeader(t, s1.RPC) 59 testutil.WaitForLeader(t, s2.RPC) 60 codec := rpcClient(t, s2) 61 62 c, cleanupC := client.TestClient(t, func(c *config.Config) { 63 c.Servers = []string{s2.config.RPCAddr.String()} 64 c.GCDiskUsageThreshold = 100.0 65 }) 66 defer cleanupC() 67 68 waitForNodes(t, s2, 1) 69 70 // Force remove the connection locally in case it exists 71 s1.nodeConnsLock.Lock() 72 delete(s1.nodeConns, c.NodeID()) 73 s1.nodeConnsLock.Unlock() 74 75 req := &cstructs.ClientCSIControllerAttachVolumeRequest{ 76 CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()}, 77 } 78 79 // Fetch the response 80 var resp structs.GenericResponse 81 err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp) 82 require.NotNil(err) 83 // Should recieve an error from the client endpoint 84 require.Contains(err.Error(), "must specify plugin name to dispense") 85 } 86 87 func TestClientCSIController_DetachVolume_Local(t *testing.T) { 88 t.Parallel() 89 require := require.New(t) 90 91 // Start a server and client 92 s, cleanupS := TestServer(t, nil) 93 defer cleanupS() 94 codec := rpcClient(t, s) 95 testutil.WaitForLeader(t, s.RPC) 96 97 c, cleanupC := client.TestClient(t, func(c *config.Config) { 98 c.Servers = []string{s.config.RPCAddr.String()} 99 }) 100 defer cleanupC() 101 102 waitForNodes(t, s, 1) 103 104 req := &cstructs.ClientCSIControllerDetachVolumeRequest{ 105 CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()}, 106 } 107 108 // Fetch the response 109 var resp structs.GenericResponse 110 err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp) 111 require.NotNil(err) 112 // Should recieve an error from the client endpoint 113 require.Contains(err.Error(), "must specify plugin name to dispense") 114 } 115 116 func TestClientCSIController_DetachVolume_Forwarded(t *testing.T) { 117 t.Parallel() 118 require := require.New(t) 119 120 // Start a server and client 121 s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) 122 defer cleanupS1() 123 s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) 124 defer cleanupS2() 125 TestJoin(t, s1, s2) 126 testutil.WaitForLeader(t, s1.RPC) 127 testutil.WaitForLeader(t, s2.RPC) 128 codec := rpcClient(t, s2) 129 130 c, cleanupC := client.TestClient(t, func(c *config.Config) { 131 c.Servers = []string{s2.config.RPCAddr.String()} 132 c.GCDiskUsageThreshold = 100.0 133 }) 134 defer cleanupC() 135 136 waitForNodes(t, s2, 1) 137 138 // Force remove the connection locally in case it exists 139 s1.nodeConnsLock.Lock() 140 delete(s1.nodeConns, c.NodeID()) 141 s1.nodeConnsLock.Unlock() 142 143 req := &cstructs.ClientCSIControllerDetachVolumeRequest{ 144 CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()}, 145 } 146 147 // Fetch the response 148 var resp structs.GenericResponse 149 err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp) 150 require.NotNil(err) 151 // Should recieve an error from the client endpoint 152 require.Contains(err.Error(), "must specify plugin name to dispense") 153 } 154 155 func TestClientCSI_NodeForControllerPlugin(t *testing.T) { 156 t.Parallel() 157 srv, shutdown := TestServer(t, func(c *Config) {}) 158 testutil.WaitForLeader(t, srv.RPC) 159 defer shutdown() 160 161 plugins := map[string]*structs.CSIInfo{ 162 "minnie": {PluginID: "minnie", 163 Healthy: true, 164 ControllerInfo: &structs.CSIControllerInfo{}, 165 NodeInfo: &structs.CSINodeInfo{}, 166 RequiresControllerPlugin: true, 167 }, 168 } 169 state := srv.fsm.State() 170 171 node1 := mock.Node() 172 node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions 173 node1.CSIControllerPlugins = plugins 174 node2 := mock.Node() 175 node2.CSIControllerPlugins = plugins 176 node2.ID = uuid.Generate() 177 node3 := mock.Node() 178 node3.ID = uuid.Generate() 179 180 err := state.UpsertNode(1002, node1) 181 require.NoError(t, err) 182 err = state.UpsertNode(1003, node2) 183 require.NoError(t, err) 184 err = state.UpsertNode(1004, node3) 185 require.NoError(t, err) 186 187 ws := memdb.NewWatchSet() 188 189 plugin, err := state.CSIPluginByID(ws, "minnie") 190 require.NoError(t, err) 191 nodeID, err := srv.staticEndpoints.ClientCSI.nodeForController(plugin.ID, "") 192 193 // only node1 has both the controller and a recent Nomad version 194 require.Equal(t, nodeID, node1.ID) 195 } 196 197 // waitForNodes waits until the server is connected to expectedNodes 198 // clients and they are in the state store 199 func waitForNodes(t *testing.T, s *Server, expectedNodes int) { 200 codec := rpcClient(t, s) 201 202 testutil.WaitForResult(func() (bool, error) { 203 connNodes := s.connectedNodes() 204 if len(connNodes) != expectedNodes { 205 return false, fmt.Errorf("expected %d nodes but found %d", expectedNodes, len(connNodes)) 206 207 } 208 209 get := &structs.NodeListRequest{ 210 QueryOptions: structs.QueryOptions{Region: "global"}, 211 } 212 var resp structs.NodeListResponse 213 err := msgpackrpc.CallWithCodec(codec, "Node.List", get, &resp) 214 if err != nil { 215 return false, err 216 } 217 218 if err != nil { 219 return false, fmt.Errorf("failed to list nodes: %v", err) 220 } 221 if len(resp.Nodes) != 1 { 222 return false, fmt.Errorf("expected %d nodes but found %d", 1, len(resp.Nodes)) 223 } 224 225 return true, nil 226 }, func(err error) { 227 require.NoError(t, err) 228 }) 229 }