gopkg.in/hashicorp/nomad.v0@v0.11.8/nomad/client_csi_endpoint.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "math/rand" 6 "time" 7 8 metrics "github.com/armon/go-metrics" 9 log "github.com/hashicorp/go-hclog" 10 memdb "github.com/hashicorp/go-memdb" 11 cstructs "github.com/hashicorp/nomad/client/structs" 12 ) 13 14 // ClientCSI is used to forward RPC requests to the targed Nomad client's 15 // CSIController endpoint. 16 type ClientCSI struct { 17 srv *Server 18 logger log.Logger 19 } 20 21 func (a *ClientCSI) ControllerAttachVolume(args *cstructs.ClientCSIControllerAttachVolumeRequest, reply *cstructs.ClientCSIControllerAttachVolumeResponse) error { 22 defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "attach_volume"}, time.Now()) 23 // Get a Nomad client node for the controller 24 nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID) 25 if err != nil { 26 return err 27 } 28 args.ControllerNodeID = nodeID 29 30 // Get the connection to the client 31 state, ok := a.srv.getNodeConn(args.ControllerNodeID) 32 if !ok { 33 return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerAttachVolume", args, reply) 34 } 35 36 // Make the RPC 37 err = NodeRpc(state.Session, "CSI.ControllerAttachVolume", args, reply) 38 if err != nil { 39 return fmt.Errorf("controller attach volume: %v", err) 40 } 41 return nil 42 } 43 44 func (a *ClientCSI) ControllerValidateVolume(args *cstructs.ClientCSIControllerValidateVolumeRequest, reply *cstructs.ClientCSIControllerValidateVolumeResponse) error { 45 defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "validate_volume"}, time.Now()) 46 47 // Get a Nomad client node for the controller 48 nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID) 49 if err != nil { 50 return err 51 } 52 args.ControllerNodeID = nodeID 53 54 // Get the connection to the client 55 state, ok := a.srv.getNodeConn(args.ControllerNodeID) 56 if !ok { 57 return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerValidateVolume", args, reply) 58 } 59 60 // Make the RPC 61 err = NodeRpc(state.Session, "CSI.ControllerValidateVolume", args, reply) 62 if err != nil { 63 return fmt.Errorf("validate volume: %v", err) 64 } 65 return nil 66 } 67 68 func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDetachVolumeRequest, reply *cstructs.ClientCSIControllerDetachVolumeResponse) error { 69 defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "detach_volume"}, time.Now()) 70 71 // Get a Nomad client node for the controller 72 nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID) 73 if err != nil { 74 return err 75 } 76 args.ControllerNodeID = nodeID 77 78 // Get the connection to the client 79 state, ok := a.srv.getNodeConn(args.ControllerNodeID) 80 if !ok { 81 return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerDetachVolume", args, reply) 82 } 83 84 // Make the RPC 85 err = NodeRpc(state.Session, "CSI.ControllerDetachVolume", args, reply) 86 if err != nil { 87 return fmt.Errorf("controller detach volume: %v", err) 88 } 89 return nil 90 91 } 92 93 func (a *ClientCSI) NodeDetachVolume(args *cstructs.ClientCSINodeDetachVolumeRequest, reply *cstructs.ClientCSINodeDetachVolumeResponse) error { 94 defer metrics.MeasureSince([]string{"nomad", "client_csi_node", "detach_volume"}, time.Now()) 95 96 // Make sure Node is valid and new enough to support RPC 97 snap, err := a.srv.State().Snapshot() 98 if err != nil { 99 return err 100 } 101 102 _, err = getNodeForRpc(snap, args.NodeID) 103 if err != nil { 104 return err 105 } 106 107 // Get the connection to the client 108 state, ok := a.srv.getNodeConn(args.NodeID) 109 if !ok { 110 return findNodeConnAndForward(a.srv, args.NodeID, "ClientCSI.NodeDetachVolume", args, reply) 111 } 112 113 // Make the RPC 114 err = NodeRpc(state.Session, "CSI.NodeDetachVolume", args, reply) 115 if err != nil { 116 return fmt.Errorf("node detach volume: %v", err) 117 } 118 return nil 119 120 } 121 122 // nodeForController validates that the Nomad client node ID for 123 // a plugin exists and is new enough to support client RPC. If no node 124 // ID is passed, select a random node ID for the controller to load-balance 125 // long blocking RPCs across client nodes. 126 func (a *ClientCSI) nodeForController(pluginID, nodeID string) (string, error) { 127 128 snap, err := a.srv.State().Snapshot() 129 if err != nil { 130 return "", err 131 } 132 133 if nodeID != "" { 134 _, err = getNodeForRpc(snap, nodeID) 135 if err == nil { 136 return nodeID, nil 137 } else { 138 // we'll fall-through and select a node at random 139 a.logger.Trace("could not be used for client RPC", "node", nodeID, "error", err) 140 } 141 } 142 143 if pluginID == "" { 144 return "", fmt.Errorf("missing plugin ID") 145 } 146 147 ws := memdb.NewWatchSet() 148 149 // note: plugin IDs are not scoped to region/DC but volumes are. 150 // so any node we get for a controller is already in the same 151 // region/DC for the volume. 152 plugin, err := snap.CSIPluginByID(ws, pluginID) 153 if err != nil { 154 return "", fmt.Errorf("error getting plugin: %s, %v", pluginID, err) 155 } 156 if plugin == nil { 157 return "", fmt.Errorf("plugin missing: %s %v", pluginID, err) 158 } 159 count := len(plugin.Controllers) 160 if count == 0 { 161 return "", fmt.Errorf("no controllers available for plugin %q", plugin.ID) 162 } 163 164 // iterating maps is "random" but unspecified and isn't particularly 165 // random with small maps, so not well-suited for load balancing. 166 // so we shuffle the keys and iterate over them. 167 clientIDs := make([]string, 0, count) 168 for clientID := range plugin.Controllers { 169 clientIDs = append(clientIDs, clientID) 170 } 171 rand.Shuffle(count, func(i, j int) { 172 clientIDs[i], clientIDs[j] = clientIDs[j], clientIDs[i] 173 }) 174 175 for _, clientID := range clientIDs { 176 controller := plugin.Controllers[clientID] 177 if !controller.IsController() { 178 // we don't have separate types for CSIInfo depending on 179 // whether it's a controller or node. this error shouldn't 180 // make it to production but is to aid developers during 181 // development 182 err = fmt.Errorf("plugin is not a controller") 183 continue 184 } 185 _, err = getNodeForRpc(snap, clientID) 186 if err != nil { 187 continue 188 } 189 return clientID, nil 190 } 191 192 return "", err 193 }