github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/nomad/csi_endpoint.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "time" 6 7 metrics "github.com/armon/go-metrics" 8 log "github.com/hashicorp/go-hclog" 9 memdb "github.com/hashicorp/go-memdb" 10 multierror "github.com/hashicorp/go-multierror" 11 "github.com/hashicorp/nomad/acl" 12 cstructs "github.com/hashicorp/nomad/client/structs" 13 "github.com/hashicorp/nomad/nomad/state" 14 "github.com/hashicorp/nomad/nomad/structs" 15 ) 16 17 // CSIVolume wraps the structs.CSIVolume with request data and server context 18 type CSIVolume struct { 19 srv *Server 20 logger log.Logger 21 } 22 23 // QueryACLObj looks up the ACL token in the request and returns the acl.ACL object 24 // - fallback to node secret ids 25 func (srv *Server) QueryACLObj(args *structs.QueryOptions, allowNodeAccess bool) (*acl.ACL, error) { 26 // Lookup the token 27 aclObj, err := srv.ResolveToken(args.AuthToken) 28 if err != nil { 29 // If ResolveToken had an unexpected error return that 30 if !structs.IsErrTokenNotFound(err) { 31 return nil, err 32 } 33 34 // If we don't allow access to this endpoint from Nodes, then return token 35 // not found. 36 if !allowNodeAccess { 37 return nil, structs.ErrTokenNotFound 38 } 39 40 ws := memdb.NewWatchSet() 41 // Attempt to lookup AuthToken as a Node.SecretID since nodes may call 42 // call this endpoint and don't have an ACL token. 43 node, stateErr := srv.fsm.State().NodeBySecretID(ws, args.AuthToken) 44 if stateErr != nil { 45 // Return the original ResolveToken error with this err 46 var merr multierror.Error 47 merr.Errors = append(merr.Errors, err, stateErr) 48 return nil, merr.ErrorOrNil() 49 } 50 51 // We did not find a Node for this ID, so return Token Not Found. 52 if node == nil { 53 return nil, structs.ErrTokenNotFound 54 } 55 } 56 57 // Return either the users aclObj, or nil if ACLs are disabled. 58 return aclObj, nil 59 } 60 61 // WriteACLObj calls QueryACLObj for a WriteRequest 62 func (srv *Server) WriteACLObj(args *structs.WriteRequest, allowNodeAccess bool) (*acl.ACL, error) { 63 opts := &structs.QueryOptions{ 64 Region: args.RequestRegion(), 65 Namespace: args.RequestNamespace(), 66 AuthToken: args.AuthToken, 67 } 68 return srv.QueryACLObj(opts, allowNodeAccess) 69 } 70 71 const ( 72 csiVolumeTable = "csi_volumes" 73 csiPluginTable = "csi_plugins" 74 ) 75 76 // replySetIndex sets the reply with the last index that modified the table 77 func (srv *Server) replySetIndex(table string, reply *structs.QueryMeta) error { 78 s := srv.fsm.State() 79 80 index, err := s.Index(table) 81 if err != nil { 82 return err 83 } 84 reply.Index = index 85 86 // Set the query response 87 srv.setQueryMeta(reply) 88 return nil 89 } 90 91 // List replies with CSIVolumes, filtered by ACL access 92 func (v *CSIVolume) List(args *structs.CSIVolumeListRequest, reply *structs.CSIVolumeListResponse) error { 93 if done, err := v.srv.forward("CSIVolume.List", args, args, reply); done { 94 return err 95 } 96 97 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIListVolume, 98 acl.NamespaceCapabilityCSIReadVolume, 99 acl.NamespaceCapabilityCSIMountVolume, 100 acl.NamespaceCapabilityListJobs) 101 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) 102 if err != nil { 103 return err 104 } 105 106 if !allowVolume(aclObj, args.RequestNamespace()) { 107 return structs.ErrPermissionDenied 108 } 109 110 metricsStart := time.Now() 111 defer metrics.MeasureSince([]string{"nomad", "volume", "list"}, metricsStart) 112 113 ns := args.RequestNamespace() 114 opts := blockingOptions{ 115 queryOpts: &args.QueryOptions, 116 queryMeta: &reply.QueryMeta, 117 run: func(ws memdb.WatchSet, state *state.StateStore) error { 118 // Query all volumes 119 var err error 120 var iter memdb.ResultIterator 121 122 if args.NodeID != "" { 123 iter, err = state.CSIVolumesByNodeID(ws, args.NodeID) 124 } else if args.PluginID != "" { 125 iter, err = state.CSIVolumesByPluginID(ws, ns, args.PluginID) 126 } else { 127 iter, err = state.CSIVolumesByNamespace(ws, ns) 128 } 129 130 if err != nil { 131 return err 132 } 133 134 // Collect results, filter by ACL access 135 vs := []*structs.CSIVolListStub{} 136 137 for { 138 raw := iter.Next() 139 if raw == nil { 140 break 141 } 142 143 vol := raw.(*structs.CSIVolume) 144 vol, err := state.CSIVolumeDenormalizePlugins(ws, vol.Copy()) 145 if err != nil { 146 return err 147 } 148 149 // Remove (possibly again) by PluginID to handle passing both NodeID and PluginID 150 if args.PluginID != "" && args.PluginID != vol.PluginID { 151 continue 152 } 153 154 // Remove by Namespace, since CSIVolumesByNodeID hasn't used the Namespace yet 155 if vol.Namespace != ns { 156 continue 157 } 158 159 vs = append(vs, vol.Stub()) 160 } 161 reply.Volumes = vs 162 return v.srv.replySetIndex(csiVolumeTable, &reply.QueryMeta) 163 }} 164 return v.srv.blockingRPC(&opts) 165 } 166 167 // Get fetches detailed information about a specific volume 168 func (v *CSIVolume) Get(args *structs.CSIVolumeGetRequest, reply *structs.CSIVolumeGetResponse) error { 169 if done, err := v.srv.forward("CSIVolume.Get", args, args, reply); done { 170 return err 171 } 172 173 allowCSIAccess := acl.NamespaceValidator(acl.NamespaceCapabilityCSIReadVolume, 174 acl.NamespaceCapabilityCSIMountVolume, 175 acl.NamespaceCapabilityReadJob) 176 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, true) 177 if err != nil { 178 return err 179 } 180 181 ns := args.RequestNamespace() 182 if !allowCSIAccess(aclObj, ns) { 183 return structs.ErrPermissionDenied 184 } 185 186 metricsStart := time.Now() 187 defer metrics.MeasureSince([]string{"nomad", "volume", "get"}, metricsStart) 188 189 opts := blockingOptions{ 190 queryOpts: &args.QueryOptions, 191 queryMeta: &reply.QueryMeta, 192 run: func(ws memdb.WatchSet, state *state.StateStore) error { 193 vol, err := state.CSIVolumeByID(ws, ns, args.ID) 194 if err != nil { 195 return err 196 } 197 if vol != nil { 198 vol, err = state.CSIVolumeDenormalize(ws, vol) 199 } 200 if err != nil { 201 return err 202 } 203 204 reply.Volume = vol 205 return v.srv.replySetIndex(csiVolumeTable, &reply.QueryMeta) 206 }} 207 return v.srv.blockingRPC(&opts) 208 } 209 210 func (v *CSIVolume) pluginValidateVolume(req *structs.CSIVolumeRegisterRequest, vol *structs.CSIVolume) (*structs.CSIPlugin, error) { 211 state := v.srv.fsm.State() 212 ws := memdb.NewWatchSet() 213 214 plugin, err := state.CSIPluginByID(ws, vol.PluginID) 215 if err != nil { 216 return nil, err 217 } 218 if plugin == nil { 219 return nil, fmt.Errorf("no CSI plugin named: %s could be found", vol.PluginID) 220 } 221 222 vol.Provider = plugin.Provider 223 vol.ProviderVersion = plugin.Version 224 return plugin, nil 225 } 226 227 func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterRequest, vol *structs.CSIVolume, plugin *structs.CSIPlugin) error { 228 229 if !plugin.ControllerRequired { 230 // The plugin does not require a controller, so for now we won't do any 231 // further validation of the volume. 232 return nil 233 } 234 235 method := "ClientCSI.ControllerValidateVolume" 236 cReq := &cstructs.ClientCSIControllerValidateVolumeRequest{ 237 VolumeID: vol.RemoteID(), 238 AttachmentMode: vol.AttachmentMode, 239 AccessMode: vol.AccessMode, 240 Secrets: vol.Secrets, 241 // Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670 242 } 243 cReq.PluginID = plugin.ID 244 cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{} 245 246 return v.srv.RPC(method, cReq, cResp) 247 } 248 249 // Register registers a new volume 250 func (v *CSIVolume) Register(args *structs.CSIVolumeRegisterRequest, reply *structs.CSIVolumeRegisterResponse) error { 251 if done, err := v.srv.forward("CSIVolume.Register", args, args, reply); done { 252 return err 253 } 254 255 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIWriteVolume) 256 aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, false) 257 if err != nil { 258 return err 259 } 260 261 metricsStart := time.Now() 262 defer metrics.MeasureSince([]string{"nomad", "volume", "register"}, metricsStart) 263 264 if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { 265 return structs.ErrPermissionDenied 266 } 267 268 // This is the only namespace we ACL checked, force all the volumes to use it. 269 // We also validate that the plugin exists for each plugin, and validate the 270 // capabilities when the plugin has a controller. 271 for _, vol := range args.Volumes { 272 vol.Namespace = args.RequestNamespace() 273 if err = vol.Validate(); err != nil { 274 return err 275 } 276 277 plugin, err := v.pluginValidateVolume(args, vol) 278 if err != nil { 279 return err 280 } 281 if err := v.controllerValidateVolume(args, vol, plugin); err != nil { 282 return err 283 } 284 } 285 286 resp, index, err := v.srv.raftApply(structs.CSIVolumeRegisterRequestType, args) 287 if err != nil { 288 v.logger.Error("csi raft apply failed", "error", err, "method", "register") 289 return err 290 } 291 if respErr, ok := resp.(error); ok { 292 return respErr 293 } 294 295 reply.Index = index 296 v.srv.setQueryMeta(&reply.QueryMeta) 297 return nil 298 } 299 300 // Deregister removes a set of volumes 301 func (v *CSIVolume) Deregister(args *structs.CSIVolumeDeregisterRequest, reply *structs.CSIVolumeDeregisterResponse) error { 302 if done, err := v.srv.forward("CSIVolume.Deregister", args, args, reply); done { 303 return err 304 } 305 306 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIWriteVolume) 307 aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, false) 308 if err != nil { 309 return err 310 } 311 312 metricsStart := time.Now() 313 defer metrics.MeasureSince([]string{"nomad", "volume", "deregister"}, metricsStart) 314 315 ns := args.RequestNamespace() 316 if !allowVolume(aclObj, ns) { 317 return structs.ErrPermissionDenied 318 } 319 320 resp, index, err := v.srv.raftApply(structs.CSIVolumeDeregisterRequestType, args) 321 if err != nil { 322 v.logger.Error("csi raft apply failed", "error", err, "method", "deregister") 323 return err 324 } 325 if respErr, ok := resp.(error); ok { 326 return respErr 327 } 328 329 reply.Index = index 330 v.srv.setQueryMeta(&reply.QueryMeta) 331 return nil 332 } 333 334 // Claim submits a change to a volume claim 335 func (v *CSIVolume) Claim(args *structs.CSIVolumeClaimRequest, reply *structs.CSIVolumeClaimResponse) error { 336 if done, err := v.srv.forward("CSIVolume.Claim", args, args, reply); done { 337 return err 338 } 339 340 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIMountVolume) 341 aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, true) 342 if err != nil { 343 return err 344 } 345 346 metricsStart := time.Now() 347 defer metrics.MeasureSince([]string{"nomad", "volume", "claim"}, metricsStart) 348 349 if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { 350 return structs.ErrPermissionDenied 351 } 352 353 // COMPAT(1.0): the NodeID field was added after 0.11.0 and so we 354 // need to ensure it's been populated during upgrades from 0.11.0 355 // to later patch versions. Remove this block in 1.0 356 if args.Claim != structs.CSIVolumeClaimRelease && args.NodeID == "" { 357 state := v.srv.fsm.State() 358 ws := memdb.NewWatchSet() 359 alloc, err := state.AllocByID(ws, args.AllocationID) 360 if err != nil { 361 return err 362 } 363 if alloc == nil { 364 return fmt.Errorf("%s: %s", 365 structs.ErrUnknownAllocationPrefix, args.AllocationID) 366 } 367 args.NodeID = alloc.NodeID 368 } 369 370 if args.Claim != structs.CSIVolumeClaimRelease { 371 // if this is a new claim, add a Volume and PublishContext from the 372 // controller (if any) to the reply 373 err = v.controllerPublishVolume(args, reply) 374 if err != nil { 375 return fmt.Errorf("controller publish: %v", err) 376 } 377 } 378 resp, index, err := v.srv.raftApply(structs.CSIVolumeClaimRequestType, args) 379 if err != nil { 380 v.logger.Error("csi raft apply failed", "error", err, "method", "claim") 381 return err 382 } 383 if respErr, ok := resp.(error); ok { 384 return respErr 385 } 386 387 reply.Index = index 388 v.srv.setQueryMeta(&reply.QueryMeta) 389 return nil 390 } 391 392 // controllerPublishVolume sends publish request to the CSI controller 393 // plugin associated with a volume, if any. 394 func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest, resp *structs.CSIVolumeClaimResponse) error { 395 plug, vol, err := v.volAndPluginLookup(req.RequestNamespace(), req.VolumeID) 396 if err != nil { 397 return err 398 } 399 400 // Set the Response volume from the lookup 401 resp.Volume = vol 402 403 // Validate the existence of the allocation, regardless of whether we need it 404 // now. 405 state := v.srv.fsm.State() 406 ws := memdb.NewWatchSet() 407 alloc, err := state.AllocByID(ws, req.AllocationID) 408 if err != nil { 409 return err 410 } 411 if alloc == nil { 412 return fmt.Errorf("%s: %s", structs.ErrUnknownAllocationPrefix, req.AllocationID) 413 } 414 415 // if no plugin was returned then controller validation is not required. 416 // Here we can return nil. 417 if plug == nil { 418 return nil 419 } 420 421 // get Nomad's ID for the client node (not the storage provider's ID) 422 targetNode, err := state.NodeByID(ws, alloc.NodeID) 423 if err != nil { 424 return err 425 } 426 if targetNode == nil { 427 return fmt.Errorf("%s: %s", structs.ErrUnknownNodePrefix, alloc.NodeID) 428 } 429 430 // get the the storage provider's ID for the client node (not 431 // Nomad's ID for the node) 432 targetCSIInfo, ok := targetNode.CSINodePlugins[plug.ID] 433 if !ok { 434 return fmt.Errorf("Failed to find NodeInfo for node: %s", targetNode.ID) 435 } 436 externalNodeID := targetCSIInfo.NodeInfo.ID 437 438 method := "ClientCSI.ControllerAttachVolume" 439 cReq := &cstructs.ClientCSIControllerAttachVolumeRequest{ 440 VolumeID: vol.RemoteID(), 441 ClientCSINodeID: externalNodeID, 442 AttachmentMode: vol.AttachmentMode, 443 AccessMode: vol.AccessMode, 444 ReadOnly: req.Claim == structs.CSIVolumeClaimRead, 445 Secrets: vol.Secrets, 446 // VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771 447 } 448 cReq.PluginID = plug.ID 449 cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{} 450 451 err = v.srv.RPC(method, cReq, cResp) 452 if err != nil { 453 return fmt.Errorf("attach volume: %v", err) 454 } 455 resp.PublishContext = cResp.PublishContext 456 return nil 457 } 458 459 func (v *CSIVolume) volAndPluginLookup(namespace, volID string) (*structs.CSIPlugin, *structs.CSIVolume, error) { 460 state := v.srv.fsm.State() 461 ws := memdb.NewWatchSet() 462 463 vol, err := state.CSIVolumeByID(ws, namespace, volID) 464 if err != nil { 465 return nil, nil, err 466 } 467 if vol == nil { 468 return nil, nil, fmt.Errorf("volume not found: %s", volID) 469 } 470 if !vol.ControllerRequired { 471 return nil, vol, nil 472 } 473 474 // note: we do this same lookup in CSIVolumeByID but then throw 475 // away the pointer to the plugin rather than attaching it to 476 // the volume so we have to do it again here. 477 plug, err := state.CSIPluginByID(ws, vol.PluginID) 478 if err != nil { 479 return nil, nil, err 480 } 481 if plug == nil { 482 return nil, nil, fmt.Errorf("plugin not found: %s", vol.PluginID) 483 } 484 return plug, vol, nil 485 } 486 487 // allowCSIMount is called on Job register to check mount permission 488 func allowCSIMount(aclObj *acl.ACL, namespace string) bool { 489 return aclObj.AllowPluginRead() && 490 aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityCSIMountVolume) 491 } 492 493 // CSIPlugin wraps the structs.CSIPlugin with request data and server context 494 type CSIPlugin struct { 495 srv *Server 496 logger log.Logger 497 } 498 499 // List replies with CSIPlugins, filtered by ACL access 500 func (v *CSIPlugin) List(args *structs.CSIPluginListRequest, reply *structs.CSIPluginListResponse) error { 501 if done, err := v.srv.forward("CSIPlugin.List", args, args, reply); done { 502 return err 503 } 504 505 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) 506 if err != nil { 507 return err 508 } 509 510 if !aclObj.AllowPluginList() { 511 return structs.ErrPermissionDenied 512 } 513 514 metricsStart := time.Now() 515 defer metrics.MeasureSince([]string{"nomad", "plugin", "list"}, metricsStart) 516 517 opts := blockingOptions{ 518 queryOpts: &args.QueryOptions, 519 queryMeta: &reply.QueryMeta, 520 run: func(ws memdb.WatchSet, state *state.StateStore) error { 521 // Query all plugins 522 iter, err := state.CSIPlugins(ws) 523 if err != nil { 524 return err 525 } 526 527 // Collect results 528 ps := []*structs.CSIPluginListStub{} 529 for { 530 raw := iter.Next() 531 if raw == nil { 532 break 533 } 534 535 plug := raw.(*structs.CSIPlugin) 536 ps = append(ps, plug.Stub()) 537 } 538 539 reply.Plugins = ps 540 return v.srv.replySetIndex(csiPluginTable, &reply.QueryMeta) 541 }} 542 return v.srv.blockingRPC(&opts) 543 } 544 545 // Get fetches detailed information about a specific plugin 546 func (v *CSIPlugin) Get(args *structs.CSIPluginGetRequest, reply *structs.CSIPluginGetResponse) error { 547 if done, err := v.srv.forward("CSIPlugin.Get", args, args, reply); done { 548 return err 549 } 550 551 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) 552 if err != nil { 553 return err 554 } 555 556 if !aclObj.AllowPluginRead() { 557 return structs.ErrPermissionDenied 558 } 559 560 withAllocs := aclObj == nil || 561 aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) 562 563 metricsStart := time.Now() 564 defer metrics.MeasureSince([]string{"nomad", "plugin", "get"}, metricsStart) 565 566 opts := blockingOptions{ 567 queryOpts: &args.QueryOptions, 568 queryMeta: &reply.QueryMeta, 569 run: func(ws memdb.WatchSet, state *state.StateStore) error { 570 plug, err := state.CSIPluginByID(ws, args.ID) 571 if err != nil { 572 return err 573 } 574 575 if plug == nil { 576 return nil 577 } 578 579 if withAllocs { 580 plug, err = state.CSIPluginDenormalize(ws, plug.Copy()) 581 if err != nil { 582 return err 583 } 584 585 // Filter the allocation stubs by our namespace. withAllocs 586 // means we're allowed 587 var as []*structs.AllocListStub 588 for _, a := range plug.Allocations { 589 if a.Namespace == args.RequestNamespace() { 590 as = append(as, a) 591 } 592 } 593 plug.Allocations = as 594 } 595 596 reply.Plugin = plug 597 return v.srv.replySetIndex(csiPluginTable, &reply.QueryMeta) 598 }} 599 return v.srv.blockingRPC(&opts) 600 } 601 602 // Delete deletes a plugin if it is unused 603 func (v *CSIPlugin) Delete(args *structs.CSIPluginDeleteRequest, reply *structs.CSIPluginDeleteResponse) error { 604 if done, err := v.srv.forward("CSIPlugin.Delete", args, args, reply); done { 605 return err 606 } 607 608 // Check that it is a management token. 609 if aclObj, err := v.srv.ResolveToken(args.AuthToken); err != nil { 610 return err 611 } else if aclObj != nil && !aclObj.IsManagement() { 612 return structs.ErrPermissionDenied 613 } 614 615 metricsStart := time.Now() 616 defer metrics.MeasureSince([]string{"nomad", "plugin", "delete"}, metricsStart) 617 618 resp, index, err := v.srv.raftApply(structs.CSIPluginDeleteRequestType, args) 619 if err != nil { 620 v.logger.Error("csi raft apply failed", "error", err, "method", "delete") 621 return err 622 } 623 624 if respErr, ok := resp.(error); ok { 625 return respErr 626 } 627 628 reply.Index = index 629 v.srv.setQueryMeta(&reply.QueryMeta) 630 return nil 631 }