github.com/uchennaokeke444/nomad@v0.11.8/nomad/csi_endpoint.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "time" 6 7 metrics "github.com/armon/go-metrics" 8 log "github.com/hashicorp/go-hclog" 9 memdb "github.com/hashicorp/go-memdb" 10 multierror "github.com/hashicorp/go-multierror" 11 "github.com/hashicorp/nomad/acl" 12 cstructs "github.com/hashicorp/nomad/client/structs" 13 "github.com/hashicorp/nomad/nomad/state" 14 "github.com/hashicorp/nomad/nomad/structs" 15 ) 16 17 // CSIVolume wraps the structs.CSIVolume with request data and server context 18 type CSIVolume struct { 19 srv *Server 20 logger log.Logger 21 } 22 23 // QueryACLObj looks up the ACL token in the request and returns the acl.ACL object 24 // - fallback to node secret ids 25 func (srv *Server) QueryACLObj(args *structs.QueryOptions, allowNodeAccess bool) (*acl.ACL, error) { 26 // Lookup the token 27 aclObj, err := srv.ResolveToken(args.AuthToken) 28 if err != nil { 29 // If ResolveToken had an unexpected error return that 30 if !structs.IsErrTokenNotFound(err) { 31 return nil, err 32 } 33 34 // If we don't allow access to this endpoint from Nodes, then return token 35 // not found. 36 if !allowNodeAccess { 37 return nil, structs.ErrTokenNotFound 38 } 39 40 ws := memdb.NewWatchSet() 41 // Attempt to lookup AuthToken as a Node.SecretID since nodes may call 42 // call this endpoint and don't have an ACL token. 43 node, stateErr := srv.fsm.State().NodeBySecretID(ws, args.AuthToken) 44 if stateErr != nil { 45 // Return the original ResolveToken error with this err 46 var merr multierror.Error 47 merr.Errors = append(merr.Errors, err, stateErr) 48 return nil, merr.ErrorOrNil() 49 } 50 51 // We did not find a Node for this ID, so return Token Not Found. 52 if node == nil { 53 return nil, structs.ErrTokenNotFound 54 } 55 } 56 57 // Return either the users aclObj, or nil if ACLs are disabled. 58 return aclObj, nil 59 } 60 61 // WriteACLObj calls QueryACLObj for a WriteRequest 62 func (srv *Server) WriteACLObj(args *structs.WriteRequest, allowNodeAccess bool) (*acl.ACL, error) { 63 opts := &structs.QueryOptions{ 64 Region: args.RequestRegion(), 65 Namespace: args.RequestNamespace(), 66 AuthToken: args.AuthToken, 67 } 68 return srv.QueryACLObj(opts, allowNodeAccess) 69 } 70 71 const ( 72 csiVolumeTable = "csi_volumes" 73 csiPluginTable = "csi_plugins" 74 ) 75 76 // replySetIndex sets the reply with the last index that modified the table 77 func (srv *Server) replySetIndex(table string, reply *structs.QueryMeta) error { 78 s := srv.fsm.State() 79 80 index, err := s.Index(table) 81 if err != nil { 82 return err 83 } 84 reply.Index = index 85 86 // Set the query response 87 srv.setQueryMeta(reply) 88 return nil 89 } 90 91 // List replies with CSIVolumes, filtered by ACL access 92 func (v *CSIVolume) List(args *structs.CSIVolumeListRequest, reply *structs.CSIVolumeListResponse) error { 93 if done, err := v.srv.forward("CSIVolume.List", args, args, reply); done { 94 return err 95 } 96 97 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIListVolume, 98 acl.NamespaceCapabilityCSIReadVolume, 99 acl.NamespaceCapabilityCSIMountVolume, 100 acl.NamespaceCapabilityListJobs) 101 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) 102 if err != nil { 103 return err 104 } 105 106 if !allowVolume(aclObj, args.RequestNamespace()) { 107 return structs.ErrPermissionDenied 108 } 109 110 metricsStart := time.Now() 111 defer metrics.MeasureSince([]string{"nomad", "volume", "list"}, metricsStart) 112 113 ns := args.RequestNamespace() 114 opts := blockingOptions{ 115 queryOpts: &args.QueryOptions, 116 queryMeta: &reply.QueryMeta, 117 run: func(ws memdb.WatchSet, state *state.StateStore) error { 118 // Query all volumes 119 var err error 120 var iter memdb.ResultIterator 121 122 if args.NodeID != "" { 123 iter, err = state.CSIVolumesByNodeID(ws, args.NodeID) 124 } else if args.PluginID != "" { 125 iter, err = state.CSIVolumesByPluginID(ws, ns, args.PluginID) 126 } else { 127 iter, err = state.CSIVolumesByNamespace(ws, ns) 128 } 129 130 if err != nil { 131 return err 132 } 133 134 // Collect results, filter by ACL access 135 vs := []*structs.CSIVolListStub{} 136 137 for { 138 raw := iter.Next() 139 if raw == nil { 140 break 141 } 142 143 vol := raw.(*structs.CSIVolume) 144 vol, err := state.CSIVolumeDenormalizePlugins(ws, vol.Copy()) 145 if err != nil { 146 return err 147 } 148 149 // Remove (possibly again) by PluginID to handle passing both NodeID and PluginID 150 if args.PluginID != "" && args.PluginID != vol.PluginID { 151 continue 152 } 153 154 // Remove by Namespace, since CSIVolumesByNodeID hasn't used the Namespace yet 155 if vol.Namespace != ns { 156 continue 157 } 158 159 vs = append(vs, vol.Stub()) 160 } 161 reply.Volumes = vs 162 return v.srv.replySetIndex(csiVolumeTable, &reply.QueryMeta) 163 }} 164 return v.srv.blockingRPC(&opts) 165 } 166 167 // Get fetches detailed information about a specific volume 168 func (v *CSIVolume) Get(args *structs.CSIVolumeGetRequest, reply *structs.CSIVolumeGetResponse) error { 169 if done, err := v.srv.forward("CSIVolume.Get", args, args, reply); done { 170 return err 171 } 172 173 allowCSIAccess := acl.NamespaceValidator(acl.NamespaceCapabilityCSIReadVolume, 174 acl.NamespaceCapabilityCSIMountVolume, 175 acl.NamespaceCapabilityReadJob) 176 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, true) 177 if err != nil { 178 return err 179 } 180 181 ns := args.RequestNamespace() 182 if !allowCSIAccess(aclObj, ns) { 183 return structs.ErrPermissionDenied 184 } 185 186 metricsStart := time.Now() 187 defer metrics.MeasureSince([]string{"nomad", "volume", "get"}, metricsStart) 188 189 if args.ID == "" { 190 return fmt.Errorf("missing volume ID") 191 } 192 193 opts := blockingOptions{ 194 queryOpts: &args.QueryOptions, 195 queryMeta: &reply.QueryMeta, 196 run: func(ws memdb.WatchSet, state *state.StateStore) error { 197 vol, err := state.CSIVolumeByID(ws, ns, args.ID) 198 if err != nil { 199 return err 200 } 201 if vol != nil { 202 vol, err = state.CSIVolumeDenormalize(ws, vol) 203 } 204 if err != nil { 205 return err 206 } 207 208 reply.Volume = vol 209 return v.srv.replySetIndex(csiVolumeTable, &reply.QueryMeta) 210 }} 211 return v.srv.blockingRPC(&opts) 212 } 213 214 func (v *CSIVolume) pluginValidateVolume(req *structs.CSIVolumeRegisterRequest, vol *structs.CSIVolume) (*structs.CSIPlugin, error) { 215 state := v.srv.fsm.State() 216 ws := memdb.NewWatchSet() 217 218 plugin, err := state.CSIPluginByID(ws, vol.PluginID) 219 if err != nil { 220 return nil, err 221 } 222 if plugin == nil { 223 return nil, fmt.Errorf("no CSI plugin named: %s could be found", vol.PluginID) 224 } 225 226 vol.Provider = plugin.Provider 227 vol.ProviderVersion = plugin.Version 228 return plugin, nil 229 } 230 231 func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterRequest, vol *structs.CSIVolume, plugin *structs.CSIPlugin) error { 232 233 if !plugin.ControllerRequired { 234 // The plugin does not require a controller, so for now we won't do any 235 // further validation of the volume. 236 return nil 237 } 238 239 method := "ClientCSI.ControllerValidateVolume" 240 cReq := &cstructs.ClientCSIControllerValidateVolumeRequest{ 241 VolumeID: vol.RemoteID(), 242 AttachmentMode: vol.AttachmentMode, 243 AccessMode: vol.AccessMode, 244 Secrets: vol.Secrets, 245 Parameters: vol.Parameters, 246 Context: vol.Context, 247 } 248 cReq.PluginID = plugin.ID 249 cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{} 250 251 return v.srv.RPC(method, cReq, cResp) 252 } 253 254 // Register registers a new volume 255 func (v *CSIVolume) Register(args *structs.CSIVolumeRegisterRequest, reply *structs.CSIVolumeRegisterResponse) error { 256 if done, err := v.srv.forward("CSIVolume.Register", args, args, reply); done { 257 return err 258 } 259 260 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIWriteVolume) 261 aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, false) 262 if err != nil { 263 return err 264 } 265 266 metricsStart := time.Now() 267 defer metrics.MeasureSince([]string{"nomad", "volume", "register"}, metricsStart) 268 269 if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { 270 return structs.ErrPermissionDenied 271 } 272 273 if args.Volumes == nil || len(args.Volumes) == 0 { 274 return fmt.Errorf("missing volume definition") 275 } 276 277 // This is the only namespace we ACL checked, force all the volumes to use it. 278 // We also validate that the plugin exists for each plugin, and validate the 279 // capabilities when the plugin has a controller. 280 for _, vol := range args.Volumes { 281 vol.Namespace = args.RequestNamespace() 282 if err = vol.Validate(); err != nil { 283 return err 284 } 285 286 plugin, err := v.pluginValidateVolume(args, vol) 287 if err != nil { 288 return err 289 } 290 if err := v.controllerValidateVolume(args, vol, plugin); err != nil { 291 return err 292 } 293 } 294 295 resp, index, err := v.srv.raftApply(structs.CSIVolumeRegisterRequestType, args) 296 if err != nil { 297 v.logger.Error("csi raft apply failed", "error", err, "method", "register") 298 return err 299 } 300 if respErr, ok := resp.(error); ok { 301 return respErr 302 } 303 304 reply.Index = index 305 v.srv.setQueryMeta(&reply.QueryMeta) 306 return nil 307 } 308 309 // Deregister removes a set of volumes 310 func (v *CSIVolume) Deregister(args *structs.CSIVolumeDeregisterRequest, reply *structs.CSIVolumeDeregisterResponse) error { 311 if done, err := v.srv.forward("CSIVolume.Deregister", args, args, reply); done { 312 return err 313 } 314 315 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIWriteVolume) 316 aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, false) 317 if err != nil { 318 return err 319 } 320 321 metricsStart := time.Now() 322 defer metrics.MeasureSince([]string{"nomad", "volume", "deregister"}, metricsStart) 323 324 ns := args.RequestNamespace() 325 if !allowVolume(aclObj, ns) { 326 return structs.ErrPermissionDenied 327 } 328 329 if len(args.VolumeIDs) == 0 { 330 return fmt.Errorf("missing volume IDs") 331 } 332 333 resp, index, err := v.srv.raftApply(structs.CSIVolumeDeregisterRequestType, args) 334 if err != nil { 335 v.logger.Error("csi raft apply failed", "error", err, "method", "deregister") 336 return err 337 } 338 if respErr, ok := resp.(error); ok { 339 return respErr 340 } 341 342 reply.Index = index 343 v.srv.setQueryMeta(&reply.QueryMeta) 344 return nil 345 } 346 347 // Claim submits a change to a volume claim 348 func (v *CSIVolume) Claim(args *structs.CSIVolumeClaimRequest, reply *structs.CSIVolumeClaimResponse) error { 349 if done, err := v.srv.forward("CSIVolume.Claim", args, args, reply); done { 350 return err 351 } 352 353 allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIMountVolume) 354 aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, true) 355 if err != nil { 356 return err 357 } 358 359 metricsStart := time.Now() 360 defer metrics.MeasureSince([]string{"nomad", "volume", "claim"}, metricsStart) 361 362 if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { 363 return structs.ErrPermissionDenied 364 } 365 366 if args.VolumeID == "" { 367 return fmt.Errorf("missing volume ID") 368 } 369 370 // COMPAT(1.0): the NodeID field was added after 0.11.0 and so we 371 // need to ensure it's been populated during upgrades from 0.11.0 372 // to later patch versions. Remove this block in 1.0 373 if args.Claim != structs.CSIVolumeClaimRelease && args.NodeID == "" { 374 state := v.srv.fsm.State() 375 ws := memdb.NewWatchSet() 376 alloc, err := state.AllocByID(ws, args.AllocationID) 377 if err != nil { 378 return err 379 } 380 if alloc == nil { 381 return fmt.Errorf("%s: %s", 382 structs.ErrUnknownAllocationPrefix, args.AllocationID) 383 } 384 args.NodeID = alloc.NodeID 385 } 386 387 if args.Claim != structs.CSIVolumeClaimRelease { 388 // if this is a new claim, add a Volume and PublishContext from the 389 // controller (if any) to the reply 390 err = v.controllerPublishVolume(args, reply) 391 if err != nil { 392 return fmt.Errorf("controller publish: %v", err) 393 } 394 } 395 resp, index, err := v.srv.raftApply(structs.CSIVolumeClaimRequestType, args) 396 if err != nil { 397 v.logger.Error("csi raft apply failed", "error", err, "method", "claim") 398 return err 399 } 400 if respErr, ok := resp.(error); ok { 401 return respErr 402 } 403 404 reply.Index = index 405 v.srv.setQueryMeta(&reply.QueryMeta) 406 return nil 407 } 408 409 // controllerPublishVolume sends publish request to the CSI controller 410 // plugin associated with a volume, if any. 411 func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest, resp *structs.CSIVolumeClaimResponse) error { 412 plug, vol, err := v.volAndPluginLookup(req.RequestNamespace(), req.VolumeID) 413 if err != nil { 414 return err 415 } 416 417 // Set the Response volume from the lookup 418 resp.Volume = vol 419 420 // Validate the existence of the allocation, regardless of whether we need it 421 // now. 422 state := v.srv.fsm.State() 423 ws := memdb.NewWatchSet() 424 alloc, err := state.AllocByID(ws, req.AllocationID) 425 if err != nil { 426 return err 427 } 428 if alloc == nil { 429 return fmt.Errorf("%s: %s", structs.ErrUnknownAllocationPrefix, req.AllocationID) 430 } 431 432 // if no plugin was returned then controller validation is not required. 433 // Here we can return nil. 434 if plug == nil { 435 return nil 436 } 437 438 // get Nomad's ID for the client node (not the storage provider's ID) 439 targetNode, err := state.NodeByID(ws, alloc.NodeID) 440 if err != nil { 441 return err 442 } 443 if targetNode == nil { 444 return fmt.Errorf("%s: %s", structs.ErrUnknownNodePrefix, alloc.NodeID) 445 } 446 447 // get the the storage provider's ID for the client node (not 448 // Nomad's ID for the node) 449 targetCSIInfo, ok := targetNode.CSINodePlugins[plug.ID] 450 if !ok { 451 return fmt.Errorf("Failed to find NodeInfo for node: %s", targetNode.ID) 452 } 453 externalNodeID := targetCSIInfo.NodeInfo.ID 454 455 method := "ClientCSI.ControllerAttachVolume" 456 cReq := &cstructs.ClientCSIControllerAttachVolumeRequest{ 457 VolumeID: vol.RemoteID(), 458 ClientCSINodeID: externalNodeID, 459 AttachmentMode: vol.AttachmentMode, 460 AccessMode: vol.AccessMode, 461 ReadOnly: req.Claim == structs.CSIVolumeClaimRead, 462 Secrets: vol.Secrets, 463 VolumeContext: vol.Context, 464 } 465 cReq.PluginID = plug.ID 466 cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{} 467 468 err = v.srv.RPC(method, cReq, cResp) 469 if err != nil { 470 return fmt.Errorf("attach volume: %v", err) 471 } 472 resp.PublishContext = cResp.PublishContext 473 return nil 474 } 475 476 func (v *CSIVolume) volAndPluginLookup(namespace, volID string) (*structs.CSIPlugin, *structs.CSIVolume, error) { 477 state := v.srv.fsm.State() 478 ws := memdb.NewWatchSet() 479 480 vol, err := state.CSIVolumeByID(ws, namespace, volID) 481 if err != nil { 482 return nil, nil, err 483 } 484 if vol == nil { 485 return nil, nil, fmt.Errorf("volume not found: %s", volID) 486 } 487 if !vol.ControllerRequired { 488 return nil, vol, nil 489 } 490 491 // note: we do this same lookup in CSIVolumeByID but then throw 492 // away the pointer to the plugin rather than attaching it to 493 // the volume so we have to do it again here. 494 plug, err := state.CSIPluginByID(ws, vol.PluginID) 495 if err != nil { 496 return nil, nil, err 497 } 498 if plug == nil { 499 return nil, nil, fmt.Errorf("plugin not found: %s", vol.PluginID) 500 } 501 return plug, vol, nil 502 } 503 504 // allowCSIMount is called on Job register to check mount permission 505 func allowCSIMount(aclObj *acl.ACL, namespace string) bool { 506 return aclObj.AllowPluginRead() && 507 aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityCSIMountVolume) 508 } 509 510 // CSIPlugin wraps the structs.CSIPlugin with request data and server context 511 type CSIPlugin struct { 512 srv *Server 513 logger log.Logger 514 } 515 516 // List replies with CSIPlugins, filtered by ACL access 517 func (v *CSIPlugin) List(args *structs.CSIPluginListRequest, reply *structs.CSIPluginListResponse) error { 518 if done, err := v.srv.forward("CSIPlugin.List", args, args, reply); done { 519 return err 520 } 521 522 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) 523 if err != nil { 524 return err 525 } 526 527 if !aclObj.AllowPluginList() { 528 return structs.ErrPermissionDenied 529 } 530 531 metricsStart := time.Now() 532 defer metrics.MeasureSince([]string{"nomad", "plugin", "list"}, metricsStart) 533 534 opts := blockingOptions{ 535 queryOpts: &args.QueryOptions, 536 queryMeta: &reply.QueryMeta, 537 run: func(ws memdb.WatchSet, state *state.StateStore) error { 538 // Query all plugins 539 iter, err := state.CSIPlugins(ws) 540 if err != nil { 541 return err 542 } 543 544 // Collect results 545 ps := []*structs.CSIPluginListStub{} 546 for { 547 raw := iter.Next() 548 if raw == nil { 549 break 550 } 551 552 plug := raw.(*structs.CSIPlugin) 553 ps = append(ps, plug.Stub()) 554 } 555 556 reply.Plugins = ps 557 return v.srv.replySetIndex(csiPluginTable, &reply.QueryMeta) 558 }} 559 return v.srv.blockingRPC(&opts) 560 } 561 562 // Get fetches detailed information about a specific plugin 563 func (v *CSIPlugin) Get(args *structs.CSIPluginGetRequest, reply *structs.CSIPluginGetResponse) error { 564 if done, err := v.srv.forward("CSIPlugin.Get", args, args, reply); done { 565 return err 566 } 567 568 aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) 569 if err != nil { 570 return err 571 } 572 573 if !aclObj.AllowPluginRead() { 574 return structs.ErrPermissionDenied 575 } 576 577 withAllocs := aclObj == nil || 578 aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) 579 580 metricsStart := time.Now() 581 defer metrics.MeasureSince([]string{"nomad", "plugin", "get"}, metricsStart) 582 583 if args.ID == "" { 584 return fmt.Errorf("missing plugin ID") 585 } 586 587 opts := blockingOptions{ 588 queryOpts: &args.QueryOptions, 589 queryMeta: &reply.QueryMeta, 590 run: func(ws memdb.WatchSet, state *state.StateStore) error { 591 plug, err := state.CSIPluginByID(ws, args.ID) 592 if err != nil { 593 return err 594 } 595 596 if plug == nil { 597 return nil 598 } 599 600 if withAllocs { 601 plug, err = state.CSIPluginDenormalize(ws, plug.Copy()) 602 if err != nil { 603 return err 604 } 605 606 // Filter the allocation stubs by our namespace. withAllocs 607 // means we're allowed 608 var as []*structs.AllocListStub 609 for _, a := range plug.Allocations { 610 if a.Namespace == args.RequestNamespace() { 611 as = append(as, a) 612 } 613 } 614 plug.Allocations = as 615 } 616 617 reply.Plugin = plug 618 return v.srv.replySetIndex(csiPluginTable, &reply.QueryMeta) 619 }} 620 return v.srv.blockingRPC(&opts) 621 } 622 623 // Delete deletes a plugin if it is unused 624 func (v *CSIPlugin) Delete(args *structs.CSIPluginDeleteRequest, reply *structs.CSIPluginDeleteResponse) error { 625 if done, err := v.srv.forward("CSIPlugin.Delete", args, args, reply); done { 626 return err 627 } 628 629 // Check that it is a management token. 630 if aclObj, err := v.srv.ResolveToken(args.AuthToken); err != nil { 631 return err 632 } else if aclObj != nil && !aclObj.IsManagement() { 633 return structs.ErrPermissionDenied 634 } 635 636 metricsStart := time.Now() 637 defer metrics.MeasureSince([]string{"nomad", "plugin", "delete"}, metricsStart) 638 639 if args.ID == "" { 640 return fmt.Errorf("missing plugin ID") 641 } 642 643 resp, index, err := v.srv.raftApply(structs.CSIPluginDeleteRequestType, args) 644 if err != nil { 645 v.logger.Error("csi raft apply failed", "error", err, "method", "delete") 646 return err 647 } 648 649 if respErr, ok := resp.(error); ok { 650 return respErr 651 } 652 653 reply.Index = index 654 v.srv.setQueryMeta(&reply.QueryMeta) 655 return nil 656 }