github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/plugins/csi/plugin.go (about) 1 package csi 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 8 csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" 9 "github.com/hashicorp/nomad/nomad/structs" 10 "github.com/hashicorp/nomad/plugins/base" 11 "google.golang.org/grpc" 12 ) 13 14 // CSIPlugin implements a lightweight abstraction layer around a CSI Plugin. 15 // It validates that responses from storage providers (SP's), correctly conform 16 // to the specification before returning response data or erroring. 17 type CSIPlugin interface { 18 base.BasePlugin 19 20 // PluginProbe is used to verify that the plugin is in a healthy state 21 PluginProbe(ctx context.Context) (bool, error) 22 23 // PluginGetInfo is used to return semantic data about the plugin. 24 // Response: 25 // - string: name, the name of the plugin in domain notation format. 26 // - string: version, the vendor version of the plugin 27 PluginGetInfo(ctx context.Context) (string, string, error) 28 29 // PluginGetCapabilities is used to return the available capabilities from the 30 // identity service. This currently only looks for the CONTROLLER_SERVICE and 31 // Accessible Topology Support 32 PluginGetCapabilities(ctx context.Context) (*PluginCapabilitySet, error) 33 34 // GetControllerCapabilities is used to get controller-specific capabilities 35 // for a plugin. 36 ControllerGetCapabilities(ctx context.Context) (*ControllerCapabilitySet, error) 37 38 // ControllerPublishVolume is used to attach a remote volume to a cluster node. 39 ControllerPublishVolume(ctx context.Context, req *ControllerPublishVolumeRequest, opts ...grpc.CallOption) (*ControllerPublishVolumeResponse, error) 40 41 // ControllerUnpublishVolume is used to deattach a remote volume from a cluster node. 42 ControllerUnpublishVolume(ctx context.Context, req *ControllerUnpublishVolumeRequest, opts ...grpc.CallOption) (*ControllerUnpublishVolumeResponse, error) 43 44 // ControllerValidateCapabilities is used to validate that a volume exists and 45 // supports the requested capability. 46 ControllerValidateCapabilities(ctx context.Context, req *ControllerValidateVolumeRequest, opts ...grpc.CallOption) error 47 48 // NodeGetCapabilities is used to return the available capabilities from the 49 // Node Service. 50 NodeGetCapabilities(ctx context.Context) (*NodeCapabilitySet, error) 51 52 // NodeGetInfo is used to return semantic data about the current node in 53 // respect to the SP. 54 NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error) 55 56 // NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability 57 // to prepare a volume for usage on a host. If err == nil, the response should 58 // be assumed to be successful. 59 NodeStageVolume(ctx context.Context, req *NodeStageVolumeRequest, opts ...grpc.CallOption) error 60 61 // NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability 62 // to undo the work performed by NodeStageVolume. If a volume has been staged, 63 // this RPC must be called before freeing the volume. 64 // 65 // If err == nil, the response should be assumed to be successful. 66 NodeUnstageVolume(ctx context.Context, volumeID string, stagingTargetPath string, opts ...grpc.CallOption) error 67 68 // NodePublishVolume is used to prepare a volume for use by an allocation. 69 // if err == nil the response should be assumed to be successful. 70 NodePublishVolume(ctx context.Context, req *NodePublishVolumeRequest, opts ...grpc.CallOption) error 71 72 // NodeUnpublishVolume is used to cleanup usage of a volume for an alloc. This 73 // MUST be called before calling NodeUnstageVolume or ControllerUnpublishVolume 74 // for the given volume. 75 NodeUnpublishVolume(ctx context.Context, volumeID, targetPath string, opts ...grpc.CallOption) error 76 77 // Shutdown the client and ensure any connections are cleaned up. 78 Close() error 79 } 80 81 type NodePublishVolumeRequest struct { 82 // The external ID of the volume to publish. 83 ExternalID string 84 85 // If the volume was attached via a call to `ControllerPublishVolume` then 86 // we need to provide the returned PublishContext here. 87 PublishContext map[string]string 88 89 // The path to which the volume was staged by `NodeStageVolume`. 90 // It MUST be an absolute path in the root filesystem of the process 91 // serving this request. 92 // E.g {the plugins internal mount path}/staging/volumeid/... 93 // 94 // It MUST be set if the Node Plugin implements the 95 // `STAGE_UNSTAGE_VOLUME` node capability. 96 StagingTargetPath string 97 98 // The path to which the volume will be published. 99 // It MUST be an absolute path in the root filesystem of the process serving this 100 // request. 101 // E.g {the plugins internal mount path}/per-alloc/allocid/volumeid/... 102 // 103 // The CO SHALL ensure uniqueness of target_path per volume. 104 // The CO SHALL ensure that the parent directory of this path exists 105 // and that the process serving the request has `read` and `write` 106 // permissions to that parent directory. 107 TargetPath string 108 109 // Volume capability describing how the CO intends to use this volume. 110 VolumeCapability *VolumeCapability 111 112 Readonly bool 113 114 // Secrets required by plugins to complete the node publish volume 115 // request. This field is OPTIONAL. 116 Secrets structs.CSISecrets 117 118 // Volume context as returned by SP in the CSI 119 // CreateVolumeResponse.Volume.volume_context which we don't implement but 120 // can be entered by hand in the volume spec. This field is OPTIONAL. 121 VolumeContext map[string]string 122 } 123 124 func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest { 125 if r == nil { 126 return nil 127 } 128 129 return &csipbv1.NodePublishVolumeRequest{ 130 VolumeId: r.ExternalID, 131 PublishContext: r.PublishContext, 132 StagingTargetPath: r.StagingTargetPath, 133 TargetPath: r.TargetPath, 134 VolumeCapability: r.VolumeCapability.ToCSIRepresentation(), 135 Readonly: r.Readonly, 136 Secrets: r.Secrets, 137 VolumeContext: r.VolumeContext, 138 } 139 } 140 141 func (r *NodePublishVolumeRequest) Validate() error { 142 if r.ExternalID == "" { 143 return errors.New("missing volume ID") 144 } 145 146 if r.TargetPath == "" { 147 return errors.New("missing TargetPath") 148 } 149 150 if r.VolumeCapability == nil { 151 return errors.New("missing VolumeCapabilities") 152 } 153 154 return nil 155 } 156 157 type NodeStageVolumeRequest struct { 158 // The external ID of the volume to stage. 159 ExternalID string 160 161 // If the volume was attached via a call to `ControllerPublishVolume` then 162 // we need to provide the returned PublishContext here. 163 PublishContext map[string]string 164 165 // The path to which the volume MAY be staged. It MUST be an 166 // absolute path in the root filesystem of the process serving this 167 // request, and MUST be a directory. The CO SHALL ensure that there 168 // is only one `staging_target_path` per volume. The CO SHALL ensure 169 // that the path is directory and that the process serving the 170 // request has `read` and `write` permission to that directory. The 171 // CO SHALL be responsible for creating the directory if it does not 172 // exist. 173 // This is a REQUIRED field. 174 StagingTargetPath string 175 176 // Volume capability describing how the CO intends to use this volume. 177 VolumeCapability *VolumeCapability 178 179 // Secrets required by plugins to complete the node stage volume 180 // request. This field is OPTIONAL. 181 Secrets structs.CSISecrets 182 183 // Volume context as returned by SP in the CSI 184 // CreateVolumeResponse.Volume.volume_context which we don't implement but 185 // can be entered by hand in the volume spec. This field is OPTIONAL. 186 VolumeContext map[string]string 187 } 188 189 func (r *NodeStageVolumeRequest) ToCSIRepresentation() *csipbv1.NodeStageVolumeRequest { 190 if r == nil { 191 return nil 192 } 193 194 return &csipbv1.NodeStageVolumeRequest{ 195 VolumeId: r.ExternalID, 196 PublishContext: r.PublishContext, 197 StagingTargetPath: r.StagingTargetPath, 198 VolumeCapability: r.VolumeCapability.ToCSIRepresentation(), 199 Secrets: r.Secrets, 200 VolumeContext: r.VolumeContext, 201 } 202 } 203 204 func (r *NodeStageVolumeRequest) Validate() error { 205 if r.ExternalID == "" { 206 return errors.New("missing volume ID") 207 } 208 209 if r.StagingTargetPath == "" { 210 return errors.New("missing StagingTargetPath") 211 } 212 213 if r.VolumeCapability == nil { 214 return errors.New("missing VolumeCapabilities") 215 } 216 217 return nil 218 } 219 220 type PluginCapabilitySet struct { 221 hasControllerService bool 222 hasTopologies bool 223 } 224 225 func (p *PluginCapabilitySet) HasControllerService() bool { 226 return p.hasControllerService 227 } 228 229 // HasTopologies indicates whether the volumes for this plugin are equally 230 // accessible by all nodes in the cluster. 231 // If true, we MUST use the topology information when scheduling workloads. 232 func (p *PluginCapabilitySet) HasToplogies() bool { 233 return p.hasTopologies 234 } 235 236 func (p *PluginCapabilitySet) IsEqual(o *PluginCapabilitySet) bool { 237 return p.hasControllerService == o.hasControllerService && p.hasTopologies == o.hasTopologies 238 } 239 240 func NewTestPluginCapabilitySet(topologies, controller bool) *PluginCapabilitySet { 241 return &PluginCapabilitySet{ 242 hasTopologies: topologies, 243 hasControllerService: controller, 244 } 245 } 246 247 func NewPluginCapabilitySet(capabilities *csipbv1.GetPluginCapabilitiesResponse) *PluginCapabilitySet { 248 cs := &PluginCapabilitySet{} 249 250 pluginCapabilities := capabilities.GetCapabilities() 251 252 for _, pcap := range pluginCapabilities { 253 if svcCap := pcap.GetService(); svcCap != nil { 254 switch svcCap.Type { 255 case csipbv1.PluginCapability_Service_UNKNOWN: 256 continue 257 case csipbv1.PluginCapability_Service_CONTROLLER_SERVICE: 258 cs.hasControllerService = true 259 case csipbv1.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS: 260 cs.hasTopologies = true 261 default: 262 continue 263 } 264 } 265 } 266 267 return cs 268 } 269 270 type ControllerCapabilitySet struct { 271 HasPublishUnpublishVolume bool 272 HasPublishReadonly bool 273 HasListVolumes bool 274 HasListVolumesPublishedNodes bool 275 } 276 277 func NewControllerCapabilitySet(resp *csipbv1.ControllerGetCapabilitiesResponse) *ControllerCapabilitySet { 278 cs := &ControllerCapabilitySet{} 279 280 pluginCapabilities := resp.GetCapabilities() 281 for _, pcap := range pluginCapabilities { 282 if c := pcap.GetRpc(); c != nil { 283 switch c.Type { 284 case csipbv1.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME: 285 cs.HasPublishUnpublishVolume = true 286 case csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY: 287 cs.HasPublishReadonly = true 288 case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES: 289 cs.HasListVolumes = true 290 case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES: 291 cs.HasListVolumesPublishedNodes = true 292 default: 293 continue 294 } 295 } 296 } 297 298 return cs 299 } 300 301 type ControllerValidateVolumeRequest struct { 302 ExternalID string 303 Secrets structs.CSISecrets 304 Capabilities *VolumeCapability 305 Parameters map[string]string 306 Context map[string]string 307 } 308 309 func (r *ControllerValidateVolumeRequest) ToCSIRepresentation() *csipbv1.ValidateVolumeCapabilitiesRequest { 310 if r == nil { 311 return nil 312 } 313 314 return &csipbv1.ValidateVolumeCapabilitiesRequest{ 315 VolumeId: r.ExternalID, 316 VolumeContext: r.Context, 317 VolumeCapabilities: []*csipbv1.VolumeCapability{ 318 r.Capabilities.ToCSIRepresentation(), 319 }, 320 Parameters: r.Parameters, 321 Secrets: r.Secrets, 322 } 323 } 324 325 type ControllerPublishVolumeRequest struct { 326 ExternalID string 327 NodeID string 328 ReadOnly bool 329 VolumeCapability *VolumeCapability 330 Secrets structs.CSISecrets 331 VolumeContext map[string]string 332 } 333 334 func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerPublishVolumeRequest { 335 if r == nil { 336 return nil 337 } 338 339 return &csipbv1.ControllerPublishVolumeRequest{ 340 VolumeId: r.ExternalID, 341 NodeId: r.NodeID, 342 Readonly: r.ReadOnly, 343 VolumeCapability: r.VolumeCapability.ToCSIRepresentation(), 344 Secrets: r.Secrets, 345 VolumeContext: r.VolumeContext, 346 } 347 } 348 349 func (r *ControllerPublishVolumeRequest) Validate() error { 350 if r.ExternalID == "" { 351 return errors.New("missing volume ID") 352 } 353 if r.NodeID == "" { 354 return errors.New("missing NodeID") 355 } 356 return nil 357 } 358 359 type ControllerPublishVolumeResponse struct { 360 PublishContext map[string]string 361 } 362 363 type ControllerUnpublishVolumeRequest struct { 364 ExternalID string 365 NodeID string 366 Secrets structs.CSISecrets 367 } 368 369 func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerUnpublishVolumeRequest { 370 if r == nil { 371 return nil 372 } 373 374 return &csipbv1.ControllerUnpublishVolumeRequest{ 375 VolumeId: r.ExternalID, 376 NodeId: r.NodeID, 377 Secrets: r.Secrets, 378 } 379 } 380 381 func (r *ControllerUnpublishVolumeRequest) Validate() error { 382 if r.ExternalID == "" { 383 return errors.New("missing ExternalID") 384 } 385 if r.NodeID == "" { 386 // the spec allows this but it would unpublish the 387 // volume from all nodes 388 return errors.New("missing NodeID") 389 } 390 return nil 391 } 392 393 type ControllerUnpublishVolumeResponse struct{} 394 395 type NodeCapabilitySet struct { 396 HasStageUnstageVolume bool 397 } 398 399 func NewNodeCapabilitySet(resp *csipbv1.NodeGetCapabilitiesResponse) *NodeCapabilitySet { 400 cs := &NodeCapabilitySet{} 401 pluginCapabilities := resp.GetCapabilities() 402 for _, pcap := range pluginCapabilities { 403 if c := pcap.GetRpc(); c != nil { 404 switch c.Type { 405 case csipbv1.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME: 406 cs.HasStageUnstageVolume = true 407 default: 408 continue 409 } 410 } 411 } 412 413 return cs 414 } 415 416 // VolumeAccessMode represents the desired access mode of the CSI Volume 417 type VolumeAccessMode csipbv1.VolumeCapability_AccessMode_Mode 418 419 var _ fmt.Stringer = VolumeAccessModeUnknown 420 421 var ( 422 VolumeAccessModeUnknown = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_UNKNOWN) 423 VolumeAccessModeSingleNodeWriter = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_WRITER) 424 VolumeAccessModeSingleNodeReaderOnly = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY) 425 VolumeAccessModeMultiNodeReaderOnly = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY) 426 VolumeAccessModeMultiNodeSingleWriter = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER) 427 VolumeAccessModeMultiNodeMultiWriter = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER) 428 ) 429 430 func (a VolumeAccessMode) String() string { 431 return a.ToCSIRepresentation().String() 432 } 433 434 func (a VolumeAccessMode) ToCSIRepresentation() csipbv1.VolumeCapability_AccessMode_Mode { 435 return csipbv1.VolumeCapability_AccessMode_Mode(a) 436 } 437 438 // VolumeAccessType represents the filesystem apis that the user intends to use 439 // with the volume. E.g whether it will be used as a block device or if they wish 440 // to have a mounted filesystem. 441 type VolumeAccessType int32 442 443 var _ fmt.Stringer = VolumeAccessTypeBlock 444 445 var ( 446 VolumeAccessTypeBlock VolumeAccessType = 1 447 VolumeAccessTypeMount VolumeAccessType = 2 448 ) 449 450 func (v VolumeAccessType) String() string { 451 if v == VolumeAccessTypeBlock { 452 return "VolumeAccessType.Block" 453 } else if v == VolumeAccessTypeMount { 454 return "VolumeAccessType.Mount" 455 } else { 456 return "VolumeAccessType.Unspecified" 457 } 458 } 459 460 // VolumeCapability describes the overall usage requirements for a given CSI Volume 461 type VolumeCapability struct { 462 AccessType VolumeAccessType 463 AccessMode VolumeAccessMode 464 465 // Indicate that the volume will be accessed via the filesystem API. 466 MountVolume *structs.CSIMountOptions 467 } 468 469 func VolumeCapabilityFromStructs(sAccessType structs.CSIVolumeAttachmentMode, sAccessMode structs.CSIVolumeAccessMode) (*VolumeCapability, error) { 470 var accessType VolumeAccessType 471 switch sAccessType { 472 case structs.CSIVolumeAttachmentModeBlockDevice: 473 accessType = VolumeAccessTypeBlock 474 case structs.CSIVolumeAttachmentModeFilesystem: 475 accessType = VolumeAccessTypeMount 476 default: 477 // These fields are validated during job submission, but here we perform a 478 // final check during transformation into the requisite CSI Data type to 479 // defend against development bugs and corrupted state - and incompatible 480 // nomad versions in the future. 481 return nil, fmt.Errorf("Unknown volume attachment mode: %s", sAccessType) 482 } 483 484 var accessMode VolumeAccessMode 485 switch sAccessMode { 486 case structs.CSIVolumeAccessModeSingleNodeReader: 487 accessMode = VolumeAccessModeSingleNodeReaderOnly 488 case structs.CSIVolumeAccessModeSingleNodeWriter: 489 accessMode = VolumeAccessModeSingleNodeWriter 490 case structs.CSIVolumeAccessModeMultiNodeMultiWriter: 491 accessMode = VolumeAccessModeMultiNodeMultiWriter 492 case structs.CSIVolumeAccessModeMultiNodeSingleWriter: 493 accessMode = VolumeAccessModeMultiNodeSingleWriter 494 case structs.CSIVolumeAccessModeMultiNodeReader: 495 accessMode = VolumeAccessModeMultiNodeReaderOnly 496 default: 497 // These fields are validated during job submission, but here we perform a 498 // final check during transformation into the requisite CSI Data type to 499 // defend against development bugs and corrupted state - and incompatible 500 // nomad versions in the future. 501 return nil, fmt.Errorf("Unknown volume access mode: %v", sAccessMode) 502 } 503 504 return &VolumeCapability{ 505 AccessType: accessType, 506 AccessMode: accessMode, 507 }, nil 508 } 509 510 func (c *VolumeCapability) ToCSIRepresentation() *csipbv1.VolumeCapability { 511 if c == nil { 512 return nil 513 } 514 515 vc := &csipbv1.VolumeCapability{ 516 AccessMode: &csipbv1.VolumeCapability_AccessMode{ 517 Mode: c.AccessMode.ToCSIRepresentation(), 518 }, 519 } 520 521 if c.AccessType == VolumeAccessTypeMount { 522 opts := &csipbv1.VolumeCapability_MountVolume{} 523 if c.MountVolume != nil { 524 opts.FsType = c.MountVolume.FSType 525 opts.MountFlags = c.MountVolume.MountFlags 526 } 527 vc.AccessType = &csipbv1.VolumeCapability_Mount{Mount: opts} 528 } else { 529 vc.AccessType = &csipbv1.VolumeCapability_Block{Block: &csipbv1.VolumeCapability_BlockVolume{}} 530 } 531 532 return vc 533 }