github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/plugins/csi/plugin.go (about)

     1  package csi
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  
     8  	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"
     9  	"github.com/hashicorp/nomad/nomad/structs"
    10  	"github.com/hashicorp/nomad/plugins/base"
    11  	"google.golang.org/grpc"
    12  )
    13  
    14  // CSIPlugin implements a lightweight abstraction layer around a CSI Plugin.
    15  // It validates that responses from storage providers (SP's), correctly conform
    16  // to the specification before returning response data or erroring.
    17  type CSIPlugin interface {
    18  	base.BasePlugin
    19  
    20  	// PluginProbe is used to verify that the plugin is in a healthy state
    21  	PluginProbe(ctx context.Context) (bool, error)
    22  
    23  	// PluginGetInfo is used to return semantic data about the plugin.
    24  	// Response:
    25  	//  - string: name, the name of the plugin in domain notation format.
    26  	//  - string: version, the vendor version of the plugin
    27  	PluginGetInfo(ctx context.Context) (string, string, error)
    28  
    29  	// PluginGetCapabilities is used to return the available capabilities from the
    30  	// identity service. This currently only looks for the CONTROLLER_SERVICE and
    31  	// Accessible Topology Support
    32  	PluginGetCapabilities(ctx context.Context) (*PluginCapabilitySet, error)
    33  
    34  	// GetControllerCapabilities is used to get controller-specific capabilities
    35  	// for a plugin.
    36  	ControllerGetCapabilities(ctx context.Context) (*ControllerCapabilitySet, error)
    37  
    38  	// ControllerPublishVolume is used to attach a remote volume to a cluster node.
    39  	ControllerPublishVolume(ctx context.Context, req *ControllerPublishVolumeRequest, opts ...grpc.CallOption) (*ControllerPublishVolumeResponse, error)
    40  
    41  	// ControllerUnpublishVolume is used to deattach a remote volume from a cluster node.
    42  	ControllerUnpublishVolume(ctx context.Context, req *ControllerUnpublishVolumeRequest, opts ...grpc.CallOption) (*ControllerUnpublishVolumeResponse, error)
    43  
    44  	// ControllerValidateCapabilities is used to validate that a volume exists and
    45  	// supports the requested capability.
    46  	ControllerValidateCapabilities(ctx context.Context, req *ControllerValidateVolumeRequest, opts ...grpc.CallOption) error
    47  
    48  	// NodeGetCapabilities is used to return the available capabilities from the
    49  	// Node Service.
    50  	NodeGetCapabilities(ctx context.Context) (*NodeCapabilitySet, error)
    51  
    52  	// NodeGetInfo is used to return semantic data about the current node in
    53  	// respect to the SP.
    54  	NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)
    55  
    56  	// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
    57  	// to prepare a volume for usage on a host. If err == nil, the response should
    58  	// be assumed to be successful.
    59  	NodeStageVolume(ctx context.Context, req *NodeStageVolumeRequest, opts ...grpc.CallOption) error
    60  
    61  	// NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
    62  	// to undo the work performed by NodeStageVolume. If a volume has been staged,
    63  	// this RPC must be called before freeing the volume.
    64  	//
    65  	// If err == nil, the response should be assumed to be successful.
    66  	NodeUnstageVolume(ctx context.Context, volumeID string, stagingTargetPath string, opts ...grpc.CallOption) error
    67  
    68  	// NodePublishVolume is used to prepare a volume for use by an allocation.
    69  	// if err == nil the response should be assumed to be successful.
    70  	NodePublishVolume(ctx context.Context, req *NodePublishVolumeRequest, opts ...grpc.CallOption) error
    71  
    72  	// NodeUnpublishVolume is used to cleanup usage of a volume for an alloc. This
    73  	// MUST be called before calling NodeUnstageVolume or ControllerUnpublishVolume
    74  	// for the given volume.
    75  	NodeUnpublishVolume(ctx context.Context, volumeID, targetPath string, opts ...grpc.CallOption) error
    76  
    77  	// Shutdown the client and ensure any connections are cleaned up.
    78  	Close() error
    79  }
    80  
    81  type NodePublishVolumeRequest struct {
    82  	// The external ID of the volume to publish.
    83  	ExternalID string
    84  
    85  	// If the volume was attached via a call to `ControllerPublishVolume` then
    86  	// we need to provide the returned PublishContext here.
    87  	PublishContext map[string]string
    88  
    89  	// The path to which the volume was staged by `NodeStageVolume`.
    90  	// It MUST be an absolute path in the root filesystem of the process
    91  	// serving this request.
    92  	// E.g {the plugins internal mount path}/staging/volumeid/...
    93  	//
    94  	// It MUST be set if the Node Plugin implements the
    95  	// `STAGE_UNSTAGE_VOLUME` node capability.
    96  	StagingTargetPath string
    97  
    98  	// The path to which the volume will be published.
    99  	// It MUST be an absolute path in the root filesystem of the process serving this
   100  	// request.
   101  	// E.g {the plugins internal mount path}/per-alloc/allocid/volumeid/...
   102  	//
   103  	// The CO SHALL ensure uniqueness of target_path per volume.
   104  	// The CO SHALL ensure that the parent directory of this path exists
   105  	// and that the process serving the request has `read` and `write`
   106  	// permissions to that parent directory.
   107  	TargetPath string
   108  
   109  	// Volume capability describing how the CO intends to use this volume.
   110  	VolumeCapability *VolumeCapability
   111  
   112  	Readonly bool
   113  
   114  	// Secrets required by plugins to complete the node publish volume
   115  	// request. This field is OPTIONAL.
   116  	Secrets structs.CSISecrets
   117  
   118  	// Volume context as returned by SP in the CSI
   119  	// CreateVolumeResponse.Volume.volume_context which we don't implement but
   120  	// can be entered by hand in the volume spec.  This field is OPTIONAL.
   121  	VolumeContext map[string]string
   122  }
   123  
   124  func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest {
   125  	if r == nil {
   126  		return nil
   127  	}
   128  
   129  	return &csipbv1.NodePublishVolumeRequest{
   130  		VolumeId:          r.ExternalID,
   131  		PublishContext:    r.PublishContext,
   132  		StagingTargetPath: r.StagingTargetPath,
   133  		TargetPath:        r.TargetPath,
   134  		VolumeCapability:  r.VolumeCapability.ToCSIRepresentation(),
   135  		Readonly:          r.Readonly,
   136  		Secrets:           r.Secrets,
   137  		VolumeContext:     r.VolumeContext,
   138  	}
   139  }
   140  
   141  func (r *NodePublishVolumeRequest) Validate() error {
   142  	if r.ExternalID == "" {
   143  		return errors.New("missing volume ID")
   144  	}
   145  
   146  	if r.TargetPath == "" {
   147  		return errors.New("missing TargetPath")
   148  	}
   149  
   150  	if r.VolumeCapability == nil {
   151  		return errors.New("missing VolumeCapabilities")
   152  	}
   153  
   154  	return nil
   155  }
   156  
   157  type NodeStageVolumeRequest struct {
   158  	// The external ID of the volume to stage.
   159  	ExternalID string
   160  
   161  	// If the volume was attached via a call to `ControllerPublishVolume` then
   162  	// we need to provide the returned PublishContext here.
   163  	PublishContext map[string]string
   164  
   165  	// The path to which the volume MAY be staged. It MUST be an
   166  	// absolute path in the root filesystem of the process serving this
   167  	// request, and MUST be a directory. The CO SHALL ensure that there
   168  	// is only one `staging_target_path` per volume. The CO SHALL ensure
   169  	// that the path is directory and that the process serving the
   170  	// request has `read` and `write` permission to that directory. The
   171  	// CO SHALL be responsible for creating the directory if it does not
   172  	// exist.
   173  	// This is a REQUIRED field.
   174  	StagingTargetPath string
   175  
   176  	// Volume capability describing how the CO intends to use this volume.
   177  	VolumeCapability *VolumeCapability
   178  
   179  	// Secrets required by plugins to complete the node stage volume
   180  	// request. This field is OPTIONAL.
   181  	Secrets structs.CSISecrets
   182  
   183  	// Volume context as returned by SP in the CSI
   184  	// CreateVolumeResponse.Volume.volume_context which we don't implement but
   185  	// can be entered by hand in the volume spec.  This field is OPTIONAL.
   186  	VolumeContext map[string]string
   187  }
   188  
   189  func (r *NodeStageVolumeRequest) ToCSIRepresentation() *csipbv1.NodeStageVolumeRequest {
   190  	if r == nil {
   191  		return nil
   192  	}
   193  
   194  	return &csipbv1.NodeStageVolumeRequest{
   195  		VolumeId:          r.ExternalID,
   196  		PublishContext:    r.PublishContext,
   197  		StagingTargetPath: r.StagingTargetPath,
   198  		VolumeCapability:  r.VolumeCapability.ToCSIRepresentation(),
   199  		Secrets:           r.Secrets,
   200  		VolumeContext:     r.VolumeContext,
   201  	}
   202  }
   203  
   204  func (r *NodeStageVolumeRequest) Validate() error {
   205  	if r.ExternalID == "" {
   206  		return errors.New("missing volume ID")
   207  	}
   208  
   209  	if r.StagingTargetPath == "" {
   210  		return errors.New("missing StagingTargetPath")
   211  	}
   212  
   213  	if r.VolumeCapability == nil {
   214  		return errors.New("missing VolumeCapabilities")
   215  	}
   216  
   217  	return nil
   218  }
   219  
   220  type PluginCapabilitySet struct {
   221  	hasControllerService bool
   222  	hasTopologies        bool
   223  }
   224  
   225  func (p *PluginCapabilitySet) HasControllerService() bool {
   226  	return p.hasControllerService
   227  }
   228  
   229  // HasTopologies indicates whether the volumes for this plugin are equally
   230  // accessible by all nodes in the cluster.
   231  // If true, we MUST use the topology information when scheduling workloads.
   232  func (p *PluginCapabilitySet) HasToplogies() bool {
   233  	return p.hasTopologies
   234  }
   235  
   236  func (p *PluginCapabilitySet) IsEqual(o *PluginCapabilitySet) bool {
   237  	return p.hasControllerService == o.hasControllerService && p.hasTopologies == o.hasTopologies
   238  }
   239  
   240  func NewTestPluginCapabilitySet(topologies, controller bool) *PluginCapabilitySet {
   241  	return &PluginCapabilitySet{
   242  		hasTopologies:        topologies,
   243  		hasControllerService: controller,
   244  	}
   245  }
   246  
   247  func NewPluginCapabilitySet(capabilities *csipbv1.GetPluginCapabilitiesResponse) *PluginCapabilitySet {
   248  	cs := &PluginCapabilitySet{}
   249  
   250  	pluginCapabilities := capabilities.GetCapabilities()
   251  
   252  	for _, pcap := range pluginCapabilities {
   253  		if svcCap := pcap.GetService(); svcCap != nil {
   254  			switch svcCap.Type {
   255  			case csipbv1.PluginCapability_Service_UNKNOWN:
   256  				continue
   257  			case csipbv1.PluginCapability_Service_CONTROLLER_SERVICE:
   258  				cs.hasControllerService = true
   259  			case csipbv1.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS:
   260  				cs.hasTopologies = true
   261  			default:
   262  				continue
   263  			}
   264  		}
   265  	}
   266  
   267  	return cs
   268  }
   269  
   270  type ControllerCapabilitySet struct {
   271  	HasPublishUnpublishVolume    bool
   272  	HasPublishReadonly           bool
   273  	HasListVolumes               bool
   274  	HasListVolumesPublishedNodes bool
   275  }
   276  
   277  func NewControllerCapabilitySet(resp *csipbv1.ControllerGetCapabilitiesResponse) *ControllerCapabilitySet {
   278  	cs := &ControllerCapabilitySet{}
   279  
   280  	pluginCapabilities := resp.GetCapabilities()
   281  	for _, pcap := range pluginCapabilities {
   282  		if c := pcap.GetRpc(); c != nil {
   283  			switch c.Type {
   284  			case csipbv1.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME:
   285  				cs.HasPublishUnpublishVolume = true
   286  			case csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY:
   287  				cs.HasPublishReadonly = true
   288  			case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES:
   289  				cs.HasListVolumes = true
   290  			case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES:
   291  				cs.HasListVolumesPublishedNodes = true
   292  			default:
   293  				continue
   294  			}
   295  		}
   296  	}
   297  
   298  	return cs
   299  }
   300  
   301  type ControllerValidateVolumeRequest struct {
   302  	ExternalID   string
   303  	Secrets      structs.CSISecrets
   304  	Capabilities *VolumeCapability
   305  	Parameters   map[string]string
   306  	Context      map[string]string
   307  }
   308  
   309  func (r *ControllerValidateVolumeRequest) ToCSIRepresentation() *csipbv1.ValidateVolumeCapabilitiesRequest {
   310  	if r == nil {
   311  		return nil
   312  	}
   313  
   314  	return &csipbv1.ValidateVolumeCapabilitiesRequest{
   315  		VolumeId:      r.ExternalID,
   316  		VolumeContext: r.Context,
   317  		VolumeCapabilities: []*csipbv1.VolumeCapability{
   318  			r.Capabilities.ToCSIRepresentation(),
   319  		},
   320  		Parameters: r.Parameters,
   321  		Secrets:    r.Secrets,
   322  	}
   323  }
   324  
   325  type ControllerPublishVolumeRequest struct {
   326  	ExternalID       string
   327  	NodeID           string
   328  	ReadOnly         bool
   329  	VolumeCapability *VolumeCapability
   330  	Secrets          structs.CSISecrets
   331  	VolumeContext    map[string]string
   332  }
   333  
   334  func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerPublishVolumeRequest {
   335  	if r == nil {
   336  		return nil
   337  	}
   338  
   339  	return &csipbv1.ControllerPublishVolumeRequest{
   340  		VolumeId:         r.ExternalID,
   341  		NodeId:           r.NodeID,
   342  		Readonly:         r.ReadOnly,
   343  		VolumeCapability: r.VolumeCapability.ToCSIRepresentation(),
   344  		Secrets:          r.Secrets,
   345  		VolumeContext:    r.VolumeContext,
   346  	}
   347  }
   348  
   349  func (r *ControllerPublishVolumeRequest) Validate() error {
   350  	if r.ExternalID == "" {
   351  		return errors.New("missing volume ID")
   352  	}
   353  	if r.NodeID == "" {
   354  		return errors.New("missing NodeID")
   355  	}
   356  	return nil
   357  }
   358  
   359  type ControllerPublishVolumeResponse struct {
   360  	PublishContext map[string]string
   361  }
   362  
   363  type ControllerUnpublishVolumeRequest struct {
   364  	ExternalID string
   365  	NodeID     string
   366  	Secrets    structs.CSISecrets
   367  }
   368  
   369  func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerUnpublishVolumeRequest {
   370  	if r == nil {
   371  		return nil
   372  	}
   373  
   374  	return &csipbv1.ControllerUnpublishVolumeRequest{
   375  		VolumeId: r.ExternalID,
   376  		NodeId:   r.NodeID,
   377  		Secrets:  r.Secrets,
   378  	}
   379  }
   380  
   381  func (r *ControllerUnpublishVolumeRequest) Validate() error {
   382  	if r.ExternalID == "" {
   383  		return errors.New("missing ExternalID")
   384  	}
   385  	if r.NodeID == "" {
   386  		// the spec allows this but it would unpublish the
   387  		// volume from all nodes
   388  		return errors.New("missing NodeID")
   389  	}
   390  	return nil
   391  }
   392  
   393  type ControllerUnpublishVolumeResponse struct{}
   394  
   395  type NodeCapabilitySet struct {
   396  	HasStageUnstageVolume bool
   397  }
   398  
   399  func NewNodeCapabilitySet(resp *csipbv1.NodeGetCapabilitiesResponse) *NodeCapabilitySet {
   400  	cs := &NodeCapabilitySet{}
   401  	pluginCapabilities := resp.GetCapabilities()
   402  	for _, pcap := range pluginCapabilities {
   403  		if c := pcap.GetRpc(); c != nil {
   404  			switch c.Type {
   405  			case csipbv1.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME:
   406  				cs.HasStageUnstageVolume = true
   407  			default:
   408  				continue
   409  			}
   410  		}
   411  	}
   412  
   413  	return cs
   414  }
   415  
   416  // VolumeAccessMode represents the desired access mode of the CSI Volume
   417  type VolumeAccessMode csipbv1.VolumeCapability_AccessMode_Mode
   418  
   419  var _ fmt.Stringer = VolumeAccessModeUnknown
   420  
   421  var (
   422  	VolumeAccessModeUnknown               = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_UNKNOWN)
   423  	VolumeAccessModeSingleNodeWriter      = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_WRITER)
   424  	VolumeAccessModeSingleNodeReaderOnly  = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY)
   425  	VolumeAccessModeMultiNodeReaderOnly   = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY)
   426  	VolumeAccessModeMultiNodeSingleWriter = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER)
   427  	VolumeAccessModeMultiNodeMultiWriter  = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER)
   428  )
   429  
   430  func (a VolumeAccessMode) String() string {
   431  	return a.ToCSIRepresentation().String()
   432  }
   433  
   434  func (a VolumeAccessMode) ToCSIRepresentation() csipbv1.VolumeCapability_AccessMode_Mode {
   435  	return csipbv1.VolumeCapability_AccessMode_Mode(a)
   436  }
   437  
   438  // VolumeAccessType represents the filesystem apis that the user intends to use
   439  // with the volume. E.g whether it will be used as a block device or if they wish
   440  // to have a mounted filesystem.
   441  type VolumeAccessType int32
   442  
   443  var _ fmt.Stringer = VolumeAccessTypeBlock
   444  
   445  var (
   446  	VolumeAccessTypeBlock VolumeAccessType = 1
   447  	VolumeAccessTypeMount VolumeAccessType = 2
   448  )
   449  
   450  func (v VolumeAccessType) String() string {
   451  	if v == VolumeAccessTypeBlock {
   452  		return "VolumeAccessType.Block"
   453  	} else if v == VolumeAccessTypeMount {
   454  		return "VolumeAccessType.Mount"
   455  	} else {
   456  		return "VolumeAccessType.Unspecified"
   457  	}
   458  }
   459  
   460  // VolumeCapability describes the overall usage requirements for a given CSI Volume
   461  type VolumeCapability struct {
   462  	AccessType VolumeAccessType
   463  	AccessMode VolumeAccessMode
   464  
   465  	// Indicate that the volume will be accessed via the filesystem API.
   466  	MountVolume *structs.CSIMountOptions
   467  }
   468  
   469  func VolumeCapabilityFromStructs(sAccessType structs.CSIVolumeAttachmentMode, sAccessMode structs.CSIVolumeAccessMode) (*VolumeCapability, error) {
   470  	var accessType VolumeAccessType
   471  	switch sAccessType {
   472  	case structs.CSIVolumeAttachmentModeBlockDevice:
   473  		accessType = VolumeAccessTypeBlock
   474  	case structs.CSIVolumeAttachmentModeFilesystem:
   475  		accessType = VolumeAccessTypeMount
   476  	default:
   477  		// These fields are validated during job submission, but here we perform a
   478  		// final check during transformation into the requisite CSI Data type to
   479  		// defend against development bugs and corrupted state - and incompatible
   480  		// nomad versions in the future.
   481  		return nil, fmt.Errorf("Unknown volume attachment mode: %s", sAccessType)
   482  	}
   483  
   484  	var accessMode VolumeAccessMode
   485  	switch sAccessMode {
   486  	case structs.CSIVolumeAccessModeSingleNodeReader:
   487  		accessMode = VolumeAccessModeSingleNodeReaderOnly
   488  	case structs.CSIVolumeAccessModeSingleNodeWriter:
   489  		accessMode = VolumeAccessModeSingleNodeWriter
   490  	case structs.CSIVolumeAccessModeMultiNodeMultiWriter:
   491  		accessMode = VolumeAccessModeMultiNodeMultiWriter
   492  	case structs.CSIVolumeAccessModeMultiNodeSingleWriter:
   493  		accessMode = VolumeAccessModeMultiNodeSingleWriter
   494  	case structs.CSIVolumeAccessModeMultiNodeReader:
   495  		accessMode = VolumeAccessModeMultiNodeReaderOnly
   496  	default:
   497  		// These fields are validated during job submission, but here we perform a
   498  		// final check during transformation into the requisite CSI Data type to
   499  		// defend against development bugs and corrupted state - and incompatible
   500  		// nomad versions in the future.
   501  		return nil, fmt.Errorf("Unknown volume access mode: %v", sAccessMode)
   502  	}
   503  
   504  	return &VolumeCapability{
   505  		AccessType: accessType,
   506  		AccessMode: accessMode,
   507  	}, nil
   508  }
   509  
   510  func (c *VolumeCapability) ToCSIRepresentation() *csipbv1.VolumeCapability {
   511  	if c == nil {
   512  		return nil
   513  	}
   514  
   515  	vc := &csipbv1.VolumeCapability{
   516  		AccessMode: &csipbv1.VolumeCapability_AccessMode{
   517  			Mode: c.AccessMode.ToCSIRepresentation(),
   518  		},
   519  	}
   520  
   521  	if c.AccessType == VolumeAccessTypeMount {
   522  		opts := &csipbv1.VolumeCapability_MountVolume{}
   523  		if c.MountVolume != nil {
   524  			opts.FsType = c.MountVolume.FSType
   525  			opts.MountFlags = c.MountVolume.MountFlags
   526  		}
   527  		vc.AccessType = &csipbv1.VolumeCapability_Mount{Mount: opts}
   528  	} else {
   529  		vc.AccessType = &csipbv1.VolumeCapability_Block{Block: &csipbv1.VolumeCapability_BlockVolume{}}
   530  	}
   531  
   532  	return vc
   533  }