github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/client/allocrunner/csi_hook.go (about)

     1  package allocrunner
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  
     7  	hclog "github.com/hashicorp/go-hclog"
     8  	multierror "github.com/hashicorp/go-multierror"
     9  	"github.com/hashicorp/nomad/client/pluginmanager/csimanager"
    10  	"github.com/hashicorp/nomad/nomad/structs"
    11  	"github.com/hashicorp/nomad/plugins/drivers"
    12  )
    13  
    14  // csiHook will wait for remote csi volumes to be attached to the host before
    15  // continuing.
    16  //
    17  // It is a noop for allocs that do not depend on CSI Volumes.
    18  type csiHook struct {
    19  	ar             *allocRunner
    20  	alloc          *structs.Allocation
    21  	logger         hclog.Logger
    22  	csimanager     csimanager.Manager
    23  	rpcClient      RPCer
    24  	updater        hookResourceSetter
    25  	volumeRequests map[string]*volumeAndRequest
    26  }
    27  
    28  func (c *csiHook) Name() string {
    29  	return "csi_hook"
    30  }
    31  
    32  func (c *csiHook) Prerun() error {
    33  	if !c.shouldRun() {
    34  		return nil
    35  	}
    36  
    37  	// We use this context only to attach hclog to the gRPC context. The
    38  	// lifetime is the lifetime of the gRPC stream, not specific RPC timeouts,
    39  	// but we manage the stream lifetime via Close in the pluginmanager.
    40  	ctx := context.Background()
    41  
    42  	volumes, err := c.claimVolumesFromAlloc()
    43  	if err != nil {
    44  		return fmt.Errorf("claim volumes: %v", err)
    45  	}
    46  	c.volumeRequests = volumes
    47  
    48  	mounts := make(map[string]*csimanager.MountInfo, len(volumes))
    49  	for alias, pair := range volumes {
    50  		mounter, err := c.csimanager.MounterForPlugin(ctx, pair.volume.PluginID)
    51  		if err != nil {
    52  			return err
    53  		}
    54  
    55  		usageOpts := &csimanager.UsageOptions{
    56  			ReadOnly:       pair.request.ReadOnly,
    57  			AttachmentMode: string(pair.volume.AttachmentMode),
    58  			AccessMode:     string(pair.volume.AccessMode),
    59  			MountOptions:   pair.request.MountOptions,
    60  		}
    61  
    62  		mountInfo, err := mounter.MountVolume(ctx, pair.volume, c.alloc, usageOpts, pair.publishContext)
    63  		if err != nil {
    64  			return err
    65  		}
    66  
    67  		mounts[alias] = mountInfo
    68  	}
    69  
    70  	res := c.updater.GetAllocHookResources()
    71  	res.CSIMounts = mounts
    72  	c.updater.SetAllocHookResources(res)
    73  
    74  	return nil
    75  }
    76  
    77  // Postrun sends an RPC to the server to unpublish the volume. This may
    78  // forward client RPCs to the node plugins or to the controller plugins,
    79  // depending on whether other allocations on this node have claims on this
    80  // volume.
    81  func (c *csiHook) Postrun() error {
    82  	if !c.shouldRun() {
    83  		return nil
    84  	}
    85  
    86  	var mErr *multierror.Error
    87  
    88  	for _, pair := range c.volumeRequests {
    89  
    90  		mode := structs.CSIVolumeClaimRead
    91  		if !pair.request.ReadOnly {
    92  			mode = structs.CSIVolumeClaimWrite
    93  		}
    94  
    95  		req := &structs.CSIVolumeUnpublishRequest{
    96  			VolumeID: pair.request.Source,
    97  			Claim: &structs.CSIVolumeClaim{
    98  				AllocationID: c.alloc.ID,
    99  				NodeID:       c.alloc.NodeID,
   100  				Mode:         mode,
   101  				State:        structs.CSIVolumeClaimStateUnpublishing,
   102  			},
   103  			WriteRequest: structs.WriteRequest{
   104  				Region:    c.alloc.Job.Region,
   105  				Namespace: c.alloc.Job.Namespace,
   106  				AuthToken: c.ar.clientConfig.Node.SecretID,
   107  			},
   108  		}
   109  		err := c.rpcClient.RPC("CSIVolume.Unpublish",
   110  			req, &structs.CSIVolumeUnpublishResponse{})
   111  		if err != nil {
   112  			mErr = multierror.Append(mErr, err)
   113  		}
   114  	}
   115  	return mErr.ErrorOrNil()
   116  }
   117  
   118  type volumeAndRequest struct {
   119  	volume  *structs.CSIVolume
   120  	request *structs.VolumeRequest
   121  
   122  	// When volumeAndRequest was returned from a volume claim, this field will be
   123  	// populated for plugins that require it.
   124  	publishContext map[string]string
   125  }
   126  
   127  // claimVolumesFromAlloc is used by the pre-run hook to fetch all of the volume
   128  // metadata and claim it for use by this alloc/node at the same time.
   129  func (c *csiHook) claimVolumesFromAlloc() (map[string]*volumeAndRequest, error) {
   130  	result := make(map[string]*volumeAndRequest)
   131  	tg := c.alloc.Job.LookupTaskGroup(c.alloc.TaskGroup)
   132  
   133  	// Initially, populate the result map with all of the requests
   134  	for alias, volumeRequest := range tg.Volumes {
   135  
   136  		if volumeRequest.Type == structs.VolumeTypeCSI {
   137  
   138  			for _, task := range tg.Tasks {
   139  				caps, err := c.ar.GetTaskDriverCapabilities(task.Name)
   140  				if err != nil {
   141  					return nil, fmt.Errorf("could not validate task driver capabilities: %v", err)
   142  				}
   143  
   144  				if caps.MountConfigs == drivers.MountConfigSupportNone {
   145  					return nil, fmt.Errorf(
   146  						"task driver %q for %q does not support CSI", task.Driver, task.Name)
   147  				}
   148  			}
   149  
   150  			result[alias] = &volumeAndRequest{request: volumeRequest}
   151  		}
   152  	}
   153  
   154  	// Iterate over the result map and upsert the volume field as each volume gets
   155  	// claimed by the server.
   156  	for alias, pair := range result {
   157  		claimType := structs.CSIVolumeClaimWrite
   158  		if pair.request.ReadOnly {
   159  			claimType = structs.CSIVolumeClaimRead
   160  		}
   161  
   162  		req := &structs.CSIVolumeClaimRequest{
   163  			VolumeID:     pair.request.Source,
   164  			AllocationID: c.alloc.ID,
   165  			NodeID:       c.alloc.NodeID,
   166  			Claim:        claimType,
   167  			WriteRequest: structs.WriteRequest{
   168  				Region:    c.alloc.Job.Region,
   169  				Namespace: c.alloc.Job.Namespace,
   170  				AuthToken: c.ar.clientConfig.Node.SecretID,
   171  			},
   172  		}
   173  
   174  		var resp structs.CSIVolumeClaimResponse
   175  		if err := c.rpcClient.RPC("CSIVolume.Claim", req, &resp); err != nil {
   176  			return nil, err
   177  		}
   178  
   179  		if resp.Volume == nil {
   180  			return nil, fmt.Errorf("Unexpected nil volume returned for ID: %v", pair.request.Source)
   181  		}
   182  
   183  		result[alias].volume = resp.Volume
   184  		result[alias].publishContext = resp.PublishContext
   185  	}
   186  
   187  	return result, nil
   188  }
   189  
   190  func newCSIHook(ar *allocRunner, logger hclog.Logger, alloc *structs.Allocation, rpcClient RPCer, csi csimanager.Manager, updater hookResourceSetter) *csiHook {
   191  	return &csiHook{
   192  		ar:         ar,
   193  		alloc:      alloc,
   194  		logger:     logger.Named("csi_hook"),
   195  		rpcClient:  rpcClient,
   196  		csimanager: csi,
   197  		updater:    updater,
   198  	}
   199  }
   200  
   201  func (h *csiHook) shouldRun() bool {
   202  	tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
   203  	for _, vol := range tg.Volumes {
   204  		if vol.Type == structs.VolumeTypeCSI {
   205  			return true
   206  		}
   207  	}
   208  
   209  	return false
   210  }