github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/client/allocrunner/csi_hook.go (about) 1 package allocrunner 2 3 import ( 4 "context" 5 "fmt" 6 7 hclog "github.com/hashicorp/go-hclog" 8 multierror "github.com/hashicorp/go-multierror" 9 "github.com/hashicorp/nomad/client/pluginmanager/csimanager" 10 "github.com/hashicorp/nomad/nomad/structs" 11 "github.com/hashicorp/nomad/plugins/drivers" 12 ) 13 14 // csiHook will wait for remote csi volumes to be attached to the host before 15 // continuing. 16 // 17 // It is a noop for allocs that do not depend on CSI Volumes. 18 type csiHook struct { 19 ar *allocRunner 20 alloc *structs.Allocation 21 logger hclog.Logger 22 csimanager csimanager.Manager 23 rpcClient RPCer 24 updater hookResourceSetter 25 volumeRequests map[string]*volumeAndRequest 26 } 27 28 func (c *csiHook) Name() string { 29 return "csi_hook" 30 } 31 32 func (c *csiHook) Prerun() error { 33 if !c.shouldRun() { 34 return nil 35 } 36 37 // We use this context only to attach hclog to the gRPC context. The 38 // lifetime is the lifetime of the gRPC stream, not specific RPC timeouts, 39 // but we manage the stream lifetime via Close in the pluginmanager. 40 ctx := context.Background() 41 42 volumes, err := c.claimVolumesFromAlloc() 43 if err != nil { 44 return fmt.Errorf("claim volumes: %v", err) 45 } 46 c.volumeRequests = volumes 47 48 mounts := make(map[string]*csimanager.MountInfo, len(volumes)) 49 for alias, pair := range volumes { 50 mounter, err := c.csimanager.MounterForPlugin(ctx, pair.volume.PluginID) 51 if err != nil { 52 return err 53 } 54 55 usageOpts := &csimanager.UsageOptions{ 56 ReadOnly: pair.request.ReadOnly, 57 AttachmentMode: string(pair.volume.AttachmentMode), 58 AccessMode: string(pair.volume.AccessMode), 59 MountOptions: pair.request.MountOptions, 60 } 61 62 mountInfo, err := mounter.MountVolume(ctx, pair.volume, c.alloc, usageOpts, pair.publishContext) 63 if err != nil { 64 return err 65 } 66 67 mounts[alias] = mountInfo 68 } 69 70 res := c.updater.GetAllocHookResources() 71 res.CSIMounts = mounts 72 c.updater.SetAllocHookResources(res) 73 74 return nil 75 } 76 77 // Postrun sends an RPC to the server to unpublish the volume. This may 78 // forward client RPCs to the node plugins or to the controller plugins, 79 // depending on whether other allocations on this node have claims on this 80 // volume. 81 func (c *csiHook) Postrun() error { 82 if !c.shouldRun() { 83 return nil 84 } 85 86 var mErr *multierror.Error 87 88 for _, pair := range c.volumeRequests { 89 90 mode := structs.CSIVolumeClaimRead 91 if !pair.request.ReadOnly { 92 mode = structs.CSIVolumeClaimWrite 93 } 94 95 req := &structs.CSIVolumeUnpublishRequest{ 96 VolumeID: pair.request.Source, 97 Claim: &structs.CSIVolumeClaim{ 98 AllocationID: c.alloc.ID, 99 NodeID: c.alloc.NodeID, 100 Mode: mode, 101 State: structs.CSIVolumeClaimStateUnpublishing, 102 }, 103 WriteRequest: structs.WriteRequest{ 104 Region: c.alloc.Job.Region, 105 Namespace: c.alloc.Job.Namespace, 106 AuthToken: c.ar.clientConfig.Node.SecretID, 107 }, 108 } 109 err := c.rpcClient.RPC("CSIVolume.Unpublish", 110 req, &structs.CSIVolumeUnpublishResponse{}) 111 if err != nil { 112 mErr = multierror.Append(mErr, err) 113 } 114 } 115 return mErr.ErrorOrNil() 116 } 117 118 type volumeAndRequest struct { 119 volume *structs.CSIVolume 120 request *structs.VolumeRequest 121 122 // When volumeAndRequest was returned from a volume claim, this field will be 123 // populated for plugins that require it. 124 publishContext map[string]string 125 } 126 127 // claimVolumesFromAlloc is used by the pre-run hook to fetch all of the volume 128 // metadata and claim it for use by this alloc/node at the same time. 129 func (c *csiHook) claimVolumesFromAlloc() (map[string]*volumeAndRequest, error) { 130 result := make(map[string]*volumeAndRequest) 131 tg := c.alloc.Job.LookupTaskGroup(c.alloc.TaskGroup) 132 133 // Initially, populate the result map with all of the requests 134 for alias, volumeRequest := range tg.Volumes { 135 136 if volumeRequest.Type == structs.VolumeTypeCSI { 137 138 for _, task := range tg.Tasks { 139 caps, err := c.ar.GetTaskDriverCapabilities(task.Name) 140 if err != nil { 141 return nil, fmt.Errorf("could not validate task driver capabilities: %v", err) 142 } 143 144 if caps.MountConfigs == drivers.MountConfigSupportNone { 145 return nil, fmt.Errorf( 146 "task driver %q for %q does not support CSI", task.Driver, task.Name) 147 } 148 } 149 150 result[alias] = &volumeAndRequest{request: volumeRequest} 151 } 152 } 153 154 // Iterate over the result map and upsert the volume field as each volume gets 155 // claimed by the server. 156 for alias, pair := range result { 157 claimType := structs.CSIVolumeClaimWrite 158 if pair.request.ReadOnly { 159 claimType = structs.CSIVolumeClaimRead 160 } 161 162 req := &structs.CSIVolumeClaimRequest{ 163 VolumeID: pair.request.Source, 164 AllocationID: c.alloc.ID, 165 NodeID: c.alloc.NodeID, 166 Claim: claimType, 167 WriteRequest: structs.WriteRequest{ 168 Region: c.alloc.Job.Region, 169 Namespace: c.alloc.Job.Namespace, 170 AuthToken: c.ar.clientConfig.Node.SecretID, 171 }, 172 } 173 174 var resp structs.CSIVolumeClaimResponse 175 if err := c.rpcClient.RPC("CSIVolume.Claim", req, &resp); err != nil { 176 return nil, err 177 } 178 179 if resp.Volume == nil { 180 return nil, fmt.Errorf("Unexpected nil volume returned for ID: %v", pair.request.Source) 181 } 182 183 result[alias].volume = resp.Volume 184 result[alias].publishContext = resp.PublishContext 185 } 186 187 return result, nil 188 } 189 190 func newCSIHook(ar *allocRunner, logger hclog.Logger, alloc *structs.Allocation, rpcClient RPCer, csi csimanager.Manager, updater hookResourceSetter) *csiHook { 191 return &csiHook{ 192 ar: ar, 193 alloc: alloc, 194 logger: logger.Named("csi_hook"), 195 rpcClient: rpcClient, 196 csimanager: csi, 197 updater: updater, 198 } 199 } 200 201 func (h *csiHook) shouldRun() bool { 202 tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup) 203 for _, vol := range tg.Volumes { 204 if vol.Type == structs.VolumeTypeCSI { 205 return true 206 } 207 } 208 209 return false 210 }