github.com/mwhudson/juju@v0.0.0-20160512215208-90ff01f3497f/provider/openstack/cinder.go (about) 1 // Copyright 2015 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package openstack 5 6 import ( 7 "math" 8 "net/url" 9 "sync" 10 "time" 11 12 "github.com/juju/errors" 13 "github.com/juju/utils" 14 "gopkg.in/goose.v1/cinder" 15 "gopkg.in/goose.v1/identity" 16 "gopkg.in/goose.v1/nova" 17 18 "github.com/juju/juju/environs/config" 19 "github.com/juju/juju/environs/tags" 20 "github.com/juju/juju/instance" 21 "github.com/juju/juju/storage" 22 ) 23 24 const ( 25 CinderProviderType = storage.ProviderType("cinder") 26 // autoAssignedMountPoint specifies the value to pass in when 27 // you'd like Cinder to automatically assign a mount point. 28 autoAssignedMountPoint = "" 29 30 volumeStatusAvailable = "available" 31 volumeStatusDeleting = "deleting" 32 volumeStatusError = "error" 33 volumeStatusInUse = "in-use" 34 ) 35 36 type cinderProvider struct { 37 newStorageAdapter func(*config.Config) (openstackStorage, error) 38 } 39 40 var _ storage.Provider = (*cinderProvider)(nil) 41 42 var cinderAttempt = utils.AttemptStrategy{ 43 Total: 1 * time.Minute, 44 Delay: 5 * time.Second, 45 } 46 47 // VolumeSource implements storage.Provider. 48 func (p *cinderProvider) VolumeSource(environConfig *config.Config, providerConfig *storage.Config) (storage.VolumeSource, error) { 49 if err := p.ValidateConfig(providerConfig); err != nil { 50 return nil, err 51 } 52 storageAdapter, err := p.newStorageAdapter(environConfig) 53 if err != nil { 54 return nil, err 55 } 56 source := &cinderVolumeSource{ 57 storageAdapter: storageAdapter, 58 envName: environConfig.Name(), 59 modelUUID: environConfig.UUID(), 60 } 61 return source, nil 62 } 63 64 // FilesystemSource implements storage.Provider. 65 func (p *cinderProvider) FilesystemSource(environConfig *config.Config, providerConfig *storage.Config) (storage.FilesystemSource, error) { 66 return nil, errors.NotSupportedf("filesystems") 67 } 68 69 // Supports implements storage.Provider. 70 func (p *cinderProvider) Supports(kind storage.StorageKind) bool { 71 switch kind { 72 case storage.StorageKindBlock: 73 return true 74 } 75 return false 76 } 77 78 // Scope implements storage.Provider. 79 func (s *cinderProvider) Scope() storage.Scope { 80 return storage.ScopeEnviron 81 } 82 83 // ValidateConfig implements storage.Provider. 84 func (p *cinderProvider) ValidateConfig(cfg *storage.Config) error { 85 // TODO(axw) 2015-05-01 #1450737 86 // Reject attempts to create non-persistent volumes. 87 return nil 88 } 89 90 // Dynamic implements storage.Provider. 91 func (p *cinderProvider) Dynamic() bool { 92 return true 93 } 94 95 type cinderVolumeSource struct { 96 storageAdapter openstackStorage 97 envName string // non unique, informational only 98 modelUUID string 99 } 100 101 var _ storage.VolumeSource = (*cinderVolumeSource)(nil) 102 103 // CreateVolumes implements storage.VolumeSource. 104 func (s *cinderVolumeSource) CreateVolumes(args []storage.VolumeParams) ([]storage.CreateVolumesResult, error) { 105 results := make([]storage.CreateVolumesResult, len(args)) 106 for i, arg := range args { 107 volume, err := s.createVolume(arg) 108 if err != nil { 109 results[i].Error = errors.Trace(err) 110 continue 111 } 112 results[i].Volume = volume 113 } 114 return results, nil 115 } 116 117 func (s *cinderVolumeSource) createVolume(arg storage.VolumeParams) (*storage.Volume, error) { 118 var metadata interface{} 119 if len(arg.ResourceTags) > 0 { 120 metadata = arg.ResourceTags 121 } 122 cinderVolume, err := s.storageAdapter.CreateVolume(cinder.CreateVolumeVolumeParams{ 123 // The Cinder documentation incorrectly states the 124 // size parameter is in GB. It is actually GiB. 125 Size: int(math.Ceil(float64(arg.Size / 1024))), 126 Name: resourceName(arg.Tag, s.envName), 127 // TODO(axw) use the AZ of the initially attached machine. 128 AvailabilityZone: "", 129 Metadata: metadata, 130 }) 131 if err != nil { 132 return nil, errors.Trace(err) 133 } 134 135 // The response may (will?) come back before the volume transitions to 136 // "creating", in which case it will not have a size or status. Wait for 137 // the volume to transition, so we can record its actual size. 138 volumeId := cinderVolume.ID 139 cinderVolume, err = waitVolume(s.storageAdapter, volumeId, func(v *cinder.Volume) (bool, error) { 140 return v.Status != "", nil 141 }) 142 if err != nil { 143 if err := s.storageAdapter.DeleteVolume(volumeId); err != nil { 144 logger.Warningf("destroying volume %s: %s", volumeId, err) 145 } 146 return nil, errors.Errorf("waiting for volume to be provisioned: %s", err) 147 } 148 logger.Debugf("created volume: %+v", cinderVolume) 149 return &storage.Volume{arg.Tag, cinderToJujuVolumeInfo(cinderVolume)}, nil 150 } 151 152 // ListVolumes is specified on the storage.VolumeSource interface. 153 func (s *cinderVolumeSource) ListVolumes() ([]string, error) { 154 volumes, err := listVolumes(s.storageAdapter, func(v *cinder.Volume) bool { 155 return v.Metadata[tags.JujuModel] == s.modelUUID 156 }) 157 if err != nil { 158 return nil, errors.Trace(err) 159 } 160 return volumeInfoToVolumeIds(volumes), nil 161 } 162 163 func volumeInfoToVolumeIds(volumes []storage.VolumeInfo) []string { 164 volumeIds := make([]string, len(volumes)) 165 for i, volume := range volumes { 166 volumeIds[i] = volume.VolumeId 167 } 168 return volumeIds 169 } 170 171 // listVolumes returns all of the volumes matching the given function. 172 func listVolumes(storageAdapter openstackStorage, match func(*cinder.Volume) bool) ([]storage.VolumeInfo, error) { 173 cinderVolumes, err := storageAdapter.GetVolumesDetail() 174 if err != nil { 175 return nil, err 176 } 177 volumes := make([]storage.VolumeInfo, 0, len(cinderVolumes)) 178 for _, volume := range cinderVolumes { 179 if !match(&volume) { 180 continue 181 } 182 volumes = append(volumes, cinderToJujuVolumeInfo(&volume)) 183 } 184 return volumes, nil 185 } 186 187 // DescribeVolumes implements storage.VolumeSource. 188 func (s *cinderVolumeSource) DescribeVolumes(volumeIds []string) ([]storage.DescribeVolumesResult, error) { 189 // In most cases, it is quicker to get all volumes and loop 190 // locally than to make several round-trips to the provider. 191 cinderVolumes, err := s.storageAdapter.GetVolumesDetail() 192 if err != nil { 193 return nil, errors.Trace(err) 194 } 195 volumesById := make(map[string]*cinder.Volume) 196 for i, volume := range cinderVolumes { 197 volumesById[volume.ID] = &cinderVolumes[i] 198 } 199 results := make([]storage.DescribeVolumesResult, len(volumeIds)) 200 for i, volumeId := range volumeIds { 201 cinderVolume, ok := volumesById[volumeId] 202 if !ok { 203 results[i].Error = errors.NotFoundf("volume %q", volumeId) 204 continue 205 } 206 info := cinderToJujuVolumeInfo(cinderVolume) 207 results[i].VolumeInfo = &info 208 } 209 return results, nil 210 } 211 212 // DestroyVolumes implements storage.VolumeSource. 213 func (s *cinderVolumeSource) DestroyVolumes(volumeIds []string) ([]error, error) { 214 return destroyVolumes(s.storageAdapter, volumeIds), nil 215 } 216 217 func destroyVolumes(storageAdapter openstackStorage, volumeIds []string) []error { 218 var wg sync.WaitGroup 219 wg.Add(len(volumeIds)) 220 results := make([]error, len(volumeIds)) 221 for i, volumeId := range volumeIds { 222 go func(i int, volumeId string) { 223 defer wg.Done() 224 results[i] = destroyVolume(storageAdapter, volumeId) 225 }(i, volumeId) 226 } 227 wg.Wait() 228 return results 229 } 230 231 func destroyVolume(storageAdapter openstackStorage, volumeId string) error { 232 logger.Debugf("destroying volume %q", volumeId) 233 // Volumes must not be in-use when destroying. A volume may 234 // still be in-use when the instance it is attached to is 235 // in the process of being terminated. 236 var issuedDetach bool 237 volume, err := waitVolume(storageAdapter, volumeId, func(v *cinder.Volume) (bool, error) { 238 switch v.Status { 239 default: 240 // Not ready for deletion; keep waiting. 241 return false, nil 242 case volumeStatusAvailable, volumeStatusDeleting, volumeStatusError: 243 return true, nil 244 case volumeStatusInUse: 245 // Detach below. 246 break 247 } 248 // Volume is still attached, so detach it. 249 if !issuedDetach { 250 args := make([]storage.VolumeAttachmentParams, len(v.Attachments)) 251 for i, a := range v.Attachments { 252 args[i].VolumeId = volumeId 253 args[i].InstanceId = instance.Id(a.ServerId) 254 } 255 if len(args) > 0 { 256 results, err := detachVolumes(storageAdapter, args) 257 if err != nil { 258 return false, errors.Trace(err) 259 } 260 for _, err := range results { 261 if err != nil { 262 return false, errors.Trace(err) 263 } 264 } 265 } 266 issuedDetach = true 267 } 268 return false, nil 269 }) 270 if err != nil { 271 return errors.Trace(err) 272 } 273 if volume.Status == volumeStatusDeleting { 274 // Already being deleted, nothing to do. 275 return nil 276 } 277 if err := storageAdapter.DeleteVolume(volumeId); err != nil { 278 return errors.Trace(err) 279 } 280 return nil 281 } 282 283 // ValidateVolumeParams implements storage.VolumeSource. 284 func (s *cinderVolumeSource) ValidateVolumeParams(params storage.VolumeParams) error { 285 return nil 286 } 287 288 // AttachVolumes implements storage.VolumeSource. 289 func (s *cinderVolumeSource) AttachVolumes(args []storage.VolumeAttachmentParams) ([]storage.AttachVolumesResult, error) { 290 results := make([]storage.AttachVolumesResult, len(args)) 291 for i, arg := range args { 292 attachment, err := s.attachVolume(arg) 293 if err != nil { 294 results[i].Error = errors.Trace(err) 295 continue 296 } 297 results[i].VolumeAttachment = attachment 298 } 299 return results, nil 300 } 301 302 func (s *cinderVolumeSource) attachVolume(arg storage.VolumeAttachmentParams) (*storage.VolumeAttachment, error) { 303 // Check to see if the volume is already attached. 304 existingAttachments, err := s.storageAdapter.ListVolumeAttachments(string(arg.InstanceId)) 305 if err != nil { 306 return nil, err 307 } 308 novaAttachment := findAttachment(arg.VolumeId, existingAttachments) 309 if novaAttachment == nil { 310 // A volume must be "available" before it can be attached. 311 if _, err := waitVolume(s.storageAdapter, arg.VolumeId, func(v *cinder.Volume) (bool, error) { 312 return v.Status == "available", nil 313 }); err != nil { 314 return nil, errors.Annotate(err, "waiting for volume to become available") 315 } 316 novaAttachment, err = s.storageAdapter.AttachVolume( 317 string(arg.InstanceId), 318 arg.VolumeId, 319 autoAssignedMountPoint, 320 ) 321 if err != nil { 322 return nil, err 323 } 324 } 325 return &storage.VolumeAttachment{ 326 arg.Volume, 327 arg.Machine, 328 storage.VolumeAttachmentInfo{ 329 DeviceName: novaAttachment.Device[len("/dev/"):], 330 }, 331 }, nil 332 } 333 334 func waitVolume( 335 storageAdapter openstackStorage, 336 volumeId string, 337 pred func(*cinder.Volume) (bool, error), 338 ) (*cinder.Volume, error) { 339 for a := cinderAttempt.Start(); a.Next(); { 340 volume, err := storageAdapter.GetVolume(volumeId) 341 if err != nil { 342 return nil, errors.Annotate(err, "getting volume") 343 } 344 ok, err := pred(volume) 345 if err != nil { 346 return nil, errors.Trace(err) 347 } 348 if ok { 349 return volume, nil 350 } 351 } 352 return nil, errors.New("timed out") 353 } 354 355 // DetachVolumes implements storage.VolumeSource. 356 func (s *cinderVolumeSource) DetachVolumes(args []storage.VolumeAttachmentParams) ([]error, error) { 357 return detachVolumes(s.storageAdapter, args) 358 } 359 360 func detachVolumes(storageAdapter openstackStorage, args []storage.VolumeAttachmentParams) ([]error, error) { 361 results := make([]error, len(args)) 362 for i, arg := range args { 363 // Check to see if the volume is already detached. 364 attachments, err := storageAdapter.ListVolumeAttachments(string(arg.InstanceId)) 365 if err != nil { 366 results[i] = errors.Annotate(err, "listing volume attachments") 367 continue 368 } 369 if err := detachVolume( 370 string(arg.InstanceId), 371 arg.VolumeId, 372 attachments, 373 storageAdapter, 374 ); err != nil { 375 results[i] = errors.Annotatef( 376 err, "detaching volume %s from server %s", 377 arg.VolumeId, arg.InstanceId, 378 ) 379 continue 380 } 381 } 382 return results, nil 383 } 384 385 func cinderToJujuVolumeInfo(volume *cinder.Volume) storage.VolumeInfo { 386 return storage.VolumeInfo{ 387 VolumeId: volume.ID, 388 Size: uint64(volume.Size * 1024), 389 Persistent: true, 390 } 391 } 392 393 func detachVolume(instanceId, volumeId string, attachments []nova.VolumeAttachment, storageAdapter openstackStorage) error { 394 // TODO(axw) verify whether we need to do this find step. From looking at the example 395 // responses in the OpenStack docs, the "attachment ID" is always the same as the 396 // volume ID. So we should just be able to issue a blind detach request, and then 397 // ignore errors that indicate the volume is already detached. 398 if findAttachment(volumeId, attachments) == nil { 399 return nil 400 } 401 return storageAdapter.DetachVolume(instanceId, volumeId) 402 } 403 404 func findAttachment(volId string, attachments []nova.VolumeAttachment) *nova.VolumeAttachment { 405 for _, attachment := range attachments { 406 if attachment.VolumeId == volId { 407 return &attachment 408 } 409 } 410 return nil 411 } 412 413 type openstackStorage interface { 414 GetVolume(volumeId string) (*cinder.Volume, error) 415 GetVolumesDetail() ([]cinder.Volume, error) 416 DeleteVolume(volumeId string) error 417 CreateVolume(cinder.CreateVolumeVolumeParams) (*cinder.Volume, error) 418 AttachVolume(serverId, volumeId, mountPoint string) (*nova.VolumeAttachment, error) 419 DetachVolume(serverId, attachmentId string) error 420 ListVolumeAttachments(serverId string) ([]nova.VolumeAttachment, error) 421 } 422 423 type endpointResolver interface { 424 EndpointsForRegion(region string) identity.ServiceURLs 425 } 426 427 func getVolumeEndpointURL(client endpointResolver, region string) (*url.URL, error) { 428 endpointMap := client.EndpointsForRegion(region) 429 // The cinder openstack charm appends 'v2' to the type for the v2 api. 430 endpoint, ok := endpointMap["volumev2"] 431 if !ok { 432 logger.Debugf(`endpoint "volumev2" not found for %q region, trying "volume"`, region) 433 endpoint, ok = endpointMap["volume"] 434 if !ok { 435 return nil, errors.NotFoundf(`endpoint "volume" in region %q`, region) 436 } 437 } 438 return url.Parse(endpoint) 439 } 440 441 func newOpenstackStorageAdapter(environConfig *config.Config) (openstackStorage, error) { 442 ecfg, err := providerInstance.newConfig(environConfig) 443 if err != nil { 444 return nil, errors.Trace(err) 445 } 446 client, err := authClient(ecfg) 447 if err != nil { 448 return nil, errors.Trace(err) 449 } else if err := client.Authenticate(); err != nil { 450 return nil, errors.Trace(err) 451 } 452 453 endpointUrl, err := getVolumeEndpointURL(client, ecfg.region()) 454 if err != nil { 455 if errors.IsNotFound(err) { 456 return nil, errors.NewNotSupported(err, "volumes not supported") 457 } 458 return nil, errors.Annotate(err, "getting volume endpoint") 459 } 460 461 return &openstackStorageAdapter{ 462 cinderClient{cinder.Basic(endpointUrl, client.TenantId(), client.Token)}, 463 novaClient{nova.New(client)}, 464 }, nil 465 } 466 467 type openstackStorageAdapter struct { 468 cinderClient 469 novaClient 470 } 471 472 type cinderClient struct { 473 *cinder.Client 474 } 475 476 type novaClient struct { 477 *nova.Client 478 } 479 480 // CreateVolume is part of the openstackStorage interface. 481 func (ga *openstackStorageAdapter) CreateVolume(args cinder.CreateVolumeVolumeParams) (*cinder.Volume, error) { 482 resp, err := ga.cinderClient.CreateVolume(args) 483 if err != nil { 484 return nil, err 485 } 486 return &resp.Volume, nil 487 } 488 489 // GetVolumesDetail is part of the openstackStorage interface. 490 func (ga *openstackStorageAdapter) GetVolumesDetail() ([]cinder.Volume, error) { 491 resp, err := ga.cinderClient.GetVolumesDetail() 492 if err != nil { 493 return nil, err 494 } 495 return resp.Volumes, nil 496 } 497 498 // GetVolume is part of the openstackStorage interface. 499 func (ga *openstackStorageAdapter) GetVolume(volumeId string) (*cinder.Volume, error) { 500 resp, err := ga.cinderClient.GetVolume(volumeId) 501 if err != nil { 502 return nil, err 503 } 504 return &resp.Volume, nil 505 }