github.com/wallyworld/juju@v0.0.0-20161013125918-6cf1bc9d917a/cloud/clouds.go (about) 1 // Copyright 2015 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 // Package cloud provides functionality to parse information 5 // describing clouds, including regions, supported auth types etc. 6 package cloud 7 8 import ( 9 "fmt" 10 "io/ioutil" 11 "os" 12 "reflect" 13 "sort" 14 "strings" 15 16 "github.com/juju/errors" 17 "github.com/juju/utils" 18 "gopkg.in/yaml.v2" 19 20 "github.com/juju/juju/juju/osenv" 21 "github.com/juju/juju/provider/lxd/lxdnames" 22 ) 23 24 //go:generate go run ../generate/filetoconst/filetoconst.go fallbackPublicCloudInfo fallback-public-cloud.yaml fallback_public_cloud.go 2015 cloud 25 26 // AuthType is the type of authentication used by the cloud. 27 type AuthType string 28 29 // AuthTypes is defined to allow sorting AuthType slices. 30 type AuthTypes []AuthType 31 32 func (a AuthTypes) Len() int { return len(a) } 33 func (a AuthTypes) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 34 func (a AuthTypes) Less(i, j int) bool { return a[i] < a[j] } 35 36 const ( 37 // AccessKeyAuthType is an authentication type using a key and secret. 38 AccessKeyAuthType AuthType = "access-key" 39 40 // UserPassAuthType is an authentication type using a username and password. 41 UserPassAuthType AuthType = "userpass" 42 43 // OAuth1AuthType is an authentication type using oauth1. 44 OAuth1AuthType AuthType = "oauth1" 45 46 // OAuth2AuthType is an authentication type using oauth2. 47 OAuth2AuthType AuthType = "oauth2" 48 49 // JSONFileAuthType is an authentication type that takes a path to 50 // a JSON file. 51 JSONFileAuthType AuthType = "jsonfile" 52 53 // CertificateAuthType is an authentication type using certificates. 54 CertificateAuthType AuthType = "certificate" 55 56 // EmptyAuthType is the authentication type used for providers 57 // that require no credentials, e.g. "lxd", and "manual". 58 EmptyAuthType AuthType = "empty" 59 ) 60 61 // Attrs serves as a map to hold regions specific configuration attributes. 62 // This serves to reduce confusion over having a nested map, i.e. 63 // map[string]map[string]interface{} 64 type Attrs map[string]interface{} 65 66 // RegionConfig holds a map of regions and the attributes that serve as the 67 // region specific configuration options. This allows model inheritance to 68 // function, providing a place to store configuration for a specific region 69 // which is passed down to other models under the same controller. 70 type RegionConfig map[string]Attrs 71 72 // Cloud is a cloud definition. 73 type Cloud struct { 74 // Type is the type of cloud, eg ec2, openstack etc. 75 // This is one of the provider names registered with 76 // environs.RegisterProvider. 77 Type string 78 79 // Description describes the type of cloud. 80 Description string 81 82 // AuthTypes are the authentication modes supported by the cloud. 83 AuthTypes AuthTypes 84 85 // Endpoint is the default endpoint for the cloud regions, may be 86 // overridden by a region. 87 Endpoint string 88 89 // IdentityEndpoint is the default identity endpoint for the cloud 90 // regions, may be overridden by a region. 91 IdentityEndpoint string 92 93 // StorageEndpoint is the default storage endpoint for the cloud 94 // regions, may be overridden by a region. 95 StorageEndpoint string 96 97 // Regions are the regions available in the cloud. 98 // 99 // Regions is a slice, and not a map, because order is important. 100 // The first region in the slice is the default region for the 101 // cloud. 102 Regions []Region 103 104 // Config contains optional cloud-specific configuration to use 105 // when bootstrapping Juju in this cloud. The cloud configuration 106 // will be combined with Juju-generated, and user-supplied values; 107 // user-supplied values taking precedence. 108 Config map[string]interface{} 109 110 // RegionConfig contains optional region specific configuration. 111 // Like Config above, this will be combined with Juju-generated and user 112 // supplied values; with user supplied values taking precedence. 113 RegionConfig RegionConfig 114 } 115 116 // Region is a cloud region. 117 type Region struct { 118 // Name is the name of the region. 119 Name string 120 121 // Endpoint is the region's primary endpoint URL. 122 Endpoint string 123 124 // IdentityEndpoint is the region's identity endpoint URL. 125 // If the cloud/region does not have an identity-specific 126 // endpoint URL, this will be empty. 127 IdentityEndpoint string 128 129 // StorageEndpoint is the region's storage endpoint URL. 130 // If the cloud/region does not have a storage-specific 131 // endpoint URL, this will be empty. 132 StorageEndpoint string 133 } 134 135 // cloudSet contains cloud definitions, used for marshalling and 136 // unmarshalling. 137 type cloudSet struct { 138 // Clouds is a map of cloud definitions, keyed on cloud name. 139 Clouds map[string]*cloud `yaml:"clouds"` 140 } 141 142 // cloud is equivalent to Cloud, for marshalling and unmarshalling. 143 type cloud struct { 144 Type string `yaml:"type"` 145 Description string `yaml:"description,omitempty"` 146 AuthTypes []AuthType `yaml:"auth-types,omitempty,flow"` 147 Endpoint string `yaml:"endpoint,omitempty"` 148 IdentityEndpoint string `yaml:"identity-endpoint,omitempty"` 149 StorageEndpoint string `yaml:"storage-endpoint,omitempty"` 150 Regions regions `yaml:"regions,omitempty"` 151 Config map[string]interface{} `yaml:"config,omitempty"` 152 RegionConfig RegionConfig `yaml:"region-config,omitempty"` 153 } 154 155 // regions is a collection of regions, either as a map and/or 156 // as a yaml.MapSlice. 157 // 158 // When marshalling, we populate the Slice field only. This is 159 // necessary for us to control the order of map items. 160 // 161 // When unmarshalling, we populate both Map and Slice. Map is 162 // populated to simplify conversion to Region objects. Slice 163 // is populated so we can identify the first map item, which 164 // becomes the default region for the cloud. 165 type regions struct { 166 Map map[string]*region 167 Slice yaml.MapSlice 168 } 169 170 // region is equivalent to Region, for marshalling and unmarshalling. 171 type region struct { 172 Endpoint string `yaml:"endpoint,omitempty"` 173 IdentityEndpoint string `yaml:"identity-endpoint,omitempty"` 174 StorageEndpoint string `yaml:"storage-endpoint,omitempty"` 175 } 176 177 //DefaultLXD is the name of the default lxd cloud. 178 const DefaultLXD = "localhost" 179 180 // BuiltInClouds work out of the box. 181 var BuiltInClouds = map[string]Cloud{ 182 DefaultLXD: { 183 Type: lxdnames.ProviderType, 184 AuthTypes: []AuthType{EmptyAuthType}, 185 Regions: []Region{{Name: lxdnames.DefaultRegion}}, 186 Description: defaultCloudDescription[lxdnames.ProviderType], 187 }, 188 } 189 190 // CloudByName returns the cloud with the specified name. 191 // If there exists no cloud with the specified name, an 192 // error satisfying errors.IsNotFound will be returned. 193 // 194 // TODO(axw) write unit tests for this. 195 func CloudByName(name string) (*Cloud, error) { 196 // Personal clouds take precedence. 197 personalClouds, err := PersonalCloudMetadata() 198 if err != nil { 199 return nil, errors.Trace(err) 200 } 201 if cloud, ok := personalClouds[name]; ok { 202 return &cloud, nil 203 } 204 clouds, _, err := PublicCloudMetadata(JujuPublicCloudsPath()) 205 if err != nil { 206 return nil, errors.Trace(err) 207 } 208 if cloud, ok := clouds[name]; ok { 209 return &cloud, nil 210 } 211 if cloud, ok := BuiltInClouds[name]; ok { 212 return &cloud, nil 213 } 214 return nil, errors.NotFoundf("cloud %s", name) 215 } 216 217 // RegionByName finds the region in the given slice with the 218 // specified name, with case folding. 219 func RegionByName(regions []Region, name string) (*Region, error) { 220 for _, region := range regions { 221 if !strings.EqualFold(region.Name, name) { 222 continue 223 } 224 return ®ion, nil 225 } 226 return nil, errors.NewNotFound(nil, fmt.Sprintf( 227 "region %q not found (expected one of %q)", 228 name, RegionNames(regions), 229 )) 230 } 231 232 // RegionNames returns a sorted list of the names of the given regions. 233 func RegionNames(regions []Region) []string { 234 names := make([]string, len(regions)) 235 for i, region := range regions { 236 names[i] = region.Name 237 } 238 sort.Strings(names) 239 return names 240 } 241 242 // JujuPublicCloudsPath is the location where public cloud information is 243 // expected to be found. Requires JUJU_HOME to be set. 244 func JujuPublicCloudsPath() string { 245 return osenv.JujuXDGDataHomePath("public-clouds.yaml") 246 } 247 248 // PublicCloudMetadata looks in searchPath for cloud metadata files and if none 249 // are found, returns the fallback public cloud metadata. 250 func PublicCloudMetadata(searchPath ...string) (result map[string]Cloud, fallbackUsed bool, err error) { 251 for _, file := range searchPath { 252 data, err := ioutil.ReadFile(file) 253 if err != nil && os.IsNotExist(err) { 254 continue 255 } 256 if err != nil { 257 return nil, false, errors.Trace(err) 258 } 259 clouds, err := ParseCloudMetadata(data) 260 if err != nil { 261 return nil, false, errors.Trace(err) 262 } 263 return clouds, false, err 264 } 265 clouds, err := ParseCloudMetadata([]byte(fallbackPublicCloudInfo)) 266 return clouds, true, err 267 } 268 269 // ParseCloudMetadata parses the given yaml bytes into Clouds metadata. 270 func ParseCloudMetadata(data []byte) (map[string]Cloud, error) { 271 var metadata cloudSet 272 if err := yaml.Unmarshal(data, &metadata); err != nil { 273 return nil, errors.Annotate(err, "cannot unmarshal yaml cloud metadata") 274 } 275 276 // Translate to the exported type. For each cloud, we store 277 // the first region for the cloud as its default region. 278 clouds := make(map[string]Cloud) 279 for name, cloud := range metadata.Clouds { 280 details := cloudFromInternal(cloud) 281 if details.Description == "" { 282 var ok bool 283 if details.Description, ok = defaultCloudDescription[name]; !ok { 284 details.Description = defaultCloudDescription[cloud.Type] 285 } 286 } 287 clouds[name] = details 288 } 289 return clouds, nil 290 } 291 292 var defaultCloudDescription = map[string]string{ 293 "aws": "Amazon Web Services", 294 "aws-china": "Amazon China", 295 "aws-gov": "Amazon (USA Government)", 296 "google": "Google Cloud Platform", 297 "azure": "Microsoft Azure", 298 "azure-china": "Microsoft Azure China", 299 "rackspace": "Rackspace Cloud", 300 "joyent": "Joyent Cloud", 301 "cloudsigma": "CloudSigma Cloud", 302 "lxd": "LXD Container Hypervisor", 303 "maas": "Metal As A Service", 304 "openstack": "Openstack Cloud", 305 } 306 307 // WritePublicCloudMetadata marshals to YAML and writes the cloud metadata 308 // to the public cloud file. 309 func WritePublicCloudMetadata(cloudsMap map[string]Cloud) error { 310 data, err := marshalCloudMetadata(cloudsMap) 311 if err != nil { 312 return errors.Trace(err) 313 } 314 return utils.AtomicWriteFile(JujuPublicCloudsPath(), data, 0600) 315 } 316 317 // IsSameCloudMetadata returns true if both meta and meta2 contain the 318 // same cloud metadata. 319 func IsSameCloudMetadata(meta1, meta2 map[string]Cloud) (bool, error) { 320 // The easiest approach is to simply marshall to YAML and compare. 321 yaml1, err := marshalCloudMetadata(meta1) 322 if err != nil { 323 return false, err 324 } 325 yaml2, err := marshalCloudMetadata(meta2) 326 if err != nil { 327 return false, err 328 } 329 return string(yaml1) == string(yaml2), nil 330 } 331 332 // marshalCloudMetadata marshals the given clouds to YAML. 333 func marshalCloudMetadata(cloudsMap map[string]Cloud) ([]byte, error) { 334 clouds := cloudSet{make(map[string]*cloud)} 335 for name, metadata := range cloudsMap { 336 clouds.Clouds[name] = cloudToInternal(metadata) 337 } 338 data, err := yaml.Marshal(clouds) 339 if err != nil { 340 return nil, errors.Annotate(err, "cannot marshal cloud metadata") 341 } 342 return data, nil 343 } 344 345 // MarshalCloud marshals a Cloud to an opaque byte array. 346 func MarshalCloud(cloud Cloud) ([]byte, error) { 347 return yaml.Marshal(cloudToInternal(cloud)) 348 } 349 350 // UnmarshalCloud unmarshals a Cloud from a byte array produced by MarshalCloud. 351 func UnmarshalCloud(in []byte) (Cloud, error) { 352 var internal cloud 353 if err := yaml.Unmarshal(in, &internal); err != nil { 354 return Cloud{}, errors.Annotate(err, "cannot unmarshal yaml cloud metadata") 355 } 356 return cloudFromInternal(&internal), nil 357 } 358 359 func cloudToInternal(in Cloud) *cloud { 360 var regions regions 361 for _, r := range in.Regions { 362 regions.Slice = append(regions.Slice, yaml.MapItem{ 363 r.Name, region{ 364 r.Endpoint, 365 r.IdentityEndpoint, 366 r.StorageEndpoint, 367 }, 368 }) 369 } 370 return &cloud{ 371 Type: in.Type, 372 AuthTypes: in.AuthTypes, 373 Endpoint: in.Endpoint, 374 IdentityEndpoint: in.IdentityEndpoint, 375 StorageEndpoint: in.StorageEndpoint, 376 Regions: regions, 377 Config: in.Config, 378 RegionConfig: in.RegionConfig, 379 } 380 } 381 382 func cloudFromInternal(in *cloud) Cloud { 383 var regions []Region 384 if len(in.Regions.Map) > 0 { 385 for _, item := range in.Regions.Slice { 386 name := fmt.Sprint(item.Key) 387 r := in.Regions.Map[name] 388 if r == nil { 389 // r will be nil if none of the fields in 390 // the YAML are set. 391 regions = append(regions, Region{Name: name}) 392 } else { 393 regions = append(regions, Region{ 394 name, 395 r.Endpoint, 396 r.IdentityEndpoint, 397 r.StorageEndpoint, 398 }) 399 } 400 } 401 } 402 meta := Cloud{ 403 Type: in.Type, 404 AuthTypes: in.AuthTypes, 405 Endpoint: in.Endpoint, 406 IdentityEndpoint: in.IdentityEndpoint, 407 StorageEndpoint: in.StorageEndpoint, 408 Regions: regions, 409 Config: in.Config, 410 RegionConfig: in.RegionConfig, 411 Description: in.Description, 412 } 413 meta.denormaliseMetadata() 414 return meta 415 } 416 417 // MarshalYAML implements the yaml.Marshaler interface. 418 func (r regions) MarshalYAML() (interface{}, error) { 419 return r.Slice, nil 420 } 421 422 // UnmarshalYAML implements the yaml.Unmarshaler interface. 423 func (r *regions) UnmarshalYAML(f func(interface{}) error) error { 424 if err := f(&r.Map); err != nil { 425 return err 426 } 427 return f(&r.Slice) 428 } 429 430 // To keep the metadata concise, attributes on the metadata struct which 431 // have the same value for each item may be moved up to a higher level in 432 // the tree. denormaliseMetadata descends the tree and fills in any missing 433 // attributes with values from a higher level. 434 func (cloud Cloud) denormaliseMetadata() { 435 for name, region := range cloud.Regions { 436 r := region 437 inherit(&r, &cloud) 438 cloud.Regions[name] = r 439 } 440 } 441 442 type structTags map[reflect.Type]map[string]int 443 444 var tagsForType structTags = make(structTags) 445 446 // RegisterStructTags ensures the yaml tags for the given structs are able to be used 447 // when parsing cloud metadata. 448 func RegisterStructTags(vals ...interface{}) { 449 tags := mkTags(vals...) 450 for k, v := range tags { 451 tagsForType[k] = v 452 } 453 } 454 455 func init() { 456 RegisterStructTags(Cloud{}, Region{}) 457 } 458 459 func mkTags(vals ...interface{}) map[reflect.Type]map[string]int { 460 typeMap := make(map[reflect.Type]map[string]int) 461 for _, v := range vals { 462 t := reflect.TypeOf(v) 463 typeMap[t] = yamlTags(t) 464 } 465 return typeMap 466 } 467 468 // yamlTags returns a map from yaml tag to the field index for the string fields in the given type. 469 func yamlTags(t reflect.Type) map[string]int { 470 if t.Kind() != reflect.Struct { 471 panic(errors.Errorf("cannot get yaml tags on type %s", t)) 472 } 473 tags := make(map[string]int) 474 for i := 0; i < t.NumField(); i++ { 475 f := t.Field(i) 476 if f.Type != reflect.TypeOf("") { 477 continue 478 } 479 if tag := f.Tag.Get("yaml"); tag != "" { 480 if i := strings.Index(tag, ","); i >= 0 { 481 tag = tag[0:i] 482 } 483 if tag == "-" { 484 continue 485 } 486 if tag != "" { 487 f.Name = tag 488 } 489 } 490 tags[f.Name] = i 491 } 492 return tags 493 } 494 495 // inherit sets any blank fields in dst to their equivalent values in fields in src that have matching json tags. 496 // The dst parameter must be a pointer to a struct. 497 func inherit(dst, src interface{}) { 498 for tag := range tags(dst) { 499 setFieldByTag(dst, tag, fieldByTag(src, tag), false) 500 } 501 } 502 503 // tags returns the field offsets for the JSON tags defined by the given value, which must be 504 // a struct or a pointer to a struct. 505 func tags(x interface{}) map[string]int { 506 t := reflect.TypeOf(x) 507 if t.Kind() == reflect.Ptr { 508 t = t.Elem() 509 } 510 if t.Kind() != reflect.Struct { 511 panic(errors.Errorf("expected struct, not %s", t)) 512 } 513 514 if tagm := tagsForType[t]; tagm != nil { 515 return tagm 516 } 517 panic(errors.Errorf("%s not found in type table", t)) 518 } 519 520 // fieldByTag returns the value for the field in x with the given JSON tag, or "" if there is no such field. 521 func fieldByTag(x interface{}, tag string) string { 522 tagm := tags(x) 523 v := reflect.ValueOf(x) 524 if v.Kind() == reflect.Ptr { 525 v = v.Elem() 526 } 527 if i, ok := tagm[tag]; ok { 528 return v.Field(i).Interface().(string) 529 } 530 return "" 531 } 532 533 // setFieldByTag sets the value for the field in x with the given JSON tag to val. 534 // The override parameter specifies whether the value will be set even if the original value is non-empty. 535 func setFieldByTag(x interface{}, tag, val string, override bool) { 536 i, ok := tags(x)[tag] 537 if !ok { 538 return 539 } 540 v := reflect.ValueOf(x).Elem() 541 f := v.Field(i) 542 if override || f.Interface().(string) == "" { 543 f.Set(reflect.ValueOf(val)) 544 } 545 }