github.com/mwhudson/juju@v0.0.0-20160512215208-90ff01f3497f/cloud/clouds.go (about) 1 // Copyright 2015 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 // Package cloud provides functionality to parse information 5 // describing clouds, including regions, supported auth types etc. 6 7 package cloud 8 9 import ( 10 "fmt" 11 "io/ioutil" 12 "os" 13 "reflect" 14 "strings" 15 16 "github.com/juju/errors" 17 "github.com/juju/utils" 18 "gopkg.in/yaml.v2" 19 20 "github.com/juju/juju/juju/osenv" 21 ) 22 23 //go:generate go run ../generate/filetoconst.go fallbackPublicCloudInfo fallback-public-cloud.yaml fallback_public_cloud.go 2015 24 25 // AuthType is the type of authentication used by the cloud. 26 type AuthType string 27 28 const ( 29 // AccessKeyAuthType is an authentication type using a key and secret. 30 AccessKeyAuthType AuthType = "access-key" 31 32 // UserPassAuthType is an authentication type using a username and password. 33 UserPassAuthType AuthType = "userpass" 34 35 // OAuth1AuthType is an authentication type using oauth1. 36 OAuth1AuthType AuthType = "oauth1" 37 38 // OAuth2AuthType is an authentication type using oauth2. 39 OAuth2AuthType AuthType = "oauth2" 40 41 // JSONFileAuthType is an authentication type that takes a path to 42 // a JSON file. 43 JSONFileAuthType AuthType = "jsonfile" 44 45 // EmptyAuthType is the authentication type used for providers 46 // that require no credentials, e.g. "lxd", and "manual". 47 EmptyAuthType AuthType = "empty" 48 ) 49 50 // Cloud is a cloud definition. 51 type Cloud struct { 52 // Type is the type of cloud, eg aws, openstack etc. 53 Type string 54 55 // AuthTypes are the authentication modes supported by the cloud. 56 AuthTypes []AuthType 57 58 // Endpoint is the default endpoint for the cloud regions, may be 59 // overridden by a region. 60 Endpoint string 61 62 // StorageEndpoint is the default storage endpoint for the cloud 63 // regions, may be overridden by a region. 64 StorageEndpoint string 65 66 // Regions are the regions available in the cloud. 67 // 68 // Regions is a slice, and not a map, because order is important. 69 // The first region in the slice is the default region for the 70 // cloud. 71 Regions []Region 72 } 73 74 // Region is a cloud region. 75 type Region struct { 76 // Name is the name of the region. 77 Name string 78 79 // Endpoint is the region's primary endpoint URL. 80 Endpoint string 81 82 // StorageEndpoint is the region's storage endpoint URL. 83 // If the cloud/region does not have a storage-specific 84 // endpoint URL, this will be empty. 85 StorageEndpoint string 86 } 87 88 // cloudSet contains cloud definitions, used for marshalling and 89 // unmarshalling. 90 type cloudSet struct { 91 // Clouds is a map of cloud definitions, keyed on cloud name. 92 Clouds map[string]*cloud `yaml:"clouds"` 93 } 94 95 // cloud is equivalent to Cloud, for marshalling and unmarshalling. 96 type cloud struct { 97 Type string `yaml:"type"` 98 AuthTypes []AuthType `yaml:"auth-types,omitempty,flow"` 99 Endpoint string `yaml:"endpoint,omitempty"` 100 StorageEndpoint string `yaml:"storage-endpoint,omitempty"` 101 Regions regions `yaml:"regions,omitempty"` 102 } 103 104 // regions is a collection of regions, either as a map and/or 105 // as a yaml.MapSlice. 106 // 107 // When marshalling, we populate the Slice field only. This is 108 // necessary for us to control the order of map items. 109 // 110 // When unmarshalling, we populate both Map and Slice. Map is 111 // populated to simplify conversion to Region objects. Slice 112 // is populated so we can identify the first map item, which 113 // becomes the default region for the cloud. 114 type regions struct { 115 Map map[string]*region 116 Slice yaml.MapSlice 117 } 118 119 // region is equivalent to Region, for marshalling and unmarshalling. 120 type region struct { 121 Endpoint string `yaml:"endpoint,omitempty"` 122 StorageEndpoint string `yaml:"storage-endpoint,omitempty"` 123 } 124 125 // BuiltInClouds work out of the box. 126 var BuiltInClouds = map[string]Cloud{ 127 "localhost": { 128 Type: "lxd", 129 AuthTypes: []AuthType{EmptyAuthType}, 130 Regions: []Region{{Name: "localhost"}}, 131 }, 132 } 133 134 // CloudByName returns the cloud with the specified name. 135 // If there exists no cloud with the specified name, an 136 // error satisfying errors.IsNotFound will be returned. 137 // 138 // TODO(axw) write unit tests for this. 139 func CloudByName(name string) (*Cloud, error) { 140 // Personal clouds take precedence. 141 personalClouds, err := PersonalCloudMetadata() 142 if err != nil { 143 return nil, errors.Trace(err) 144 } 145 if cloud, ok := personalClouds[name]; ok { 146 return &cloud, nil 147 } 148 clouds, _, err := PublicCloudMetadata(JujuPublicCloudsPath()) 149 if err != nil { 150 return nil, errors.Trace(err) 151 } 152 if cloud, ok := clouds[name]; ok { 153 return &cloud, nil 154 } 155 if cloud, ok := BuiltInClouds[name]; ok { 156 return &cloud, nil 157 } 158 return nil, errors.NotFoundf("cloud %s", name) 159 } 160 161 // JujuPublicCloudsPath is the location where public cloud information is 162 // expected to be found. Requires JUJU_HOME to be set. 163 func JujuPublicCloudsPath() string { 164 return osenv.JujuXDGDataHomePath("public-clouds.yaml") 165 } 166 167 // PublicCloudMetadata looks in searchPath for cloud metadata files and if none 168 // are found, returns the fallback public cloud metadata. 169 func PublicCloudMetadata(searchPath ...string) (result map[string]Cloud, fallbackUsed bool, err error) { 170 for _, file := range searchPath { 171 data, err := ioutil.ReadFile(file) 172 if err != nil && os.IsNotExist(err) { 173 continue 174 } 175 if err != nil { 176 return nil, false, errors.Trace(err) 177 } 178 clouds, err := ParseCloudMetadata(data) 179 if err != nil { 180 return nil, false, errors.Trace(err) 181 } 182 return clouds, false, err 183 } 184 clouds, err := ParseCloudMetadata([]byte(fallbackPublicCloudInfo)) 185 return clouds, true, err 186 } 187 188 // ParseCloudMetadata parses the given yaml bytes into Clouds metadata. 189 func ParseCloudMetadata(data []byte) (map[string]Cloud, error) { 190 var metadata cloudSet 191 if err := yaml.Unmarshal(data, &metadata); err != nil { 192 return nil, errors.Annotate(err, "cannot unmarshal yaml cloud metadata") 193 } 194 195 // Translate to the exported type. For each cloud, we store 196 // the first region for the cloud as its default region. 197 clouds := make(map[string]Cloud) 198 for name, cloud := range metadata.Clouds { 199 var regions []Region 200 if len(cloud.Regions.Map) > 0 { 201 for _, item := range cloud.Regions.Slice { 202 name := fmt.Sprint(item.Key) 203 r := cloud.Regions.Map[name] 204 if r == nil { 205 // r will be nil if none of the fields in 206 // the YAML are set. 207 regions = append(regions, Region{Name: name}) 208 } else { 209 regions = append(regions, Region{ 210 name, r.Endpoint, r.StorageEndpoint, 211 }) 212 } 213 } 214 } 215 meta := Cloud{ 216 Type: cloud.Type, 217 AuthTypes: cloud.AuthTypes, 218 Endpoint: cloud.Endpoint, 219 StorageEndpoint: cloud.StorageEndpoint, 220 Regions: regions, 221 } 222 meta.denormaliseMetadata() 223 clouds[name] = meta 224 } 225 return clouds, nil 226 } 227 228 // WritePublicCloudMetadata marshals to YAML and writes the cloud metadata 229 // to the public cloud file. 230 func WritePublicCloudMetadata(cloudsMap map[string]Cloud) error { 231 data, err := marshalCloudMetadata(cloudsMap) 232 if err != nil { 233 return errors.Trace(err) 234 } 235 return utils.AtomicWriteFile(JujuPublicCloudsPath(), data, 0600) 236 } 237 238 // IsSameCloudMetadata returns true if both meta and meta2 contain the 239 // same cloud metadata. 240 func IsSameCloudMetadata(meta1, meta2 map[string]Cloud) (bool, error) { 241 // The easiest approach is to simply marshall to YAML and compare. 242 yaml1, err := marshalCloudMetadata(meta1) 243 if err != nil { 244 return false, err 245 } 246 yaml2, err := marshalCloudMetadata(meta2) 247 if err != nil { 248 return false, err 249 } 250 return string(yaml1) == string(yaml2), nil 251 } 252 253 // marshalCloudMetadata marshals the given clouds to YAML. 254 func marshalCloudMetadata(cloudsMap map[string]Cloud) ([]byte, error) { 255 clouds := cloudSet{make(map[string]*cloud)} 256 for name, metadata := range cloudsMap { 257 var regions regions 258 for _, r := range metadata.Regions { 259 regions.Slice = append(regions.Slice, yaml.MapItem{ 260 r.Name, region{r.Endpoint, r.StorageEndpoint}, 261 }) 262 } 263 clouds.Clouds[name] = &cloud{ 264 Type: metadata.Type, 265 AuthTypes: metadata.AuthTypes, 266 Endpoint: metadata.Endpoint, 267 StorageEndpoint: metadata.StorageEndpoint, 268 Regions: regions, 269 } 270 } 271 data, err := yaml.Marshal(clouds) 272 if err != nil { 273 return nil, errors.Annotate(err, "cannot marshal cloud metadata") 274 } 275 return data, nil 276 } 277 278 // MarshalYAML implements the yaml.Marshaler interface. 279 func (r regions) MarshalYAML() (interface{}, error) { 280 return r.Slice, nil 281 } 282 283 // UnmarshalYAML implements the yaml.Unmarshaler interface. 284 func (r *regions) UnmarshalYAML(f func(interface{}) error) error { 285 if err := f(&r.Map); err != nil { 286 return err 287 } 288 return f(&r.Slice) 289 } 290 291 // To keep the metadata concise, attributes on the metadata struct which 292 // have the same value for each item may be moved up to a higher level in 293 // the tree. denormaliseMetadata descends the tree and fills in any missing 294 // attributes with values from a higher level. 295 func (cloud Cloud) denormaliseMetadata() { 296 for name, region := range cloud.Regions { 297 r := region 298 inherit(&r, &cloud) 299 cloud.Regions[name] = r 300 } 301 } 302 303 type structTags map[reflect.Type]map[string]int 304 305 var tagsForType structTags = make(structTags) 306 307 // RegisterStructTags ensures the yaml tags for the given structs are able to be used 308 // when parsing cloud metadata. 309 func RegisterStructTags(vals ...interface{}) { 310 tags := mkTags(vals...) 311 for k, v := range tags { 312 tagsForType[k] = v 313 } 314 } 315 316 func init() { 317 RegisterStructTags(Cloud{}, Region{}) 318 } 319 320 func mkTags(vals ...interface{}) map[reflect.Type]map[string]int { 321 typeMap := make(map[reflect.Type]map[string]int) 322 for _, v := range vals { 323 t := reflect.TypeOf(v) 324 typeMap[t] = yamlTags(t) 325 } 326 return typeMap 327 } 328 329 // yamlTags returns a map from yaml tag to the field index for the string fields in the given type. 330 func yamlTags(t reflect.Type) map[string]int { 331 if t.Kind() != reflect.Struct { 332 panic(errors.Errorf("cannot get yaml tags on type %s", t)) 333 } 334 tags := make(map[string]int) 335 for i := 0; i < t.NumField(); i++ { 336 f := t.Field(i) 337 if f.Type != reflect.TypeOf("") { 338 continue 339 } 340 if tag := f.Tag.Get("yaml"); tag != "" { 341 if i := strings.Index(tag, ","); i >= 0 { 342 tag = tag[0:i] 343 } 344 if tag == "-" { 345 continue 346 } 347 if tag != "" { 348 f.Name = tag 349 } 350 } 351 tags[f.Name] = i 352 } 353 return tags 354 } 355 356 // inherit sets any blank fields in dst to their equivalent values in fields in src that have matching json tags. 357 // The dst parameter must be a pointer to a struct. 358 func inherit(dst, src interface{}) { 359 for tag := range tags(dst) { 360 setFieldByTag(dst, tag, fieldByTag(src, tag), false) 361 } 362 } 363 364 // tags returns the field offsets for the JSON tags defined by the given value, which must be 365 // a struct or a pointer to a struct. 366 func tags(x interface{}) map[string]int { 367 t := reflect.TypeOf(x) 368 if t.Kind() == reflect.Ptr { 369 t = t.Elem() 370 } 371 if t.Kind() != reflect.Struct { 372 panic(errors.Errorf("expected struct, not %s", t)) 373 } 374 375 if tagm := tagsForType[t]; tagm != nil { 376 return tagm 377 } 378 panic(errors.Errorf("%s not found in type table", t)) 379 } 380 381 // fieldByTag returns the value for the field in x with the given JSON tag, or "" if there is no such field. 382 func fieldByTag(x interface{}, tag string) string { 383 tagm := tags(x) 384 v := reflect.ValueOf(x) 385 if v.Kind() == reflect.Ptr { 386 v = v.Elem() 387 } 388 if i, ok := tagm[tag]; ok { 389 return v.Field(i).Interface().(string) 390 } 391 return "" 392 } 393 394 // setFieldByTag sets the value for the field in x with the given JSON tag to val. 395 // The override parameter specifies whether the value will be set even if the original value is non-empty. 396 func setFieldByTag(x interface{}, tag, val string, override bool) { 397 i, ok := tags(x)[tag] 398 if !ok { 399 return 400 } 401 v := reflect.ValueOf(x).Elem() 402 f := v.Field(i) 403 if override || f.Interface().(string) == "" { 404 f.Set(reflect.ValueOf(val)) 405 } 406 }