github.com/mwhudson/juju@v0.0.0-20160512215208-90ff01f3497f/cloud/clouds.go (about)

     1  // Copyright 2015 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  // Package cloud provides functionality to parse information
     5  // describing clouds, including regions, supported auth types etc.
     6  
     7  package cloud
     8  
     9  import (
    10  	"fmt"
    11  	"io/ioutil"
    12  	"os"
    13  	"reflect"
    14  	"strings"
    15  
    16  	"github.com/juju/errors"
    17  	"github.com/juju/utils"
    18  	"gopkg.in/yaml.v2"
    19  
    20  	"github.com/juju/juju/juju/osenv"
    21  )
    22  
    23  //go:generate go run ../generate/filetoconst.go fallbackPublicCloudInfo fallback-public-cloud.yaml fallback_public_cloud.go 2015
    24  
    25  // AuthType is the type of authentication used by the cloud.
    26  type AuthType string
    27  
    28  const (
    29  	// AccessKeyAuthType is an authentication type using a key and secret.
    30  	AccessKeyAuthType AuthType = "access-key"
    31  
    32  	// UserPassAuthType is an authentication type using a username and password.
    33  	UserPassAuthType AuthType = "userpass"
    34  
    35  	// OAuth1AuthType is an authentication type using oauth1.
    36  	OAuth1AuthType AuthType = "oauth1"
    37  
    38  	// OAuth2AuthType is an authentication type using oauth2.
    39  	OAuth2AuthType AuthType = "oauth2"
    40  
    41  	// JSONFileAuthType is an authentication type that takes a path to
    42  	// a JSON file.
    43  	JSONFileAuthType AuthType = "jsonfile"
    44  
    45  	// EmptyAuthType is the authentication type used for providers
    46  	// that require no credentials, e.g. "lxd", and "manual".
    47  	EmptyAuthType AuthType = "empty"
    48  )
    49  
    50  // Cloud is a cloud definition.
    51  type Cloud struct {
    52  	// Type is the type of cloud, eg aws, openstack etc.
    53  	Type string
    54  
    55  	// AuthTypes are the authentication modes supported by the cloud.
    56  	AuthTypes []AuthType
    57  
    58  	// Endpoint is the default endpoint for the cloud regions, may be
    59  	// overridden by a region.
    60  	Endpoint string
    61  
    62  	// StorageEndpoint is the default storage endpoint for the cloud
    63  	// regions, may be overridden by a region.
    64  	StorageEndpoint string
    65  
    66  	// Regions are the regions available in the cloud.
    67  	//
    68  	// Regions is a slice, and not a map, because order is important.
    69  	// The first region in the slice is the default region for the
    70  	// cloud.
    71  	Regions []Region
    72  }
    73  
    74  // Region is a cloud region.
    75  type Region struct {
    76  	// Name is the name of the region.
    77  	Name string
    78  
    79  	// Endpoint is the region's primary endpoint URL.
    80  	Endpoint string
    81  
    82  	// StorageEndpoint is the region's storage endpoint URL.
    83  	// If the cloud/region does not have a storage-specific
    84  	// endpoint URL, this will be empty.
    85  	StorageEndpoint string
    86  }
    87  
    88  // cloudSet contains cloud definitions, used for marshalling and
    89  // unmarshalling.
    90  type cloudSet struct {
    91  	// Clouds is a map of cloud definitions, keyed on cloud name.
    92  	Clouds map[string]*cloud `yaml:"clouds"`
    93  }
    94  
    95  // cloud is equivalent to Cloud, for marshalling and unmarshalling.
    96  type cloud struct {
    97  	Type            string     `yaml:"type"`
    98  	AuthTypes       []AuthType `yaml:"auth-types,omitempty,flow"`
    99  	Endpoint        string     `yaml:"endpoint,omitempty"`
   100  	StorageEndpoint string     `yaml:"storage-endpoint,omitempty"`
   101  	Regions         regions    `yaml:"regions,omitempty"`
   102  }
   103  
   104  // regions is a collection of regions, either as a map and/or
   105  // as a yaml.MapSlice.
   106  //
   107  // When marshalling, we populate the Slice field only. This is
   108  // necessary for us to control the order of map items.
   109  //
   110  // When unmarshalling, we populate both Map and Slice. Map is
   111  // populated to simplify conversion to Region objects. Slice
   112  // is populated so we can identify the first map item, which
   113  // becomes the default region for the cloud.
   114  type regions struct {
   115  	Map   map[string]*region
   116  	Slice yaml.MapSlice
   117  }
   118  
   119  // region is equivalent to Region, for marshalling and unmarshalling.
   120  type region struct {
   121  	Endpoint        string `yaml:"endpoint,omitempty"`
   122  	StorageEndpoint string `yaml:"storage-endpoint,omitempty"`
   123  }
   124  
   125  // BuiltInClouds work out of the box.
   126  var BuiltInClouds = map[string]Cloud{
   127  	"localhost": {
   128  		Type:      "lxd",
   129  		AuthTypes: []AuthType{EmptyAuthType},
   130  		Regions:   []Region{{Name: "localhost"}},
   131  	},
   132  }
   133  
   134  // CloudByName returns the cloud with the specified name.
   135  // If there exists no cloud with the specified name, an
   136  // error satisfying errors.IsNotFound will be returned.
   137  //
   138  // TODO(axw) write unit tests for this.
   139  func CloudByName(name string) (*Cloud, error) {
   140  	// Personal clouds take precedence.
   141  	personalClouds, err := PersonalCloudMetadata()
   142  	if err != nil {
   143  		return nil, errors.Trace(err)
   144  	}
   145  	if cloud, ok := personalClouds[name]; ok {
   146  		return &cloud, nil
   147  	}
   148  	clouds, _, err := PublicCloudMetadata(JujuPublicCloudsPath())
   149  	if err != nil {
   150  		return nil, errors.Trace(err)
   151  	}
   152  	if cloud, ok := clouds[name]; ok {
   153  		return &cloud, nil
   154  	}
   155  	if cloud, ok := BuiltInClouds[name]; ok {
   156  		return &cloud, nil
   157  	}
   158  	return nil, errors.NotFoundf("cloud %s", name)
   159  }
   160  
   161  // JujuPublicCloudsPath is the location where public cloud information is
   162  // expected to be found. Requires JUJU_HOME to be set.
   163  func JujuPublicCloudsPath() string {
   164  	return osenv.JujuXDGDataHomePath("public-clouds.yaml")
   165  }
   166  
   167  // PublicCloudMetadata looks in searchPath for cloud metadata files and if none
   168  // are found, returns the fallback public cloud metadata.
   169  func PublicCloudMetadata(searchPath ...string) (result map[string]Cloud, fallbackUsed bool, err error) {
   170  	for _, file := range searchPath {
   171  		data, err := ioutil.ReadFile(file)
   172  		if err != nil && os.IsNotExist(err) {
   173  			continue
   174  		}
   175  		if err != nil {
   176  			return nil, false, errors.Trace(err)
   177  		}
   178  		clouds, err := ParseCloudMetadata(data)
   179  		if err != nil {
   180  			return nil, false, errors.Trace(err)
   181  		}
   182  		return clouds, false, err
   183  	}
   184  	clouds, err := ParseCloudMetadata([]byte(fallbackPublicCloudInfo))
   185  	return clouds, true, err
   186  }
   187  
   188  // ParseCloudMetadata parses the given yaml bytes into Clouds metadata.
   189  func ParseCloudMetadata(data []byte) (map[string]Cloud, error) {
   190  	var metadata cloudSet
   191  	if err := yaml.Unmarshal(data, &metadata); err != nil {
   192  		return nil, errors.Annotate(err, "cannot unmarshal yaml cloud metadata")
   193  	}
   194  
   195  	// Translate to the exported type. For each cloud, we store
   196  	// the first region for the cloud as its default region.
   197  	clouds := make(map[string]Cloud)
   198  	for name, cloud := range metadata.Clouds {
   199  		var regions []Region
   200  		if len(cloud.Regions.Map) > 0 {
   201  			for _, item := range cloud.Regions.Slice {
   202  				name := fmt.Sprint(item.Key)
   203  				r := cloud.Regions.Map[name]
   204  				if r == nil {
   205  					// r will be nil if none of the fields in
   206  					// the YAML are set.
   207  					regions = append(regions, Region{Name: name})
   208  				} else {
   209  					regions = append(regions, Region{
   210  						name, r.Endpoint, r.StorageEndpoint,
   211  					})
   212  				}
   213  			}
   214  		}
   215  		meta := Cloud{
   216  			Type:            cloud.Type,
   217  			AuthTypes:       cloud.AuthTypes,
   218  			Endpoint:        cloud.Endpoint,
   219  			StorageEndpoint: cloud.StorageEndpoint,
   220  			Regions:         regions,
   221  		}
   222  		meta.denormaliseMetadata()
   223  		clouds[name] = meta
   224  	}
   225  	return clouds, nil
   226  }
   227  
   228  // WritePublicCloudMetadata marshals to YAML and writes the cloud metadata
   229  // to the public cloud file.
   230  func WritePublicCloudMetadata(cloudsMap map[string]Cloud) error {
   231  	data, err := marshalCloudMetadata(cloudsMap)
   232  	if err != nil {
   233  		return errors.Trace(err)
   234  	}
   235  	return utils.AtomicWriteFile(JujuPublicCloudsPath(), data, 0600)
   236  }
   237  
   238  // IsSameCloudMetadata returns true if both meta and meta2 contain the
   239  // same cloud metadata.
   240  func IsSameCloudMetadata(meta1, meta2 map[string]Cloud) (bool, error) {
   241  	// The easiest approach is to simply marshall to YAML and compare.
   242  	yaml1, err := marshalCloudMetadata(meta1)
   243  	if err != nil {
   244  		return false, err
   245  	}
   246  	yaml2, err := marshalCloudMetadata(meta2)
   247  	if err != nil {
   248  		return false, err
   249  	}
   250  	return string(yaml1) == string(yaml2), nil
   251  }
   252  
   253  // marshalCloudMetadata marshals the given clouds to YAML.
   254  func marshalCloudMetadata(cloudsMap map[string]Cloud) ([]byte, error) {
   255  	clouds := cloudSet{make(map[string]*cloud)}
   256  	for name, metadata := range cloudsMap {
   257  		var regions regions
   258  		for _, r := range metadata.Regions {
   259  			regions.Slice = append(regions.Slice, yaml.MapItem{
   260  				r.Name, region{r.Endpoint, r.StorageEndpoint},
   261  			})
   262  		}
   263  		clouds.Clouds[name] = &cloud{
   264  			Type:            metadata.Type,
   265  			AuthTypes:       metadata.AuthTypes,
   266  			Endpoint:        metadata.Endpoint,
   267  			StorageEndpoint: metadata.StorageEndpoint,
   268  			Regions:         regions,
   269  		}
   270  	}
   271  	data, err := yaml.Marshal(clouds)
   272  	if err != nil {
   273  		return nil, errors.Annotate(err, "cannot marshal cloud metadata")
   274  	}
   275  	return data, nil
   276  }
   277  
   278  // MarshalYAML implements the yaml.Marshaler interface.
   279  func (r regions) MarshalYAML() (interface{}, error) {
   280  	return r.Slice, nil
   281  }
   282  
   283  // UnmarshalYAML implements the yaml.Unmarshaler interface.
   284  func (r *regions) UnmarshalYAML(f func(interface{}) error) error {
   285  	if err := f(&r.Map); err != nil {
   286  		return err
   287  	}
   288  	return f(&r.Slice)
   289  }
   290  
   291  // To keep the metadata concise, attributes on the metadata struct which
   292  // have the same value for each item may be moved up to a higher level in
   293  // the tree. denormaliseMetadata descends the tree and fills in any missing
   294  // attributes with values from a higher level.
   295  func (cloud Cloud) denormaliseMetadata() {
   296  	for name, region := range cloud.Regions {
   297  		r := region
   298  		inherit(&r, &cloud)
   299  		cloud.Regions[name] = r
   300  	}
   301  }
   302  
   303  type structTags map[reflect.Type]map[string]int
   304  
   305  var tagsForType structTags = make(structTags)
   306  
   307  // RegisterStructTags ensures the yaml tags for the given structs are able to be used
   308  // when parsing cloud metadata.
   309  func RegisterStructTags(vals ...interface{}) {
   310  	tags := mkTags(vals...)
   311  	for k, v := range tags {
   312  		tagsForType[k] = v
   313  	}
   314  }
   315  
   316  func init() {
   317  	RegisterStructTags(Cloud{}, Region{})
   318  }
   319  
   320  func mkTags(vals ...interface{}) map[reflect.Type]map[string]int {
   321  	typeMap := make(map[reflect.Type]map[string]int)
   322  	for _, v := range vals {
   323  		t := reflect.TypeOf(v)
   324  		typeMap[t] = yamlTags(t)
   325  	}
   326  	return typeMap
   327  }
   328  
   329  // yamlTags returns a map from yaml tag to the field index for the string fields in the given type.
   330  func yamlTags(t reflect.Type) map[string]int {
   331  	if t.Kind() != reflect.Struct {
   332  		panic(errors.Errorf("cannot get yaml tags on type %s", t))
   333  	}
   334  	tags := make(map[string]int)
   335  	for i := 0; i < t.NumField(); i++ {
   336  		f := t.Field(i)
   337  		if f.Type != reflect.TypeOf("") {
   338  			continue
   339  		}
   340  		if tag := f.Tag.Get("yaml"); tag != "" {
   341  			if i := strings.Index(tag, ","); i >= 0 {
   342  				tag = tag[0:i]
   343  			}
   344  			if tag == "-" {
   345  				continue
   346  			}
   347  			if tag != "" {
   348  				f.Name = tag
   349  			}
   350  		}
   351  		tags[f.Name] = i
   352  	}
   353  	return tags
   354  }
   355  
   356  // inherit sets any blank fields in dst to their equivalent values in fields in src that have matching json tags.
   357  // The dst parameter must be a pointer to a struct.
   358  func inherit(dst, src interface{}) {
   359  	for tag := range tags(dst) {
   360  		setFieldByTag(dst, tag, fieldByTag(src, tag), false)
   361  	}
   362  }
   363  
   364  // tags returns the field offsets for the JSON tags defined by the given value, which must be
   365  // a struct or a pointer to a struct.
   366  func tags(x interface{}) map[string]int {
   367  	t := reflect.TypeOf(x)
   368  	if t.Kind() == reflect.Ptr {
   369  		t = t.Elem()
   370  	}
   371  	if t.Kind() != reflect.Struct {
   372  		panic(errors.Errorf("expected struct, not %s", t))
   373  	}
   374  
   375  	if tagm := tagsForType[t]; tagm != nil {
   376  		return tagm
   377  	}
   378  	panic(errors.Errorf("%s not found in type table", t))
   379  }
   380  
   381  // fieldByTag returns the value for the field in x with the given JSON tag, or "" if there is no such field.
   382  func fieldByTag(x interface{}, tag string) string {
   383  	tagm := tags(x)
   384  	v := reflect.ValueOf(x)
   385  	if v.Kind() == reflect.Ptr {
   386  		v = v.Elem()
   387  	}
   388  	if i, ok := tagm[tag]; ok {
   389  		return v.Field(i).Interface().(string)
   390  	}
   391  	return ""
   392  }
   393  
   394  // setFieldByTag sets the value for the field in x with the given JSON tag to val.
   395  // The override parameter specifies whether the value will be set even if the original value is non-empty.
   396  func setFieldByTag(x interface{}, tag, val string, override bool) {
   397  	i, ok := tags(x)[tag]
   398  	if !ok {
   399  		return
   400  	}
   401  	v := reflect.ValueOf(x).Elem()
   402  	f := v.Field(i)
   403  	if override || f.Interface().(string) == "" {
   404  		f.Set(reflect.ValueOf(val))
   405  	}
   406  }