github.com/mattyw/juju@v0.0.0-20140610034352-732aecd63861/environs/imagemetadata/simplestreams.go (about)

     1  // Copyright 2013 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  // The imagemetadata package supports locating, parsing, and filtering Ubuntu image metadata in simplestreams format.
     5  // See http://launchpad.net/simplestreams and in particular the doc/README file in that project for more information
     6  // about the file formats.
     7  package imagemetadata
     8  
     9  import (
    10  	"fmt"
    11  	"sort"
    12  
    13  	"github.com/juju/juju/environs/simplestreams"
    14  	"github.com/juju/juju/juju/arch"
    15  	"github.com/juju/juju/version/ubuntu"
    16  )
    17  
    18  func init() {
    19  	simplestreams.RegisterStructTags(ImageMetadata{})
    20  }
    21  
    22  const (
    23  	ImageIds = "image-ids"
    24  )
    25  
    26  // simplestreamsImagesPublicKey is the public key required to
    27  // authenticate the simple streams data on http://cloud-images.ubuntu.com.
    28  // Declared as a var so it can be overidden for testing.
    29  // See http://bazaar.launchpad.net/~smoser/simplestreams/trunk/view/head:/examples/keys/cloud-images.pub
    30  var simplestreamsImagesPublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK-----
    31  Version: GnuPG v1.4.12 (GNU/Linux)
    32  
    33  mQINBFCMc9EBEADDKn9mOi9VZhW+0cxmu3aFZWMg0p7NEKuIokkEdd6P+BRITccO
    34  ddDLaBuuamMbt/V1vrxWC5J+UXe33TwgO6KGfH+ECnXD5gYdEOyjVKkUyIzYV5RV
    35  U5BMrxTukHuh+PkcMVUy5vossCk9MivtCRIqM6eRqfeXv6IBV9MFkAbG3x96ZNI/
    36  TqaWTlaHGszz2Axf9JccHCNfb3muLI2uVnUaojtDiZPm9SHTn6O0p7Tz7M7+P8qy
    37  vc6bdn5FYAk+Wbo+zejYVBG/HLLE4+fNZPESGVCWZtbZODBPxppTnNVm3E84CTFt
    38  pmWFBvBE/q2G9e8s5/mP2ATrzLdUKMxr3vcbNX+NY1Uyvn0Z02PjbxThiz1G+4qh
    39  6Ct7gprtwXPOB/bCITZL9YLrchwXiNgLLKcGF0XjlpD1hfELGi0aPZaHFLAa6qq8
    40  Ro9WSJljY/Z0g3woj6sXpM9TdWe/zaWhxBGmteJl33WBV7a1GucN0zF1dHIvev4F
    41  krp13Uej3bMWLKUWCmZ01OHStLASshTqVxIBj2rgsxIcqH66DKTSdZWyBQtgm/kC
    42  qBvuoQLFfUgIlGZihTQ96YZXqn+VfBiFbpnh1vLt24CfnVdKmzibp48KkhfqduDE
    43  Xxx/f/uZENH7t8xCuNd3p+u1zemGNnxuO8jxS6Ico3bvnJaG4DAl48vaBQARAQAB
    44  tG9VYnVudHUgQ2xvdWQgSW1hZ2UgQnVpbGRlciAoQ2Fub25pY2FsIEludGVybmFs
    45  IENsb3VkIEltYWdlIEJ1aWxkZXIpIDx1YnVudHUtY2xvdWRidWlsZGVyLW5vcmVw
    46  bHlAY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAlCMc9ECGwMGCwkIBwMCBhUIAgkK
    47  CwQWAgMBAh4BAheAAAoJEH/z9AhHbPEAvRIQAMLE4ZMYiLvwSoWPAicM+3FInaqP
    48  2rf1ZEf1k6175/G2n8cG3vK0nIFQE9Cus+ty2LrTggm79onV2KBGGScKe3ga+meO
    49  txj601Wd7zde10IWUa1wlTxPXBxLo6tpF4s4aw6xWOf4OFqYfPU4esKblFYn1eMK
    50  Dd53s3/123u8BZqzFC8WSMokY6WgBa+hvr5J3qaNT95UXo1tkMf65ZXievcQJ+Hr
    51  bp1m5pslHgd5PqzlultNWePwzqmHXXf14zI1QKtbc4UjXPQ+a59ulZLVdcpvmbjx
    52  HdZfK0NJpQX+j5PU6bMuQ3QTMscuvrH4W41/zcZPFaPkdJE5+VcYDL17DBFVzknJ
    53  eC1uzNHxRqSMRQy9fzOuZ72ARojvL3+cyPR1qrqSCceX1/Kp838P2/CbeNvJxadt
    54  liwI6rzUgK7mq1Bw5LTyBo3mLwzRJ0+eJHevNpxl6VoFyuoA3rCeoyE4on3oah1G
    55  iAJt576xXMDoa1Gdj3YtnZItEaX3jb9ZB3iz9WkzZWlZsssdyZMNmpYV30Ayj3CE
    56  KyurYF9lzIQWyYsNPBoXORNh73jkHJmL6g1sdMaxAZeQqKqznXbuhBbt8lkbEHMJ
    57  Stxc2IGZaNpQ+/3LCwbwCphVnSMq+xl3iLg6c0s4uRn6FGX+8aknmc/fepvRe+ba
    58  ntqvgz+SMPKrjeevuQINBFCMc9EBEADKGFPKBL7/pMSTKf5YH1zhFH2lr7tf5hbz
    59  ztsx6j3y+nODiaQumdG+TPMbrFlgRlJ6Ah1FTuJZqdPYObGSQ7qd/VvvYZGnDYJv
    60  Z1kPkNDmCJrWJs+6PwNARvyLw2bMtjCIOAq/k8wByKkMzegobJgWsbr2Jb5fT4cv
    61  FxYpm3l0QxQSw49rriO5HmwyiyG1ncvaFUcpxXJY8A2s7qX1jmjsqDY1fWsv5PaN
    62  ue0Fr3VXfOi9p+0CfaPY0Pl4GHzat/D+wLwnOhnjl3hFtfbhY5bPl5+cD51SbOnh
    63  2nFv+bUK5HxiZlz0bw8hTUBN3oSbAC+2zViRD/9GaBYY1QjimOuAfpO1GZmqohVI
    64  msZKxHNIIsk5H98mN2+LB3vH+B6zrSMDm3d2Hi7ZA8wH26mLIKLbVkh7hr8RGQjf
    65  UZRxeQEf+f8F3KVoSqmfXGJfBMUtGQMTkaIeEFpMobVeHZZ3wk+Wj3dCMZ6bbt2i
    66  QBaoa7SU5ZmRShJkPJzCG3SkqN+g9ZcbFMQsybl+wLN7UnZ2MbSk7JEy6SLsyuVi
    67  7EjLmqHmG2gkybisnTu3wjJezpG12oz//cuylOzjuPWUWowVQQiLs3oANzYdZ0Hp
    68  SuNjjtEILSRnN5FAeogs0AKH6sy3kKjxtlj764CIgn1hNidSr2Hyb4xbJ/1GE3Rk
    69  sjJi6uYIJwARAQABiQIfBBgBAgAJBQJQjHPRAhsMAAoJEH/z9AhHbPEA6IsP/3jJ
    70  DaowJcKOBhU2TXZglHM+ZRMauHRZavo+xAKmqgQc/izgtyMxsLwJQ+wcTEQT5uqE
    71  4DoWH2T7DGiHZd/89Qe6HuRExR4p7lQwUop7kdoabqm1wQfcqr+77Znp1+KkRDyS
    72  lWfbsh9ARU6krQGryODEOpXJdqdzTgYhdbVRxq6dUopz1Gf+XDreFgnqJ+okGve2
    73  fJGERKYynUmHxkFZJPWZg5ifeGVt+YY6vuOCg489dzx/CmULpjZeiOQmWyqUzqy2
    74  QJ70/sC8BJYCjsESId9yPmgdDoMFd+gf3jhjpuZ0JHTeUUw+ncf+1kRf7LAALPJp
    75  2PTSo7VXUwoEXDyUTM+dI02dIMcjTcY4yxvnpxRFFOtklvXt8Pwa9x/aCmJb9f0E
    76  5FO0nj7l9pRd2g7UCJWETFRfSW52iktvdtDrBCft9OytmTl492wAmgbbGeoRq3ze
    77  QtzkRx9cPiyNQokjXXF+SQcq586oEd8K/JUSFPdvth3IoKlfnXSQnt/hRKv71kbZ
    78  IXmR3B/q5x2Msr+NfUxyXfUnYOZ5KertdprUfbZjudjmQ78LOvqPF8TdtHg3gD2H
    79  +G2z+IoH7qsOsc7FaJsIIa4+dljwV3QZTE7JFmsas90bRcMuM4D37p3snOpHAHY3
    80  p7vH1ewg+vd9ySST0+OkWXYpbMOIARfBKyrGM3nu
    81  =+MFT
    82  -----END PGP PUBLIC KEY BLOCK-----
    83  `
    84  
    85  const (
    86  	// The location where Ubuntu cloud image metadata is published for
    87  	// public consumption.
    88  	UbuntuCloudImagesURL = "http://cloud-images.ubuntu.com"
    89  	// The path where released image metadata is found.
    90  	ReleasedImagesPath = "releases"
    91  )
    92  
    93  // This needs to be a var so we can override it for testing and in bootstrap.
    94  var DefaultBaseURL = UbuntuCloudImagesURL
    95  
    96  // ImageConstraint defines criteria used to find an image metadata record.
    97  type ImageConstraint struct {
    98  	simplestreams.LookupParams
    99  }
   100  
   101  func NewImageConstraint(params simplestreams.LookupParams) *ImageConstraint {
   102  	if len(params.Series) == 0 {
   103  		params.Series = ubuntu.SupportedSeries()
   104  	}
   105  	if len(params.Arches) == 0 {
   106  		params.Arches = arch.AllSupportedArches
   107  	}
   108  	return &ImageConstraint{LookupParams: params}
   109  }
   110  
   111  const (
   112  	// Used to specify the released image metadata.
   113  	ReleasedStream = "released"
   114  )
   115  
   116  // idStream returns the string to use in making a product id
   117  // for the given product stream.
   118  func idStream(stream string) string {
   119  	idstream := ""
   120  	if stream != "" && stream != ReleasedStream {
   121  		idstream = "." + stream
   122  	}
   123  	return idstream
   124  }
   125  
   126  // Generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN).
   127  func (ic *ImageConstraint) Ids() ([]string, error) {
   128  	stream := idStream(ic.Stream)
   129  	nrArches := len(ic.Arches)
   130  	nrSeries := len(ic.Series)
   131  	ids := make([]string, nrArches*nrSeries)
   132  	for i, arch := range ic.Arches {
   133  		for j, series := range ic.Series {
   134  			version, err := ubuntu.SeriesVersion(series)
   135  			if err != nil {
   136  				return nil, err
   137  			}
   138  			ids[j*nrArches+i] = fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, version, arch)
   139  		}
   140  	}
   141  	return ids, nil
   142  }
   143  
   144  // ImageMetadata holds information about a particular cloud image.
   145  type ImageMetadata struct {
   146  	Id          string `json:"id"`
   147  	Storage     string `json:"root_store,omitempty"`
   148  	VirtType    string `json:"virt,omitempty"`
   149  	Arch        string `json:"arch,omitempty"`
   150  	Version     string `json:"version,omitempty"`
   151  	RegionAlias string `json:"crsn,omitempty"`
   152  	RegionName  string `json:"region,omitempty"`
   153  	Endpoint    string `json:"endpoint,omitempty"`
   154  	Stream      string `json:"-"`
   155  }
   156  
   157  func (im *ImageMetadata) String() string {
   158  	return fmt.Sprintf("%#v", im)
   159  }
   160  
   161  func (im *ImageMetadata) productId() string {
   162  	stream := idStream(im.Stream)
   163  	return fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, im.Version, im.Arch)
   164  }
   165  
   166  // Fetch returns a list of images for the specified cloud matching the constraint.
   167  // The base URL locations are as specified - the first location which has a file is the one used.
   168  // Signed data is preferred, but if there is no signed data available and onlySigned is false,
   169  // then unsigned data is used.
   170  func Fetch(
   171  	sources []simplestreams.DataSource, indexPath string, cons *ImageConstraint,
   172  	onlySigned bool) ([]*ImageMetadata, *simplestreams.ResolveInfo, error) {
   173  	params := simplestreams.ValueParams{
   174  		DataType:      ImageIds,
   175  		FilterFunc:    appendMatchingImages,
   176  		ValueTemplate: ImageMetadata{},
   177  		PublicKey:     simplestreamsImagesPublicKey,
   178  	}
   179  	items, resolveInfo, err := simplestreams.GetMetadata(sources, indexPath, cons, onlySigned, params)
   180  	if err != nil {
   181  		return nil, resolveInfo, err
   182  	}
   183  	metadata := make([]*ImageMetadata, len(items))
   184  	for i, md := range items {
   185  		metadata[i] = md.(*ImageMetadata)
   186  	}
   187  	// Sorting the metadata is not strictly necessary, but it ensures consistent ordering for
   188  	// all compilers, and it just makes it easier to look at the data.
   189  	Sort(metadata)
   190  	return metadata, resolveInfo, nil
   191  }
   192  
   193  // Sort sorts a slice of ImageMetadata in ascending order of their id
   194  // in order to ensure the results of Fetch are ordered deterministically.
   195  func Sort(metadata []*ImageMetadata) {
   196  	sort.Sort(byId(metadata))
   197  }
   198  
   199  type byId []*ImageMetadata
   200  
   201  func (b byId) Len() int           { return len(b) }
   202  func (b byId) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
   203  func (b byId) Less(i, j int) bool { return b[i].Id < b[j].Id }
   204  
   205  type imageKey struct {
   206  	vtype   string
   207  	arch    string
   208  	version string
   209  	region  string
   210  	storage string
   211  }
   212  
   213  // appendMatchingImages updates matchingImages with image metadata records from images which belong to the
   214  // specified region. If an image already exists in matchingImages, it is not overwritten.
   215  func appendMatchingImages(source simplestreams.DataSource, matchingImages []interface{},
   216  	images map[string]interface{}, cons simplestreams.LookupConstraint) []interface{} {
   217  
   218  	imagesMap := make(map[imageKey]*ImageMetadata, len(matchingImages))
   219  	for _, val := range matchingImages {
   220  		im := val.(*ImageMetadata)
   221  		imagesMap[imageKey{im.VirtType, im.Arch, im.Version, im.RegionName, im.Storage}] = im
   222  	}
   223  	for _, val := range images {
   224  		im := val.(*ImageMetadata)
   225  		if cons != nil && cons.Params().Region != "" && cons.Params().Region != im.RegionName {
   226  			continue
   227  		}
   228  		if _, ok := imagesMap[imageKey{im.VirtType, im.Arch, im.Version, im.RegionName, im.Storage}]; !ok {
   229  			matchingImages = append(matchingImages, im)
   230  		}
   231  	}
   232  	return matchingImages
   233  }
   234  
   235  // GetLatestImageIdMetadata is provided so it can be call by tests outside the imagemetadata package.
   236  func GetLatestImageIdMetadata(data []byte, source simplestreams.DataSource, cons *ImageConstraint) ([]*ImageMetadata, error) {
   237  	metadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", "<unknown>", ImageMetadata{})
   238  	if err != nil {
   239  		return nil, err
   240  	}
   241  	items, err := simplestreams.GetLatestMetadata(metadata, cons, source, appendMatchingImages)
   242  	if err != nil {
   243  		return nil, err
   244  	}
   245  	result := make([]*ImageMetadata, len(items))
   246  	for i, md := range items {
   247  		result[i] = md.(*ImageMetadata)
   248  	}
   249  	return result, nil
   250  }