github.com/mattyw/juju@v0.0.0-20140610034352-732aecd63861/environs/imagemetadata/simplestreams.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 // The imagemetadata package supports locating, parsing, and filtering Ubuntu image metadata in simplestreams format. 5 // See http://launchpad.net/simplestreams and in particular the doc/README file in that project for more information 6 // about the file formats. 7 package imagemetadata 8 9 import ( 10 "fmt" 11 "sort" 12 13 "github.com/juju/juju/environs/simplestreams" 14 "github.com/juju/juju/juju/arch" 15 "github.com/juju/juju/version/ubuntu" 16 ) 17 18 func init() { 19 simplestreams.RegisterStructTags(ImageMetadata{}) 20 } 21 22 const ( 23 ImageIds = "image-ids" 24 ) 25 26 // simplestreamsImagesPublicKey is the public key required to 27 // authenticate the simple streams data on http://cloud-images.ubuntu.com. 28 // Declared as a var so it can be overidden for testing. 29 // See http://bazaar.launchpad.net/~smoser/simplestreams/trunk/view/head:/examples/keys/cloud-images.pub 30 var simplestreamsImagesPublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- 31 Version: GnuPG v1.4.12 (GNU/Linux) 32 33 mQINBFCMc9EBEADDKn9mOi9VZhW+0cxmu3aFZWMg0p7NEKuIokkEdd6P+BRITccO 34 ddDLaBuuamMbt/V1vrxWC5J+UXe33TwgO6KGfH+ECnXD5gYdEOyjVKkUyIzYV5RV 35 U5BMrxTukHuh+PkcMVUy5vossCk9MivtCRIqM6eRqfeXv6IBV9MFkAbG3x96ZNI/ 36 TqaWTlaHGszz2Axf9JccHCNfb3muLI2uVnUaojtDiZPm9SHTn6O0p7Tz7M7+P8qy 37 vc6bdn5FYAk+Wbo+zejYVBG/HLLE4+fNZPESGVCWZtbZODBPxppTnNVm3E84CTFt 38 pmWFBvBE/q2G9e8s5/mP2ATrzLdUKMxr3vcbNX+NY1Uyvn0Z02PjbxThiz1G+4qh 39 6Ct7gprtwXPOB/bCITZL9YLrchwXiNgLLKcGF0XjlpD1hfELGi0aPZaHFLAa6qq8 40 Ro9WSJljY/Z0g3woj6sXpM9TdWe/zaWhxBGmteJl33WBV7a1GucN0zF1dHIvev4F 41 krp13Uej3bMWLKUWCmZ01OHStLASshTqVxIBj2rgsxIcqH66DKTSdZWyBQtgm/kC 42 qBvuoQLFfUgIlGZihTQ96YZXqn+VfBiFbpnh1vLt24CfnVdKmzibp48KkhfqduDE 43 Xxx/f/uZENH7t8xCuNd3p+u1zemGNnxuO8jxS6Ico3bvnJaG4DAl48vaBQARAQAB 44 tG9VYnVudHUgQ2xvdWQgSW1hZ2UgQnVpbGRlciAoQ2Fub25pY2FsIEludGVybmFs 45 IENsb3VkIEltYWdlIEJ1aWxkZXIpIDx1YnVudHUtY2xvdWRidWlsZGVyLW5vcmVw 46 bHlAY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAlCMc9ECGwMGCwkIBwMCBhUIAgkK 47 CwQWAgMBAh4BAheAAAoJEH/z9AhHbPEAvRIQAMLE4ZMYiLvwSoWPAicM+3FInaqP 48 2rf1ZEf1k6175/G2n8cG3vK0nIFQE9Cus+ty2LrTggm79onV2KBGGScKe3ga+meO 49 txj601Wd7zde10IWUa1wlTxPXBxLo6tpF4s4aw6xWOf4OFqYfPU4esKblFYn1eMK 50 Dd53s3/123u8BZqzFC8WSMokY6WgBa+hvr5J3qaNT95UXo1tkMf65ZXievcQJ+Hr 51 bp1m5pslHgd5PqzlultNWePwzqmHXXf14zI1QKtbc4UjXPQ+a59ulZLVdcpvmbjx 52 HdZfK0NJpQX+j5PU6bMuQ3QTMscuvrH4W41/zcZPFaPkdJE5+VcYDL17DBFVzknJ 53 eC1uzNHxRqSMRQy9fzOuZ72ARojvL3+cyPR1qrqSCceX1/Kp838P2/CbeNvJxadt 54 liwI6rzUgK7mq1Bw5LTyBo3mLwzRJ0+eJHevNpxl6VoFyuoA3rCeoyE4on3oah1G 55 iAJt576xXMDoa1Gdj3YtnZItEaX3jb9ZB3iz9WkzZWlZsssdyZMNmpYV30Ayj3CE 56 KyurYF9lzIQWyYsNPBoXORNh73jkHJmL6g1sdMaxAZeQqKqznXbuhBbt8lkbEHMJ 57 Stxc2IGZaNpQ+/3LCwbwCphVnSMq+xl3iLg6c0s4uRn6FGX+8aknmc/fepvRe+ba 58 ntqvgz+SMPKrjeevuQINBFCMc9EBEADKGFPKBL7/pMSTKf5YH1zhFH2lr7tf5hbz 59 ztsx6j3y+nODiaQumdG+TPMbrFlgRlJ6Ah1FTuJZqdPYObGSQ7qd/VvvYZGnDYJv 60 Z1kPkNDmCJrWJs+6PwNARvyLw2bMtjCIOAq/k8wByKkMzegobJgWsbr2Jb5fT4cv 61 FxYpm3l0QxQSw49rriO5HmwyiyG1ncvaFUcpxXJY8A2s7qX1jmjsqDY1fWsv5PaN 62 ue0Fr3VXfOi9p+0CfaPY0Pl4GHzat/D+wLwnOhnjl3hFtfbhY5bPl5+cD51SbOnh 63 2nFv+bUK5HxiZlz0bw8hTUBN3oSbAC+2zViRD/9GaBYY1QjimOuAfpO1GZmqohVI 64 msZKxHNIIsk5H98mN2+LB3vH+B6zrSMDm3d2Hi7ZA8wH26mLIKLbVkh7hr8RGQjf 65 UZRxeQEf+f8F3KVoSqmfXGJfBMUtGQMTkaIeEFpMobVeHZZ3wk+Wj3dCMZ6bbt2i 66 QBaoa7SU5ZmRShJkPJzCG3SkqN+g9ZcbFMQsybl+wLN7UnZ2MbSk7JEy6SLsyuVi 67 7EjLmqHmG2gkybisnTu3wjJezpG12oz//cuylOzjuPWUWowVQQiLs3oANzYdZ0Hp 68 SuNjjtEILSRnN5FAeogs0AKH6sy3kKjxtlj764CIgn1hNidSr2Hyb4xbJ/1GE3Rk 69 sjJi6uYIJwARAQABiQIfBBgBAgAJBQJQjHPRAhsMAAoJEH/z9AhHbPEA6IsP/3jJ 70 DaowJcKOBhU2TXZglHM+ZRMauHRZavo+xAKmqgQc/izgtyMxsLwJQ+wcTEQT5uqE 71 4DoWH2T7DGiHZd/89Qe6HuRExR4p7lQwUop7kdoabqm1wQfcqr+77Znp1+KkRDyS 72 lWfbsh9ARU6krQGryODEOpXJdqdzTgYhdbVRxq6dUopz1Gf+XDreFgnqJ+okGve2 73 fJGERKYynUmHxkFZJPWZg5ifeGVt+YY6vuOCg489dzx/CmULpjZeiOQmWyqUzqy2 74 QJ70/sC8BJYCjsESId9yPmgdDoMFd+gf3jhjpuZ0JHTeUUw+ncf+1kRf7LAALPJp 75 2PTSo7VXUwoEXDyUTM+dI02dIMcjTcY4yxvnpxRFFOtklvXt8Pwa9x/aCmJb9f0E 76 5FO0nj7l9pRd2g7UCJWETFRfSW52iktvdtDrBCft9OytmTl492wAmgbbGeoRq3ze 77 QtzkRx9cPiyNQokjXXF+SQcq586oEd8K/JUSFPdvth3IoKlfnXSQnt/hRKv71kbZ 78 IXmR3B/q5x2Msr+NfUxyXfUnYOZ5KertdprUfbZjudjmQ78LOvqPF8TdtHg3gD2H 79 +G2z+IoH7qsOsc7FaJsIIa4+dljwV3QZTE7JFmsas90bRcMuM4D37p3snOpHAHY3 80 p7vH1ewg+vd9ySST0+OkWXYpbMOIARfBKyrGM3nu 81 =+MFT 82 -----END PGP PUBLIC KEY BLOCK----- 83 ` 84 85 const ( 86 // The location where Ubuntu cloud image metadata is published for 87 // public consumption. 88 UbuntuCloudImagesURL = "http://cloud-images.ubuntu.com" 89 // The path where released image metadata is found. 90 ReleasedImagesPath = "releases" 91 ) 92 93 // This needs to be a var so we can override it for testing and in bootstrap. 94 var DefaultBaseURL = UbuntuCloudImagesURL 95 96 // ImageConstraint defines criteria used to find an image metadata record. 97 type ImageConstraint struct { 98 simplestreams.LookupParams 99 } 100 101 func NewImageConstraint(params simplestreams.LookupParams) *ImageConstraint { 102 if len(params.Series) == 0 { 103 params.Series = ubuntu.SupportedSeries() 104 } 105 if len(params.Arches) == 0 { 106 params.Arches = arch.AllSupportedArches 107 } 108 return &ImageConstraint{LookupParams: params} 109 } 110 111 const ( 112 // Used to specify the released image metadata. 113 ReleasedStream = "released" 114 ) 115 116 // idStream returns the string to use in making a product id 117 // for the given product stream. 118 func idStream(stream string) string { 119 idstream := "" 120 if stream != "" && stream != ReleasedStream { 121 idstream = "." + stream 122 } 123 return idstream 124 } 125 126 // Generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN). 127 func (ic *ImageConstraint) Ids() ([]string, error) { 128 stream := idStream(ic.Stream) 129 nrArches := len(ic.Arches) 130 nrSeries := len(ic.Series) 131 ids := make([]string, nrArches*nrSeries) 132 for i, arch := range ic.Arches { 133 for j, series := range ic.Series { 134 version, err := ubuntu.SeriesVersion(series) 135 if err != nil { 136 return nil, err 137 } 138 ids[j*nrArches+i] = fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, version, arch) 139 } 140 } 141 return ids, nil 142 } 143 144 // ImageMetadata holds information about a particular cloud image. 145 type ImageMetadata struct { 146 Id string `json:"id"` 147 Storage string `json:"root_store,omitempty"` 148 VirtType string `json:"virt,omitempty"` 149 Arch string `json:"arch,omitempty"` 150 Version string `json:"version,omitempty"` 151 RegionAlias string `json:"crsn,omitempty"` 152 RegionName string `json:"region,omitempty"` 153 Endpoint string `json:"endpoint,omitempty"` 154 Stream string `json:"-"` 155 } 156 157 func (im *ImageMetadata) String() string { 158 return fmt.Sprintf("%#v", im) 159 } 160 161 func (im *ImageMetadata) productId() string { 162 stream := idStream(im.Stream) 163 return fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, im.Version, im.Arch) 164 } 165 166 // Fetch returns a list of images for the specified cloud matching the constraint. 167 // The base URL locations are as specified - the first location which has a file is the one used. 168 // Signed data is preferred, but if there is no signed data available and onlySigned is false, 169 // then unsigned data is used. 170 func Fetch( 171 sources []simplestreams.DataSource, indexPath string, cons *ImageConstraint, 172 onlySigned bool) ([]*ImageMetadata, *simplestreams.ResolveInfo, error) { 173 params := simplestreams.ValueParams{ 174 DataType: ImageIds, 175 FilterFunc: appendMatchingImages, 176 ValueTemplate: ImageMetadata{}, 177 PublicKey: simplestreamsImagesPublicKey, 178 } 179 items, resolveInfo, err := simplestreams.GetMetadata(sources, indexPath, cons, onlySigned, params) 180 if err != nil { 181 return nil, resolveInfo, err 182 } 183 metadata := make([]*ImageMetadata, len(items)) 184 for i, md := range items { 185 metadata[i] = md.(*ImageMetadata) 186 } 187 // Sorting the metadata is not strictly necessary, but it ensures consistent ordering for 188 // all compilers, and it just makes it easier to look at the data. 189 Sort(metadata) 190 return metadata, resolveInfo, nil 191 } 192 193 // Sort sorts a slice of ImageMetadata in ascending order of their id 194 // in order to ensure the results of Fetch are ordered deterministically. 195 func Sort(metadata []*ImageMetadata) { 196 sort.Sort(byId(metadata)) 197 } 198 199 type byId []*ImageMetadata 200 201 func (b byId) Len() int { return len(b) } 202 func (b byId) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 203 func (b byId) Less(i, j int) bool { return b[i].Id < b[j].Id } 204 205 type imageKey struct { 206 vtype string 207 arch string 208 version string 209 region string 210 storage string 211 } 212 213 // appendMatchingImages updates matchingImages with image metadata records from images which belong to the 214 // specified region. If an image already exists in matchingImages, it is not overwritten. 215 func appendMatchingImages(source simplestreams.DataSource, matchingImages []interface{}, 216 images map[string]interface{}, cons simplestreams.LookupConstraint) []interface{} { 217 218 imagesMap := make(map[imageKey]*ImageMetadata, len(matchingImages)) 219 for _, val := range matchingImages { 220 im := val.(*ImageMetadata) 221 imagesMap[imageKey{im.VirtType, im.Arch, im.Version, im.RegionName, im.Storage}] = im 222 } 223 for _, val := range images { 224 im := val.(*ImageMetadata) 225 if cons != nil && cons.Params().Region != "" && cons.Params().Region != im.RegionName { 226 continue 227 } 228 if _, ok := imagesMap[imageKey{im.VirtType, im.Arch, im.Version, im.RegionName, im.Storage}]; !ok { 229 matchingImages = append(matchingImages, im) 230 } 231 } 232 return matchingImages 233 } 234 235 // GetLatestImageIdMetadata is provided so it can be call by tests outside the imagemetadata package. 236 func GetLatestImageIdMetadata(data []byte, source simplestreams.DataSource, cons *ImageConstraint) ([]*ImageMetadata, error) { 237 metadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", "<unknown>", ImageMetadata{}) 238 if err != nil { 239 return nil, err 240 } 241 items, err := simplestreams.GetLatestMetadata(metadata, cons, source, appendMatchingImages) 242 if err != nil { 243 return nil, err 244 } 245 result := make([]*ImageMetadata, len(items)) 246 for i, md := range items { 247 result[i] = md.(*ImageMetadata) 248 } 249 return result, nil 250 }