github.com/mattyw/juju@v0.0.0-20140610034352-732aecd63861/environs/tools/simplestreams.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 // The tools package supports locating, parsing, and filtering Ubuntu tools metadata in simplestreams format. 5 // See http://launchpad.net/simplestreams and in particular the doc/README file in that project for more information 6 // about the file formats. 7 package tools 8 9 import ( 10 "bytes" 11 "crypto/sha256" 12 "fmt" 13 "hash" 14 "io" 15 "path" 16 "sort" 17 "strings" 18 "time" 19 20 "github.com/juju/errors" 21 "github.com/juju/utils/set" 22 23 "github.com/juju/juju/environs/simplestreams" 24 "github.com/juju/juju/environs/storage" 25 coretools "github.com/juju/juju/tools" 26 "github.com/juju/juju/version" 27 "github.com/juju/juju/version/ubuntu" 28 ) 29 30 func init() { 31 simplestreams.RegisterStructTags(ToolsMetadata{}) 32 } 33 34 const ( 35 ContentDownload = "content-download" 36 ) 37 38 // simplestreamsToolsPublicKey is the public key required to 39 // authenticate the simple streams data on http://streams.canonical.com. 40 // Declared as a var so it can be overidden for testing. 41 var simplestreamsToolsPublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- 42 Version: GnuPG v1.4.11 (GNU/Linux) 43 44 mQINBFJN1n8BEAC1vt2w08Y4ztJrv3maOycMezBb7iUs6DLH8hOZoqRO9EW9558W 45 8CN6G4sVbC/nIhivvn/paw0gSicfYXGs5teCJL3ShrcsGkhTs+5q7UO2TVGAUPwb 46 CFWCqPkCB/+CiQ/fnEAWV5c11KzMTBtQ2nfJFS8rEQfc2PJMKqd/Y+LDItOc5E5Y 47 SseGT/60coyTZO0iE3mKv1osFjSJlUv/6f/ziHGgV+IowOtEeeaEz8H/oU4vHhyA 48 THL/k9DSNb0I/+aI8R84OB7EqrQ/ck6B6+CTbwGwkQUBK6z/Isl3uq9MhGjsiPjy 49 EfOJNTfa+knlQcedc3/2S/jTUBDxU+myga9gQ2jF4oEzb74LarpV4y1KXpsqyLwd 50 8/vpNG5rTLtjZ3ZTJu7EkAra6pNK/Uxj9guIkCIGIVS1SWtsR0mCY+6TOdfJu7bt 51 qOcSWkp3gaYcnCid8ecZuD8KDcxJscdYBetxCV4TLVV5CwO4MMVkxcI3zL1ORzHS 52 j0W+aYzdtycHu2w8ZQwQRuFB2y5zsxE69MOoS857FzwhRctPSiwIPWH+Qo2BkNAM 53 K5fVc19z9kzgtRP1+rHgBox2w+hOSZiYf0vluaG7NPUsMfVOGBFTxn1W+rb3NL/m 54 hUoDPl2e2zoViEsaT2p+ATwFDN0DlQLLQxsVIbxdL6cfMQASHmADOHA6dwARAQAB 55 tEtKdWp1IFRvb2xzIChDYW5vbmljYWwgSnVqdSBUb29sIEJ1aWxkZXIpIDxqdWp1 56 LXRvb2xzLW5vcmVwbHlAY2Fub25pY2FsLmNvbT6JAjkEEwEKACMFAlJN1n8CGwMH 57 CwkNCAwHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRA3j2KvahV9szBED/wOlDTMpevL 58 bYyh+mFaeNBw/mwCdWqpwQkpIRLwxt0al1eV9KIVhu6CK1g1UMZ24H3gy5Btj5N5 59 ga02xgqfQRrP4Mqv2dYZOL5p8WFuZjbow9a+e89mqqFuW6/os57cFwZ7Z3imbBDa 60 aWzuzdeWLEK7PfT6rpik6ZMIpI1LGywI93abaZX8v6ouwFeQovXcS0HKt906+ElI 61 oWgSh8dL2hqZ71SR/74sehkEZSYfQRLa7RJCDvA/iInXeGRuyaheQ1iTrY606aBh 62 +NyOgr4cG+7Sy3FIbqgBx0hxkY8LZv4L7l2IDDjgbTEGILpQ2tkykDnFY7QgEdE4 63 5TzPONg9zyk91NRHqjLIm9CFt8P3rcs+MBjaxv+S45RIHQEu+ewkr6BihnPPldkN 64 eSIi4Z0OTTQfAI0oDkREVFnnOHfzZ8uafHXOnhUYsovZ3YrowoiNXOWRxeOvt5cL 65 XE0Gyq7n8ESe9JOCg3AZcrDX12xWX+gaSgDaD66fI5xr+A3128BLpYQTMXOpe1n9 66 rfsiA8XBEFsB6+xMJBtSSPUsaWjes/aziI87fBv7FpEMagnWLqJ7xk2E2RR06B9t 67 F+SoiLF3aQ0ZJFqKpDDYBO5kZkHIql0jVkuPEz5fxTOZjZE4irTZiSMdJ6xsm9AU 68 axxW8e4pax116l4D2toMJPvXkA9lCZ3RIrkCDQRSTdZ/ARAA7SonLFZQrrLD93Jp 69 GpgJnYha6rr3pdIm9wH5PnV9Ysgyt/aM9RVrMXzSjMRpxdV6qxK7Lbzh/V9QxpoI 70 YvFIi4Yu5k0wDPSm/sowBtVI/X2WMSSvd3DUaigTFBQ1giIY3R46wqcY99RfUPJ1 71 VsHFZ0mZq5GuAPSv/Ky7r9SByMDtQk+Pt8jiOIiJ8eGgKy/W0Wau8ImNqSUyj+67 72 QeOCpEKTjS2gQypi6vgCtUCDfy4yHPxppARary/GDjVIAvwjdu/+0rshWcWUOwq8 73 ex2ddPYQf9dGmF9CesaFknpVnkXb9pbw+qBF/CSdk6Z/ApgtXFGwWszP5/Wqq2Pd 74 ilM1C80WcZVhuwk+acYztk5P5hGw0XL2nDeNg08hcDy2NEL/hA9PM2DSFpoWy1aA 75 Gjt/8ICPY3SNJlfJUhMIBOK0nmHIoHGU/tX7AiuwEKyP8Qh5kp8fYoO4c59WfeKq 76 e6rbttt7IEywAlY6HiLMymqC/d0nPk0Cy5bujacH2y3ahAgCwNVvo+E77J7m7Ui2 77 vqzvpcW6Fla2EzbXus4nIgqEV/qX6fQXqItptKZFvZeznj0epRswkmFm7KLXD5p1 78 SzkmfAujy5xQJktZKvtTKRROnX5JdBB8RT83MIJr+U4FOT3UPQYc2V1O2k4PYF9G 79 g5YZtNPTvdx8dvN7qwiO7R7xenkAEQEAAYkCHwQYAQoACQUCUk3WfwIbDAAKCRA3 80 j2KvahV9s4+SD/sEKOBs6YE2dhax0y/wx1AKJbkneVhxTjgCggY/rbnLm6w85xQl 81 EgGycmdRq4JkBDhmzsevx+THNJicBwN9qP12Z14kM1pr7WWw9fOmshPQx5kJXYs+ 82 FiK6f5vHXcNiTyvC8oOGquGrDoB7SACgTr+Lkm/dNfpRn0XsApUy6vQSqChAzqkJ 83 qYZCIIbHTea1DIoNhVI+VTaJ1Z5IqMM9mi43RVYeq7yyBNLwhdjEIOX9qBK4Secn 84 mFz94SCz+b5titGyFiBAJzPBP/NSwM6DP2OfRhsBC6K4xDELn8Dpucb9FHqaLG75 85 K3oDhTEUfTBiG3PRfc57974+V3KrkK71rMzWpQJ2IyMtxzl8qO4JYhLRSL0kMq8/ 86 hYlXGcNwyUUtiDPOwvG44KDVgXbrnFTVqLU6nc9k/yPD1pfommaTAWrb2tTitkGf 87 zOxHnpWTP48l+6qzfEM1PUKvx3U04BZe8JCaU+JVdy6O/rLjEVjYq/vBY6EGOxa2 88 C4Vs43YdFOXSa38ze0J4nFRGO8gOBP/EJyE8Nwqg7i+6VvkD+H2KbZVUXiWld+v/ 89 vwtaXhWd7JS+v38YZ4CijEBe69VYHpSNIz87uhVKgdkFBhoOGtf9/NEO7NYwk7/N 90 qsH+JQgcphKkC+JH0Dw7Q/0e16LClkPPa21NseVGUWzS0WmS+0egtDDutg== 91 =hQAI 92 -----END PGP PUBLIC KEY BLOCK----- 93 ` 94 95 // This needs to be a var so we can override it for testing. 96 var DefaultBaseURL = "https://streams.canonical.com/juju/tools" 97 98 // ToolsConstraint defines criteria used to find a tools metadata record. 99 type ToolsConstraint struct { 100 simplestreams.LookupParams 101 Version version.Number 102 MajorVersion int 103 MinorVersion int 104 Released bool 105 } 106 107 // NewVersionedToolsConstraint returns a ToolsConstraint for a tools with a specific version. 108 func NewVersionedToolsConstraint(vers version.Number, params simplestreams.LookupParams) *ToolsConstraint { 109 return &ToolsConstraint{LookupParams: params, Version: vers} 110 } 111 112 // NewGeneralToolsConstraint returns a ToolsConstraint for tools with matching major/minor version numbers. 113 func NewGeneralToolsConstraint(majorVersion, minorVersion int, released bool, params simplestreams.LookupParams) *ToolsConstraint { 114 return &ToolsConstraint{LookupParams: params, Version: version.Zero, 115 MajorVersion: majorVersion, MinorVersion: minorVersion, Released: released} 116 } 117 118 // Ids generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN). 119 func (tc *ToolsConstraint) Ids() ([]string, error) { 120 var allIds []string 121 for _, series := range tc.Series { 122 version, err := ubuntu.SeriesVersion(series) 123 if err != nil { 124 return nil, err 125 } 126 ids := make([]string, len(tc.Arches)) 127 for i, arch := range tc.Arches { 128 ids[i] = fmt.Sprintf("com.ubuntu.juju:%s:%s", version, arch) 129 } 130 allIds = append(allIds, ids...) 131 } 132 return allIds, nil 133 } 134 135 // ToolsMetadata holds information about a particular tools tarball. 136 type ToolsMetadata struct { 137 Release string `json:"release"` 138 Version string `json:"version"` 139 Arch string `json:"arch"` 140 Size int64 `json:"size"` 141 Path string `json:"path"` 142 FullPath string `json:"-"` 143 FileType string `json:"ftype"` 144 SHA256 string `json:"sha256"` 145 } 146 147 func (t *ToolsMetadata) String() string { 148 return fmt.Sprintf("%+v", *t) 149 } 150 151 // binary returns the tools metadata's binary version, 152 // which may be used for map lookup. 153 func (t *ToolsMetadata) binary() version.Binary { 154 return version.Binary{ 155 Number: version.MustParse(t.Version), 156 Series: t.Release, 157 Arch: t.Arch, 158 } 159 } 160 161 func (t *ToolsMetadata) productId() (string, error) { 162 seriesVersion, err := ubuntu.SeriesVersion(t.Release) 163 if err != nil { 164 return "", err 165 } 166 return fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, t.Arch), nil 167 } 168 169 // Fetch returns a list of tools for the specified cloud matching the constraint. 170 // The base URL locations are as specified - the first location which has a file is the one used. 171 // Signed data is preferred, but if there is no signed data available and onlySigned is false, 172 // then unsigned data is used. 173 func Fetch( 174 sources []simplestreams.DataSource, indexPath string, cons *ToolsConstraint, 175 onlySigned bool) ([]*ToolsMetadata, *simplestreams.ResolveInfo, error) { 176 177 params := simplestreams.ValueParams{ 178 DataType: ContentDownload, 179 FilterFunc: appendMatchingTools, 180 MirrorContentId: ToolsContentId, 181 ValueTemplate: ToolsMetadata{}, 182 PublicKey: simplestreamsToolsPublicKey, 183 } 184 items, resolveInfo, err := simplestreams.GetMetadata(sources, indexPath, cons, onlySigned, params) 185 if err != nil { 186 return nil, nil, err 187 } 188 metadata := make([]*ToolsMetadata, len(items)) 189 for i, md := range items { 190 metadata[i] = md.(*ToolsMetadata) 191 } 192 // Sorting the metadata is not strictly necessary, but it ensures consistent ordering for 193 // all compilers, and it just makes it easier to look at the data. 194 Sort(metadata) 195 return metadata, resolveInfo, nil 196 } 197 198 // Sort sorts a slice of ToolsMetadata in ascending order of their version 199 // in order to ensure the results of Fetch are ordered deterministically. 200 func Sort(metadata []*ToolsMetadata) { 201 sort.Sort(byVersion(metadata)) 202 } 203 204 type byVersion []*ToolsMetadata 205 206 func (b byVersion) Len() int { return len(b) } 207 func (b byVersion) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 208 func (b byVersion) Less(i, j int) bool { return b[i].binary().String() < b[j].binary().String() } 209 210 // appendMatchingTools updates matchingTools with tools metadata records from tools which belong to the 211 // specified series. If a tools record already exists in matchingTools, it is not overwritten. 212 func appendMatchingTools(source simplestreams.DataSource, matchingTools []interface{}, 213 tools map[string]interface{}, cons simplestreams.LookupConstraint) []interface{} { 214 215 toolsMap := make(map[version.Binary]*ToolsMetadata, len(matchingTools)) 216 for _, val := range matchingTools { 217 tm := val.(*ToolsMetadata) 218 toolsMap[tm.binary()] = tm 219 } 220 for _, val := range tools { 221 tm := val.(*ToolsMetadata) 222 if !set.NewStrings(cons.Params().Series...).Contains(tm.Release) { 223 continue 224 } 225 if toolsConstraint, ok := cons.(*ToolsConstraint); ok { 226 tmNumber := version.MustParse(tm.Version) 227 if toolsConstraint.Version == version.Zero { 228 if toolsConstraint.Released && tmNumber.IsDev() { 229 continue 230 } 231 if toolsConstraint.MajorVersion >= 0 && toolsConstraint.MajorVersion != tmNumber.Major { 232 continue 233 } 234 if toolsConstraint.MinorVersion >= 0 && toolsConstraint.MinorVersion != tmNumber.Minor { 235 continue 236 } 237 } else { 238 if toolsConstraint.Version != tmNumber { 239 continue 240 } 241 } 242 } 243 if _, ok := toolsMap[tm.binary()]; !ok { 244 tm.FullPath, _ = source.URL(tm.Path) 245 matchingTools = append(matchingTools, tm) 246 } 247 } 248 return matchingTools 249 } 250 251 type MetadataFile struct { 252 Path string 253 Data []byte 254 } 255 256 // MetadataFromTools returns a tools metadata list derived from the 257 // given tools list. The size and sha256 will not be computed if 258 // missing. 259 func MetadataFromTools(toolsList coretools.List) []*ToolsMetadata { 260 metadata := make([]*ToolsMetadata, len(toolsList)) 261 for i, t := range toolsList { 262 path := fmt.Sprintf("releases/juju-%s-%s-%s.tgz", t.Version.Number, t.Version.Series, t.Version.Arch) 263 metadata[i] = &ToolsMetadata{ 264 Release: t.Version.Series, 265 Version: t.Version.Number.String(), 266 Arch: t.Version.Arch, 267 Path: path, 268 FileType: "tar.gz", 269 Size: t.Size, 270 SHA256: t.SHA256, 271 } 272 } 273 return metadata 274 } 275 276 // ResolveMetadata resolves incomplete metadata 277 // by fetching the tools from storage and computing 278 // the size and hash locally. 279 func ResolveMetadata(stor storage.StorageReader, metadata []*ToolsMetadata) error { 280 for _, md := range metadata { 281 if md.Size != 0 { 282 continue 283 } 284 binary := md.binary() 285 logger.Infof("Fetching tools to generate hash: %v", binary) 286 size, sha256hash, err := fetchToolsHash(stor, binary) 287 if err != nil { 288 return err 289 } 290 md.Size = size 291 md.SHA256 = fmt.Sprintf("%x", sha256hash.Sum(nil)) 292 } 293 return nil 294 } 295 296 // MergeMetadata merges the given tools metadata. 297 // If metadata for the same tools version exists in both lists, 298 // an entry with non-empty size/SHA256 takes precedence; if 299 // the two entries have different sizes/hashes, then an error is 300 // returned. 301 func MergeMetadata(tmlist1, tmlist2 []*ToolsMetadata) ([]*ToolsMetadata, error) { 302 merged := make(map[version.Binary]*ToolsMetadata) 303 for _, tm := range tmlist1 { 304 merged[tm.binary()] = tm 305 } 306 for _, tm := range tmlist2 { 307 binary := tm.binary() 308 if existing, ok := merged[binary]; ok { 309 if tm.Size != 0 { 310 if existing.Size == 0 { 311 merged[binary] = tm 312 } else if existing.Size != tm.Size || existing.SHA256 != tm.SHA256 { 313 return nil, fmt.Errorf( 314 "metadata mismatch for %s: sizes=(%v,%v) sha256=(%v,%v)", 315 binary.String(), 316 existing.Size, tm.Size, 317 existing.SHA256, tm.SHA256, 318 ) 319 } 320 } 321 } else { 322 merged[binary] = tm 323 } 324 } 325 list := make([]*ToolsMetadata, 0, len(merged)) 326 for _, metadata := range merged { 327 list = append(list, metadata) 328 } 329 Sort(list) 330 return list, nil 331 } 332 333 // ReadMetadata returns the tools metadata from the given storage. 334 func ReadMetadata(store storage.StorageReader) ([]*ToolsMetadata, error) { 335 dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", store, storage.BaseToolsPath) 336 toolsConstraint, err := makeToolsConstraint(simplestreams.CloudSpec{}, -1, -1, coretools.Filter{}) 337 if err != nil { 338 return nil, err 339 } 340 metadata, _, err := Fetch( 341 []simplestreams.DataSource{dataSource}, simplestreams.DefaultIndexPath, toolsConstraint, false) 342 if err != nil && !errors.IsNotFound(err) { 343 return nil, err 344 } 345 return metadata, nil 346 } 347 348 var PublicMirrorsInfo = `{ 349 "mirrors": { 350 "com.ubuntu.juju:released:tools": [ 351 { 352 "datatype": "content-download", 353 "path": "streams/v1/cpc-mirrors.json", 354 "updated": "{{updated}}", 355 "format": "mirrors:1.0" 356 } 357 ] 358 } 359 } 360 ` 361 362 // WriteMetadata writes the given tools metadata to the given storage. 363 func WriteMetadata(stor storage.Storage, metadata []*ToolsMetadata, writeMirrors ShouldWriteMirrors) error { 364 updated := time.Now() 365 index, products, err := MarshalToolsMetadataJSON(metadata, updated) 366 if err != nil { 367 return err 368 } 369 metadataInfo := []MetadataFile{ 370 {simplestreams.UnsignedIndex, index}, 371 {ProductMetadataPath, products}, 372 } 373 if writeMirrors { 374 mirrorsUpdated := updated.Format("20060102") // YYYYMMDD 375 mirrorsInfo := strings.Replace(PublicMirrorsInfo, "{{updated}}", mirrorsUpdated, -1) 376 metadataInfo = append(metadataInfo, MetadataFile{simplestreams.UnsignedMirror, []byte(mirrorsInfo)}) 377 } 378 for _, md := range metadataInfo { 379 logger.Infof("Writing %s", "tools/"+md.Path) 380 err = stor.Put(path.Join(storage.BaseToolsPath, md.Path), bytes.NewReader(md.Data), int64(len(md.Data))) 381 if err != nil { 382 return err 383 } 384 } 385 return nil 386 } 387 388 type ShouldWriteMirrors bool 389 390 const ( 391 WriteMirrors = ShouldWriteMirrors(true) 392 DoNotWriteMirrors = ShouldWriteMirrors(false) 393 ) 394 395 // MergeAndWriteMetadata reads the existing metadata from storage (if any), 396 // and merges it with metadata generated from the given tools list. The 397 // resulting metadata is written to storage. 398 func MergeAndWriteMetadata(stor storage.Storage, tools coretools.List, writeMirrors ShouldWriteMirrors) error { 399 existing, err := ReadMetadata(stor) 400 if err != nil { 401 return err 402 } 403 metadata := MetadataFromTools(tools) 404 if metadata, err = MergeMetadata(metadata, existing); err != nil { 405 return err 406 } 407 return WriteMetadata(stor, metadata, writeMirrors) 408 } 409 410 // fetchToolsHash fetches the tools from storage and calculates 411 // its size in bytes and computes a SHA256 hash of its contents. 412 func fetchToolsHash(stor storage.StorageReader, ver version.Binary) (size int64, sha256hash hash.Hash, err error) { 413 r, err := storage.Get(stor, StorageName(ver)) 414 if err != nil { 415 return 0, nil, err 416 } 417 defer r.Close() 418 sha256hash = sha256.New() 419 size, err = io.Copy(sha256hash, r) 420 return size, sha256hash, err 421 }