launchpad.net/~rogpeppe/juju-core/500-errgo-fix@v0.0.0-20140213181702-000000002356/environs/tools/simplestreams.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 // The tools package supports locating, parsing, and filtering Ubuntu tools metadata in simplestreams format. 5 // See http://launchpad.net/simplestreams and in particular the doc/README file in that project for more information 6 // about the file formats. 7 package tools 8 9 import ( 10 "bytes" 11 "crypto/sha256" 12 "fmt" 13 "hash" 14 "io" 15 "path" 16 "strings" 17 "time" 18 19 "launchpad.net/juju-core/environs/simplestreams" 20 "launchpad.net/juju-core/environs/storage" 21 "launchpad.net/juju-core/errors" 22 coretools "launchpad.net/juju-core/tools" 23 "launchpad.net/juju-core/utils/set" 24 "launchpad.net/juju-core/version" 25 ) 26 27 func init() { 28 simplestreams.RegisterStructTags(ToolsMetadata{}) 29 } 30 31 const ( 32 ContentDownload = "content-download" 33 ) 34 35 // simplestreamsToolsPublicKey is the public key required to 36 // authenticate the simple streams data on http://streams.canonical.com. 37 // Declared as a var so it can be overidden for testing. 38 var simplestreamsToolsPublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- 39 Version: GnuPG v1.4.11 (GNU/Linux) 40 41 mQINBFJN1n8BEAC1vt2w08Y4ztJrv3maOycMezBb7iUs6DLH8hOZoqRO9EW9558W 42 8CN6G4sVbC/nIhivvn/paw0gSicfYXGs5teCJL3ShrcsGkhTs+5q7UO2TVGAUPwb 43 CFWCqPkCB/+CiQ/fnEAWV5c11KzMTBtQ2nfJFS8rEQfc2PJMKqd/Y+LDItOc5E5Y 44 SseGT/60coyTZO0iE3mKv1osFjSJlUv/6f/ziHGgV+IowOtEeeaEz8H/oU4vHhyA 45 THL/k9DSNb0I/+aI8R84OB7EqrQ/ck6B6+CTbwGwkQUBK6z/Isl3uq9MhGjsiPjy 46 EfOJNTfa+knlQcedc3/2S/jTUBDxU+myga9gQ2jF4oEzb74LarpV4y1KXpsqyLwd 47 8/vpNG5rTLtjZ3ZTJu7EkAra6pNK/Uxj9guIkCIGIVS1SWtsR0mCY+6TOdfJu7bt 48 qOcSWkp3gaYcnCid8ecZuD8KDcxJscdYBetxCV4TLVV5CwO4MMVkxcI3zL1ORzHS 49 j0W+aYzdtycHu2w8ZQwQRuFB2y5zsxE69MOoS857FzwhRctPSiwIPWH+Qo2BkNAM 50 K5fVc19z9kzgtRP1+rHgBox2w+hOSZiYf0vluaG7NPUsMfVOGBFTxn1W+rb3NL/m 51 hUoDPl2e2zoViEsaT2p+ATwFDN0DlQLLQxsVIbxdL6cfMQASHmADOHA6dwARAQAB 52 tEtKdWp1IFRvb2xzIChDYW5vbmljYWwgSnVqdSBUb29sIEJ1aWxkZXIpIDxqdWp1 53 LXRvb2xzLW5vcmVwbHlAY2Fub25pY2FsLmNvbT6JAjkEEwEKACMFAlJN1n8CGwMH 54 CwkNCAwHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRA3j2KvahV9szBED/wOlDTMpevL 55 bYyh+mFaeNBw/mwCdWqpwQkpIRLwxt0al1eV9KIVhu6CK1g1UMZ24H3gy5Btj5N5 56 ga02xgqfQRrP4Mqv2dYZOL5p8WFuZjbow9a+e89mqqFuW6/os57cFwZ7Z3imbBDa 57 aWzuzdeWLEK7PfT6rpik6ZMIpI1LGywI93abaZX8v6ouwFeQovXcS0HKt906+ElI 58 oWgSh8dL2hqZ71SR/74sehkEZSYfQRLa7RJCDvA/iInXeGRuyaheQ1iTrY606aBh 59 +NyOgr4cG+7Sy3FIbqgBx0hxkY8LZv4L7l2IDDjgbTEGILpQ2tkykDnFY7QgEdE4 60 5TzPONg9zyk91NRHqjLIm9CFt8P3rcs+MBjaxv+S45RIHQEu+ewkr6BihnPPldkN 61 eSIi4Z0OTTQfAI0oDkREVFnnOHfzZ8uafHXOnhUYsovZ3YrowoiNXOWRxeOvt5cL 62 XE0Gyq7n8ESe9JOCg3AZcrDX12xWX+gaSgDaD66fI5xr+A3128BLpYQTMXOpe1n9 63 rfsiA8XBEFsB6+xMJBtSSPUsaWjes/aziI87fBv7FpEMagnWLqJ7xk2E2RR06B9t 64 F+SoiLF3aQ0ZJFqKpDDYBO5kZkHIql0jVkuPEz5fxTOZjZE4irTZiSMdJ6xsm9AU 65 axxW8e4pax116l4D2toMJPvXkA9lCZ3RIrkCDQRSTdZ/ARAA7SonLFZQrrLD93Jp 66 GpgJnYha6rr3pdIm9wH5PnV9Ysgyt/aM9RVrMXzSjMRpxdV6qxK7Lbzh/V9QxpoI 67 YvFIi4Yu5k0wDPSm/sowBtVI/X2WMSSvd3DUaigTFBQ1giIY3R46wqcY99RfUPJ1 68 VsHFZ0mZq5GuAPSv/Ky7r9SByMDtQk+Pt8jiOIiJ8eGgKy/W0Wau8ImNqSUyj+67 69 QeOCpEKTjS2gQypi6vgCtUCDfy4yHPxppARary/GDjVIAvwjdu/+0rshWcWUOwq8 70 ex2ddPYQf9dGmF9CesaFknpVnkXb9pbw+qBF/CSdk6Z/ApgtXFGwWszP5/Wqq2Pd 71 ilM1C80WcZVhuwk+acYztk5P5hGw0XL2nDeNg08hcDy2NEL/hA9PM2DSFpoWy1aA 72 Gjt/8ICPY3SNJlfJUhMIBOK0nmHIoHGU/tX7AiuwEKyP8Qh5kp8fYoO4c59WfeKq 73 e6rbttt7IEywAlY6HiLMymqC/d0nPk0Cy5bujacH2y3ahAgCwNVvo+E77J7m7Ui2 74 vqzvpcW6Fla2EzbXus4nIgqEV/qX6fQXqItptKZFvZeznj0epRswkmFm7KLXD5p1 75 SzkmfAujy5xQJktZKvtTKRROnX5JdBB8RT83MIJr+U4FOT3UPQYc2V1O2k4PYF9G 76 g5YZtNPTvdx8dvN7qwiO7R7xenkAEQEAAYkCHwQYAQoACQUCUk3WfwIbDAAKCRA3 77 j2KvahV9s4+SD/sEKOBs6YE2dhax0y/wx1AKJbkneVhxTjgCggY/rbnLm6w85xQl 78 EgGycmdRq4JkBDhmzsevx+THNJicBwN9qP12Z14kM1pr7WWw9fOmshPQx5kJXYs+ 79 FiK6f5vHXcNiTyvC8oOGquGrDoB7SACgTr+Lkm/dNfpRn0XsApUy6vQSqChAzqkJ 80 qYZCIIbHTea1DIoNhVI+VTaJ1Z5IqMM9mi43RVYeq7yyBNLwhdjEIOX9qBK4Secn 81 mFz94SCz+b5titGyFiBAJzPBP/NSwM6DP2OfRhsBC6K4xDELn8Dpucb9FHqaLG75 82 K3oDhTEUfTBiG3PRfc57974+V3KrkK71rMzWpQJ2IyMtxzl8qO4JYhLRSL0kMq8/ 83 hYlXGcNwyUUtiDPOwvG44KDVgXbrnFTVqLU6nc9k/yPD1pfommaTAWrb2tTitkGf 84 zOxHnpWTP48l+6qzfEM1PUKvx3U04BZe8JCaU+JVdy6O/rLjEVjYq/vBY6EGOxa2 85 C4Vs43YdFOXSa38ze0J4nFRGO8gOBP/EJyE8Nwqg7i+6VvkD+H2KbZVUXiWld+v/ 86 vwtaXhWd7JS+v38YZ4CijEBe69VYHpSNIz87uhVKgdkFBhoOGtf9/NEO7NYwk7/N 87 qsH+JQgcphKkC+JH0Dw7Q/0e16LClkPPa21NseVGUWzS0WmS+0egtDDutg== 88 =hQAI 89 -----END PGP PUBLIC KEY BLOCK----- 90 ` 91 92 // This needs to be a var so we can override it for testing. 93 var DefaultBaseURL = "https://streams.canonical.com/juju/tools" 94 95 // ToolsConstraint defines criteria used to find a tools metadata record. 96 type ToolsConstraint struct { 97 simplestreams.LookupParams 98 Version version.Number 99 MajorVersion int 100 MinorVersion int 101 Released bool 102 } 103 104 // NewVersionedToolsConstraint returns a ToolsConstraint for a tools with a specific version. 105 func NewVersionedToolsConstraint(vers string, params simplestreams.LookupParams) *ToolsConstraint { 106 versNum := version.MustParse(vers) 107 return &ToolsConstraint{LookupParams: params, Version: versNum} 108 } 109 110 // NewGeneralToolsConstraint returns a ToolsConstraint for tools with matching major/minor version numbers. 111 func NewGeneralToolsConstraint(majorVersion, minorVersion int, released bool, params simplestreams.LookupParams) *ToolsConstraint { 112 return &ToolsConstraint{LookupParams: params, Version: version.Zero, 113 MajorVersion: majorVersion, MinorVersion: minorVersion, Released: released} 114 } 115 116 // Ids generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN). 117 func (tc *ToolsConstraint) Ids() ([]string, error) { 118 var allIds []string 119 for _, series := range tc.Series { 120 version, err := simplestreams.SeriesVersion(series) 121 if err != nil { 122 return nil, err 123 } 124 ids := make([]string, len(tc.Arches)) 125 for i, arch := range tc.Arches { 126 ids[i] = fmt.Sprintf("com.ubuntu.juju:%s:%s", version, arch) 127 } 128 allIds = append(allIds, ids...) 129 } 130 return allIds, nil 131 } 132 133 // ToolsMetadata holds information about a particular tools tarball. 134 type ToolsMetadata struct { 135 Release string `json:"release"` 136 Version string `json:"version"` 137 Arch string `json:"arch"` 138 Size int64 `json:"size"` 139 Path string `json:"path"` 140 FullPath string `json:"-"` 141 FileType string `json:"ftype"` 142 SHA256 string `json:"sha256"` 143 } 144 145 func (t *ToolsMetadata) String() string { 146 return fmt.Sprintf("%+v", *t) 147 } 148 149 // binary returns the tools metadata's binary version, 150 // which may be used for map lookup. 151 func (t *ToolsMetadata) binary() version.Binary { 152 return version.Binary{ 153 Number: version.MustParse(t.Version), 154 Series: t.Release, 155 Arch: t.Arch, 156 } 157 } 158 159 func (t *ToolsMetadata) productId() (string, error) { 160 seriesVersion, err := simplestreams.SeriesVersion(t.Release) 161 if err != nil { 162 return "", err 163 } 164 return fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, t.Arch), nil 165 } 166 167 // Fetch returns a list of tools for the specified cloud matching the constraint. 168 // The base URL locations are as specified - the first location which has a file is the one used. 169 // Signed data is preferred, but if there is no signed data available and onlySigned is false, 170 // then unsigned data is used. 171 func Fetch(sources []simplestreams.DataSource, indexPath string, cons *ToolsConstraint, onlySigned bool) ([]*ToolsMetadata, error) { 172 params := simplestreams.ValueParams{ 173 DataType: ContentDownload, 174 FilterFunc: appendMatchingTools, 175 MirrorContentId: ToolsContentId, 176 ValueTemplate: ToolsMetadata{}, 177 PublicKey: simplestreamsToolsPublicKey, 178 } 179 items, err := simplestreams.GetMetadata(sources, indexPath, cons, onlySigned, params) 180 if err != nil { 181 return nil, err 182 } 183 metadata := make([]*ToolsMetadata, len(items)) 184 for i, md := range items { 185 metadata[i] = md.(*ToolsMetadata) 186 } 187 return metadata, nil 188 } 189 190 // appendMatchingTools updates matchingTools with tools metadata records from tools which belong to the 191 // specified series. If a tools record already exists in matchingTools, it is not overwritten. 192 func appendMatchingTools(source simplestreams.DataSource, matchingTools []interface{}, 193 tools map[string]interface{}, cons simplestreams.LookupConstraint) []interface{} { 194 195 toolsMap := make(map[version.Binary]*ToolsMetadata, len(matchingTools)) 196 for _, val := range matchingTools { 197 tm := val.(*ToolsMetadata) 198 toolsMap[tm.binary()] = tm 199 } 200 for _, val := range tools { 201 tm := val.(*ToolsMetadata) 202 if !set.NewStrings(cons.Params().Series...).Contains(tm.Release) { 203 continue 204 } 205 if toolsConstraint, ok := cons.(*ToolsConstraint); ok { 206 tmNumber := version.MustParse(tm.Version) 207 if toolsConstraint.Version == version.Zero { 208 if toolsConstraint.Released && tmNumber.IsDev() { 209 continue 210 } 211 if toolsConstraint.MajorVersion >= 0 && toolsConstraint.MajorVersion != tmNumber.Major { 212 continue 213 } 214 if toolsConstraint.MinorVersion >= 0 && toolsConstraint.MinorVersion != tmNumber.Minor { 215 continue 216 } 217 } else { 218 if toolsConstraint.Version != tmNumber { 219 continue 220 } 221 } 222 } 223 if _, ok := toolsMap[tm.binary()]; !ok { 224 tm.FullPath, _ = source.URL(tm.Path) 225 matchingTools = append(matchingTools, tm) 226 } 227 } 228 return matchingTools 229 } 230 231 type MetadataFile struct { 232 Path string 233 Data []byte 234 } 235 236 // MetadataFromTools returns a tools metadata list derived from the 237 // given tools list. The size and sha256 will not be computed if 238 // missing. 239 func MetadataFromTools(toolsList coretools.List) []*ToolsMetadata { 240 metadata := make([]*ToolsMetadata, len(toolsList)) 241 for i, t := range toolsList { 242 path := fmt.Sprintf("releases/juju-%s-%s-%s.tgz", t.Version.Number, t.Version.Series, t.Version.Arch) 243 metadata[i] = &ToolsMetadata{ 244 Release: t.Version.Series, 245 Version: t.Version.Number.String(), 246 Arch: t.Version.Arch, 247 Path: path, 248 FileType: "tar.gz", 249 Size: t.Size, 250 SHA256: t.SHA256, 251 } 252 } 253 return metadata 254 } 255 256 // ResolveMetadata resolves incomplete metadata 257 // by fetching the tools from storage and computing 258 // the size and hash locally. 259 func ResolveMetadata(stor storage.StorageReader, metadata []*ToolsMetadata) error { 260 for _, md := range metadata { 261 if md.Size != 0 { 262 continue 263 } 264 binary := md.binary() 265 logger.Infof("Fetching tools to generate hash: %v", binary) 266 size, sha256hash, err := fetchToolsHash(stor, binary) 267 if err != nil { 268 return err 269 } 270 md.Size = size 271 md.SHA256 = fmt.Sprintf("%x", sha256hash.Sum(nil)) 272 } 273 return nil 274 } 275 276 // MergeMetadata merges the given tools metadata. 277 // If metadata for the same tools version exists in both lists, 278 // an entry with non-empty size/SHA256 takes precedence; if 279 // the two entries have different sizes/hashes, then an error is 280 // returned. 281 func MergeMetadata(tmlist1, tmlist2 []*ToolsMetadata) ([]*ToolsMetadata, error) { 282 merged := make(map[version.Binary]*ToolsMetadata) 283 for _, tm := range tmlist1 { 284 merged[tm.binary()] = tm 285 } 286 for _, tm := range tmlist2 { 287 binary := tm.binary() 288 if existing, ok := merged[binary]; ok { 289 if tm.Size != 0 { 290 if existing.Size == 0 { 291 merged[binary] = tm 292 } else if existing.Size != tm.Size || existing.SHA256 != tm.SHA256 { 293 return nil, fmt.Errorf( 294 "metadata mismatch for %s: sizes=(%v,%v) sha256=(%v,%v)", 295 binary.String(), 296 existing.Size, tm.Size, 297 existing.SHA256, tm.SHA256, 298 ) 299 } 300 } 301 } else { 302 merged[binary] = tm 303 } 304 } 305 list := make([]*ToolsMetadata, 0, len(merged)) 306 for _, metadata := range merged { 307 list = append(list, metadata) 308 } 309 return list, nil 310 } 311 312 // ReadMetadata returns the tools metadata from the given storage. 313 func ReadMetadata(store storage.StorageReader) ([]*ToolsMetadata, error) { 314 dataSource := storage.NewStorageSimpleStreamsDataSource(store, storage.BaseToolsPath) 315 toolsConstraint, err := makeToolsConstraint(simplestreams.CloudSpec{}, -1, -1, coretools.Filter{}) 316 if err != nil { 317 return nil, err 318 } 319 metadata, err := Fetch([]simplestreams.DataSource{dataSource}, simplestreams.DefaultIndexPath, toolsConstraint, false) 320 if err != nil && !errors.IsNotFoundError(err) { 321 return nil, err 322 } 323 return metadata, nil 324 } 325 326 var PublicMirrorsInfo = `{ 327 "mirrors": { 328 "com.ubuntu.juju:released:tools": [ 329 { 330 "datatype": "content-download", 331 "path": "streams/v1/cpc-mirrors.json", 332 "updated": "{{updated}}", 333 "format": "mirrors:1.0" 334 } 335 ] 336 } 337 } 338 ` 339 340 // WriteMetadata writes the given tools metadata to the given storage. 341 func WriteMetadata(stor storage.Storage, metadata []*ToolsMetadata, writeMirrors ShouldWriteMirrors) error { 342 updated := time.Now() 343 index, products, err := MarshalToolsMetadataJSON(metadata, updated) 344 if err != nil { 345 return err 346 } 347 metadataInfo := []MetadataFile{ 348 {simplestreams.UnsignedIndex, index}, 349 {ProductMetadataPath, products}, 350 } 351 if writeMirrors { 352 mirrorsUpdated := updated.Format("20060102") // YYYYMMDD 353 mirrorsInfo := strings.Replace(PublicMirrorsInfo, "{{updated}}", mirrorsUpdated, -1) 354 metadataInfo = append(metadataInfo, MetadataFile{simplestreams.UnsignedMirror, []byte(mirrorsInfo)}) 355 } 356 for _, md := range metadataInfo { 357 logger.Infof("Writing %s", "tools/"+md.Path) 358 err = stor.Put(path.Join(storage.BaseToolsPath, md.Path), bytes.NewReader(md.Data), int64(len(md.Data))) 359 if err != nil { 360 return err 361 } 362 } 363 return nil 364 } 365 366 type ShouldWriteMirrors bool 367 368 const ( 369 WriteMirrors = ShouldWriteMirrors(true) 370 DoNotWriteMirrors = ShouldWriteMirrors(false) 371 ) 372 373 // MergeAndWriteMetadata reads the existing metadata from storage (if any), 374 // and merges it with metadata generated from the given tools list. The 375 // resulting metadata is written to storage. 376 func MergeAndWriteMetadata(stor storage.Storage, tools coretools.List, writeMirrors ShouldWriteMirrors) error { 377 existing, err := ReadMetadata(stor) 378 if err != nil { 379 return err 380 } 381 metadata := MetadataFromTools(tools) 382 if metadata, err = MergeMetadata(metadata, existing); err != nil { 383 return err 384 } 385 return WriteMetadata(stor, metadata, writeMirrors) 386 } 387 388 // fetchToolsHash fetches the tools from storage and calculates 389 // its size in bytes and computes a SHA256 hash of its contents. 390 func fetchToolsHash(stor storage.StorageReader, ver version.Binary) (size int64, sha256hash hash.Hash, err error) { 391 r, err := storage.Get(stor, StorageName(ver)) 392 if err != nil { 393 return 0, nil, err 394 } 395 defer r.Close() 396 sha256hash = sha256.New() 397 size, err = io.Copy(sha256hash, r) 398 return size, sha256hash, err 399 }