github.com/mhilton/juju-juju@v0.0.0-20150901100907-a94dd2c73455/environs/tools/simplestreams.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 // The tools package supports locating, parsing, and filtering Ubuntu tools metadata in simplestreams format. 5 // See http://launchpad.net/simplestreams and in particular the doc/README file in that project for more information 6 // about the file formats. 7 package tools 8 9 import ( 10 "bytes" 11 "crypto/sha256" 12 "encoding/json" 13 "fmt" 14 "hash" 15 "io" 16 "io/ioutil" 17 "path" 18 "sort" 19 "strings" 20 "time" 21 22 "github.com/juju/errors" 23 "github.com/juju/utils/set" 24 25 "github.com/juju/juju/environs/simplestreams" 26 "github.com/juju/juju/environs/storage" 27 "github.com/juju/juju/juju/arch" 28 coretools "github.com/juju/juju/tools" 29 "github.com/juju/juju/version" 30 ) 31 32 func init() { 33 simplestreams.RegisterStructTags(ToolsMetadata{}) 34 } 35 36 const ( 37 // ImageIds is the simplestreams tools content type. 38 ContentDownload = "content-download" 39 40 // StreamsVersionV1 is used to construct the path for accessing streams data. 41 StreamsVersionV1 = "v1" 42 43 // IndexFileVersion is used to construct the streams index file. 44 IndexFileVersion = 2 45 ) 46 47 var currentStreamsVersion = StreamsVersionV1 48 49 // simplestreamsToolsPublicKey is the public key required to 50 // authenticate the simple streams data on http://streams.canonical.com. 51 // Declared as a var so it can be overidden for testing. 52 var simplestreamsToolsPublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- 53 Version: GnuPG v1.4.11 (GNU/Linux) 54 55 mQINBFJN1n8BEAC1vt2w08Y4ztJrv3maOycMezBb7iUs6DLH8hOZoqRO9EW9558W 56 8CN6G4sVbC/nIhivvn/paw0gSicfYXGs5teCJL3ShrcsGkhTs+5q7UO2TVGAUPwb 57 CFWCqPkCB/+CiQ/fnEAWV5c11KzMTBtQ2nfJFS8rEQfc2PJMKqd/Y+LDItOc5E5Y 58 SseGT/60coyTZO0iE3mKv1osFjSJlUv/6f/ziHGgV+IowOtEeeaEz8H/oU4vHhyA 59 THL/k9DSNb0I/+aI8R84OB7EqrQ/ck6B6+CTbwGwkQUBK6z/Isl3uq9MhGjsiPjy 60 EfOJNTfa+knlQcedc3/2S/jTUBDxU+myga9gQ2jF4oEzb74LarpV4y1KXpsqyLwd 61 8/vpNG5rTLtjZ3ZTJu7EkAra6pNK/Uxj9guIkCIGIVS1SWtsR0mCY+6TOdfJu7bt 62 qOcSWkp3gaYcnCid8ecZuD8KDcxJscdYBetxCV4TLVV5CwO4MMVkxcI3zL1ORzHS 63 j0W+aYzdtycHu2w8ZQwQRuFB2y5zsxE69MOoS857FzwhRctPSiwIPWH+Qo2BkNAM 64 K5fVc19z9kzgtRP1+rHgBox2w+hOSZiYf0vluaG7NPUsMfVOGBFTxn1W+rb3NL/m 65 hUoDPl2e2zoViEsaT2p+ATwFDN0DlQLLQxsVIbxdL6cfMQASHmADOHA6dwARAQAB 66 tEtKdWp1IFRvb2xzIChDYW5vbmljYWwgSnVqdSBUb29sIEJ1aWxkZXIpIDxqdWp1 67 LXRvb2xzLW5vcmVwbHlAY2Fub25pY2FsLmNvbT6JAjkEEwEKACMFAlJN1n8CGwMH 68 CwkNCAwHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRA3j2KvahV9szBED/wOlDTMpevL 69 bYyh+mFaeNBw/mwCdWqpwQkpIRLwxt0al1eV9KIVhu6CK1g1UMZ24H3gy5Btj5N5 70 ga02xgqfQRrP4Mqv2dYZOL5p8WFuZjbow9a+e89mqqFuW6/os57cFwZ7Z3imbBDa 71 aWzuzdeWLEK7PfT6rpik6ZMIpI1LGywI93abaZX8v6ouwFeQovXcS0HKt906+ElI 72 oWgSh8dL2hqZ71SR/74sehkEZSYfQRLa7RJCDvA/iInXeGRuyaheQ1iTrY606aBh 73 +NyOgr4cG+7Sy3FIbqgBx0hxkY8LZv4L7l2IDDjgbTEGILpQ2tkykDnFY7QgEdE4 74 5TzPONg9zyk91NRHqjLIm9CFt8P3rcs+MBjaxv+S45RIHQEu+ewkr6BihnPPldkN 75 eSIi4Z0OTTQfAI0oDkREVFnnOHfzZ8uafHXOnhUYsovZ3YrowoiNXOWRxeOvt5cL 76 XE0Gyq7n8ESe9JOCg3AZcrDX12xWX+gaSgDaD66fI5xr+A3128BLpYQTMXOpe1n9 77 rfsiA8XBEFsB6+xMJBtSSPUsaWjes/aziI87fBv7FpEMagnWLqJ7xk2E2RR06B9t 78 F+SoiLF3aQ0ZJFqKpDDYBO5kZkHIql0jVkuPEz5fxTOZjZE4irTZiSMdJ6xsm9AU 79 axxW8e4pax116l4D2toMJPvXkA9lCZ3RIrkCDQRSTdZ/ARAA7SonLFZQrrLD93Jp 80 GpgJnYha6rr3pdIm9wH5PnV9Ysgyt/aM9RVrMXzSjMRpxdV6qxK7Lbzh/V9QxpoI 81 YvFIi4Yu5k0wDPSm/sowBtVI/X2WMSSvd3DUaigTFBQ1giIY3R46wqcY99RfUPJ1 82 VsHFZ0mZq5GuAPSv/Ky7r9SByMDtQk+Pt8jiOIiJ8eGgKy/W0Wau8ImNqSUyj+67 83 QeOCpEKTjS2gQypi6vgCtUCDfy4yHPxppARary/GDjVIAvwjdu/+0rshWcWUOwq8 84 ex2ddPYQf9dGmF9CesaFknpVnkXb9pbw+qBF/CSdk6Z/ApgtXFGwWszP5/Wqq2Pd 85 ilM1C80WcZVhuwk+acYztk5P5hGw0XL2nDeNg08hcDy2NEL/hA9PM2DSFpoWy1aA 86 Gjt/8ICPY3SNJlfJUhMIBOK0nmHIoHGU/tX7AiuwEKyP8Qh5kp8fYoO4c59WfeKq 87 e6rbttt7IEywAlY6HiLMymqC/d0nPk0Cy5bujacH2y3ahAgCwNVvo+E77J7m7Ui2 88 vqzvpcW6Fla2EzbXus4nIgqEV/qX6fQXqItptKZFvZeznj0epRswkmFm7KLXD5p1 89 SzkmfAujy5xQJktZKvtTKRROnX5JdBB8RT83MIJr+U4FOT3UPQYc2V1O2k4PYF9G 90 g5YZtNPTvdx8dvN7qwiO7R7xenkAEQEAAYkCHwQYAQoACQUCUk3WfwIbDAAKCRA3 91 j2KvahV9s4+SD/sEKOBs6YE2dhax0y/wx1AKJbkneVhxTjgCggY/rbnLm6w85xQl 92 EgGycmdRq4JkBDhmzsevx+THNJicBwN9qP12Z14kM1pr7WWw9fOmshPQx5kJXYs+ 93 FiK6f5vHXcNiTyvC8oOGquGrDoB7SACgTr+Lkm/dNfpRn0XsApUy6vQSqChAzqkJ 94 qYZCIIbHTea1DIoNhVI+VTaJ1Z5IqMM9mi43RVYeq7yyBNLwhdjEIOX9qBK4Secn 95 mFz94SCz+b5titGyFiBAJzPBP/NSwM6DP2OfRhsBC6K4xDELn8Dpucb9FHqaLG75 96 K3oDhTEUfTBiG3PRfc57974+V3KrkK71rMzWpQJ2IyMtxzl8qO4JYhLRSL0kMq8/ 97 hYlXGcNwyUUtiDPOwvG44KDVgXbrnFTVqLU6nc9k/yPD1pfommaTAWrb2tTitkGf 98 zOxHnpWTP48l+6qzfEM1PUKvx3U04BZe8JCaU+JVdy6O/rLjEVjYq/vBY6EGOxa2 99 C4Vs43YdFOXSa38ze0J4nFRGO8gOBP/EJyE8Nwqg7i+6VvkD+H2KbZVUXiWld+v/ 100 vwtaXhWd7JS+v38YZ4CijEBe69VYHpSNIz87uhVKgdkFBhoOGtf9/NEO7NYwk7/N 101 qsH+JQgcphKkC+JH0Dw7Q/0e16LClkPPa21NseVGUWzS0WmS+0egtDDutg== 102 =hQAI 103 -----END PGP PUBLIC KEY BLOCK----- 104 ` 105 106 // This needs to be a var so we can override it for testing. 107 var DefaultBaseURL = "https://streams.canonical.com/juju/tools" 108 109 const ( 110 // Legacy release directory for Juju < 1.21. 111 LegacyReleaseDirectory = "releases" 112 113 // Used to specify the released tools metadata. 114 ReleasedStream = "released" 115 116 // Used to specify metadata for testing tools. 117 TestingStream = "testing" 118 119 // Used to specify the proposed tools metadata. 120 ProposedStream = "proposed" 121 122 // Used to specify the devel tools metadata. 123 DevelStream = "devel" 124 ) 125 126 // ToolsConstraint defines criteria used to find a tools metadata record. 127 type ToolsConstraint struct { 128 simplestreams.LookupParams 129 Version version.Number 130 MajorVersion int 131 MinorVersion int 132 } 133 134 // NewVersionedToolsConstraint returns a ToolsConstraint for a tools with a specific version. 135 func NewVersionedToolsConstraint(vers version.Number, params simplestreams.LookupParams) *ToolsConstraint { 136 return &ToolsConstraint{LookupParams: params, Version: vers} 137 } 138 139 // NewGeneralToolsConstraint returns a ToolsConstraint for tools with matching major/minor version numbers. 140 func NewGeneralToolsConstraint(majorVersion, minorVersion int, params simplestreams.LookupParams) *ToolsConstraint { 141 return &ToolsConstraint{LookupParams: params, Version: version.Zero, 142 MajorVersion: majorVersion, MinorVersion: minorVersion} 143 } 144 145 // IndexIds generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN). 146 func (tc *ToolsConstraint) IndexIds() []string { 147 if tc.Stream == "" { 148 return nil 149 } 150 return []string{ToolsContentId(tc.Stream)} 151 } 152 153 // ProductIds generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN). 154 func (tc *ToolsConstraint) ProductIds() ([]string, error) { 155 var allIds []string 156 for _, series := range tc.Series { 157 version, err := version.SeriesVersion(series) 158 if err != nil { 159 return nil, err 160 } 161 ids := make([]string, len(tc.Arches)) 162 for i, arch := range tc.Arches { 163 ids[i] = fmt.Sprintf("com.ubuntu.juju:%s:%s", version, arch) 164 } 165 allIds = append(allIds, ids...) 166 } 167 return allIds, nil 168 } 169 170 // ToolsMetadata holds information about a particular tools tarball. 171 type ToolsMetadata struct { 172 Release string `json:"release"` 173 Version string `json:"version"` 174 Arch string `json:"arch"` 175 Size int64 `json:"size"` 176 Path string `json:"path"` 177 FullPath string `json:"-"` 178 FileType string `json:"ftype"` 179 SHA256 string `json:"sha256"` 180 } 181 182 func (t *ToolsMetadata) String() string { 183 return fmt.Sprintf("%+v", *t) 184 } 185 186 // sortString is used by byVersion to sort a list of ToolsMetadata. 187 func (t *ToolsMetadata) sortString() string { 188 return fmt.Sprintf("%v-%s-%s", t.Version, t.Release, t.Arch) 189 } 190 191 // binary returns the tools metadata's binary version, which may be used for 192 // map lookup. It is possible for a binary to have an unkown OS. 193 func (t *ToolsMetadata) binary() (version.Binary, error) { 194 num, err := version.Parse(t.Version) 195 if err != nil { 196 return version.Binary{}, errors.Trace(err) 197 } 198 toolsOS, err := version.GetOSFromSeries(t.Release) 199 if err != nil && !version.IsUnknownOSForSeriesError(err) { 200 return version.Binary{}, errors.Trace(err) 201 } 202 return version.Binary{ 203 Number: num, 204 Series: t.Release, 205 Arch: t.Arch, 206 OS: toolsOS, 207 }, nil 208 } 209 210 func (t *ToolsMetadata) productId() (string, error) { 211 seriesVersion, err := version.SeriesVersion(t.Release) 212 if err != nil { 213 return "", err 214 } 215 return fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, t.Arch), nil 216 } 217 218 // Fetch returns a list of tools for the specified cloud matching the constraint. 219 // The base URL locations are as specified - the first location which has a file is the one used. 220 // Signed data is preferred, but if there is no signed data available and onlySigned is false, 221 // then unsigned data is used. 222 func Fetch( 223 sources []simplestreams.DataSource, cons *ToolsConstraint, 224 onlySigned bool) ([]*ToolsMetadata, *simplestreams.ResolveInfo, error) { 225 226 params := simplestreams.GetMetadataParams{ 227 StreamsVersion: currentStreamsVersion, 228 OnlySigned: onlySigned, 229 LookupConstraint: cons, 230 ValueParams: simplestreams.ValueParams{ 231 DataType: ContentDownload, 232 FilterFunc: appendMatchingTools, 233 MirrorContentId: ToolsContentId(cons.Stream), 234 ValueTemplate: ToolsMetadata{}, 235 PublicKey: simplestreamsToolsPublicKey, 236 }, 237 } 238 items, resolveInfo, err := simplestreams.GetMetadata(sources, params) 239 if err != nil { 240 return nil, nil, err 241 } 242 metadata := make([]*ToolsMetadata, len(items)) 243 for i, md := range items { 244 metadata[i] = md.(*ToolsMetadata) 245 } 246 // Sorting the metadata is not strictly necessary, but it ensures consistent ordering for 247 // all compilers, and it just makes it easier to look at the data. 248 Sort(metadata) 249 return metadata, resolveInfo, nil 250 } 251 252 // Sort sorts a slice of ToolsMetadata in ascending order of their version 253 // in order to ensure the results of Fetch are ordered deterministically. 254 func Sort(metadata []*ToolsMetadata) { 255 sort.Sort(byVersion(metadata)) 256 } 257 258 type byVersion []*ToolsMetadata 259 260 func (b byVersion) Len() int { return len(b) } 261 func (b byVersion) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 262 func (b byVersion) Less(i, j int) bool { return b[i].sortString() < b[j].sortString() } 263 264 // appendMatchingTools updates matchingTools with tools metadata records from tools which belong to the 265 // specified series. If a tools record already exists in matchingTools, it is not overwritten. 266 func appendMatchingTools(source simplestreams.DataSource, matchingTools []interface{}, 267 tools map[string]interface{}, cons simplestreams.LookupConstraint) ([]interface{}, error) { 268 269 toolsMap := make(map[version.Binary]*ToolsMetadata, len(matchingTools)) 270 for _, val := range matchingTools { 271 tm := val.(*ToolsMetadata) 272 binary, err := tm.binary() 273 if err != nil { 274 return nil, errors.Trace(err) 275 } 276 toolsMap[binary] = tm 277 } 278 for _, val := range tools { 279 tm := val.(*ToolsMetadata) 280 if !set.NewStrings(cons.Params().Series...).Contains(tm.Release) { 281 continue 282 } 283 if toolsConstraint, ok := cons.(*ToolsConstraint); ok { 284 tmNumber := version.MustParse(tm.Version) 285 if toolsConstraint.Version == version.Zero { 286 if toolsConstraint.MajorVersion >= 0 && toolsConstraint.MajorVersion != tmNumber.Major { 287 continue 288 } 289 if toolsConstraint.MinorVersion >= 0 && toolsConstraint.MinorVersion != tmNumber.Minor { 290 continue 291 } 292 } else { 293 if toolsConstraint.Version != tmNumber { 294 continue 295 } 296 } 297 } 298 binary, err := tm.binary() 299 if err != nil { 300 return nil, errors.Trace(err) 301 } 302 if _, ok := toolsMap[binary]; !ok { 303 tm.FullPath, _ = source.URL(tm.Path) 304 matchingTools = append(matchingTools, tm) 305 } 306 } 307 return matchingTools, nil 308 } 309 310 type MetadataFile struct { 311 Path string 312 Data []byte 313 } 314 315 // MetadataFromTools returns a tools metadata list derived from the 316 // given tools list. The size and sha256 will not be computed if 317 // missing. 318 func MetadataFromTools(toolsList coretools.List, toolsDir string) []*ToolsMetadata { 319 metadata := make([]*ToolsMetadata, len(toolsList)) 320 for i, t := range toolsList { 321 path := fmt.Sprintf("%s/juju-%s-%s-%s.tgz", toolsDir, t.Version.Number, t.Version.Series, t.Version.Arch) 322 metadata[i] = &ToolsMetadata{ 323 Release: t.Version.Series, 324 Version: t.Version.Number.String(), 325 Arch: t.Version.Arch, 326 Path: path, 327 FileType: "tar.gz", 328 Size: t.Size, 329 SHA256: t.SHA256, 330 } 331 } 332 return metadata 333 } 334 335 // ResolveMetadata resolves incomplete metadata 336 // by fetching the tools from storage and computing 337 // the size and hash locally. 338 func ResolveMetadata(stor storage.StorageReader, toolsDir string, metadata []*ToolsMetadata) error { 339 for _, md := range metadata { 340 if md.Size != 0 { 341 continue 342 } 343 binary, err := md.binary() 344 if err != nil { 345 return errors.Annotate(err, "cannot resolve metadata") 346 } 347 logger.Infof("Fetching tools from dir %q to generate hash: %v", toolsDir, binary) 348 size, sha256hash, err := fetchToolsHash(stor, toolsDir, binary) 349 // Older versions of Juju only know about ppc64, not ppc64el, 350 // so if there's no metadata for ppc64, dd metadata for that arch. 351 if errors.IsNotFound(err) && binary.Arch == arch.LEGACY_PPC64 { 352 ppc64elBinary := binary 353 ppc64elBinary.Arch = arch.PPC64EL 354 md.Path = strings.Replace(md.Path, binary.Arch, ppc64elBinary.Arch, -1) 355 size, sha256hash, err = fetchToolsHash(stor, toolsDir, ppc64elBinary) 356 } 357 if err != nil { 358 return err 359 } 360 md.Size = size 361 md.SHA256 = fmt.Sprintf("%x", sha256hash.Sum(nil)) 362 } 363 return nil 364 } 365 366 // MergeMetadata merges the given tools metadata. 367 // If metadata for the same tools version exists in both lists, 368 // an entry with non-empty size/SHA256 takes precedence; if 369 // the two entries have different sizes/hashes, then an error is 370 // returned. 371 func MergeMetadata(tmlist1, tmlist2 []*ToolsMetadata) ([]*ToolsMetadata, error) { 372 merged := make(map[version.Binary]*ToolsMetadata) 373 for _, tm := range tmlist1 { 374 binary, err := tm.binary() 375 if err != nil { 376 return nil, errors.Annotate(err, "cannot merge metadata") 377 } 378 merged[binary] = tm 379 } 380 for _, tm := range tmlist2 { 381 binary, err := tm.binary() 382 if err != nil { 383 return nil, errors.Annotate(err, "cannot merge metadata") 384 } 385 if existing, ok := merged[binary]; ok { 386 if tm.Size != 0 { 387 if existing.Size == 0 { 388 merged[binary] = tm 389 } else if existing.Size != tm.Size || existing.SHA256 != tm.SHA256 { 390 return nil, fmt.Errorf( 391 "metadata mismatch for %s: sizes=(%v,%v) sha256=(%v,%v)", 392 binary.String(), 393 existing.Size, tm.Size, 394 existing.SHA256, tm.SHA256, 395 ) 396 } 397 } 398 } else { 399 merged[binary] = tm 400 } 401 } 402 list := make([]*ToolsMetadata, 0, len(merged)) 403 for _, metadata := range merged { 404 list = append(list, metadata) 405 } 406 Sort(list) 407 return list, nil 408 } 409 410 // ReadMetadata returns the tools metadata from the given storage for the specified stream. 411 func ReadMetadata(store storage.StorageReader, stream string) ([]*ToolsMetadata, error) { 412 dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", store, storage.BaseToolsPath) 413 toolsConstraint, err := makeToolsConstraint(simplestreams.CloudSpec{}, stream, -1, -1, coretools.Filter{}) 414 if err != nil { 415 return nil, err 416 } 417 metadata, _, err := Fetch( 418 []simplestreams.DataSource{dataSource}, toolsConstraint, false) 419 if err != nil && !errors.IsNotFound(err) { 420 return nil, err 421 } 422 return metadata, nil 423 } 424 425 // AllMetadataStreams is the set of streams for which there will be simplestreams tools metadata. 426 var AllMetadataStreams = []string{ReleasedStream, ProposedStream, TestingStream, DevelStream} 427 428 // ReadAllMetadata returns the tools metadata from the given storage for all streams. 429 // The result is a map of metadata slices, keyed on stream. 430 func ReadAllMetadata(store storage.StorageReader) (map[string][]*ToolsMetadata, error) { 431 streamMetadata := make(map[string][]*ToolsMetadata) 432 for _, stream := range AllMetadataStreams { 433 metadata, err := ReadMetadata(store, stream) 434 if err != nil { 435 return nil, err 436 } 437 if len(metadata) == 0 { 438 continue 439 } 440 streamMetadata[stream] = metadata 441 } 442 return streamMetadata, nil 443 } 444 445 // removeMetadataUpdated unmarshalls simplestreams metadata, clears the 446 // updated attribute, and then marshalls back to a string. 447 func removeMetadataUpdated(metadataBytes []byte) (string, error) { 448 var metadata map[string]interface{} 449 err := json.Unmarshal(metadataBytes, &metadata) 450 if err != nil { 451 return "", err 452 } 453 delete(metadata, "updated") 454 455 metadataJson, err := json.Marshal(metadata) 456 if err != nil { 457 return "", err 458 } 459 return string(metadataJson), nil 460 } 461 462 // metadataUnchanged returns true if the content of metadata for stream in stor is the same 463 // as generatedMetadata, ignoring the "updated" attribute. 464 func metadataUnchanged(stor storage.Storage, stream string, generatedMetadata []byte) (bool, error) { 465 mdPath := ProductMetadataPath(stream) 466 filePath := path.Join(storage.BaseToolsPath, mdPath) 467 existingDataReader, err := stor.Get(filePath) 468 // If the file can't be retrieved, consider it has changed. 469 if err != nil { 470 return false, nil 471 } 472 defer existingDataReader.Close() 473 existingData, err := ioutil.ReadAll(existingDataReader) 474 if err != nil { 475 return false, err 476 } 477 478 // To do the comparison, we unmarshall the metadata, clear the 479 // updated value, and marshall back to a string. 480 existingMetadata, err := removeMetadataUpdated(existingData) 481 if err != nil { 482 return false, err 483 } 484 newMetadata, err := removeMetadataUpdated(generatedMetadata) 485 if err != nil { 486 return false, err 487 } 488 return existingMetadata == newMetadata, nil 489 } 490 491 // WriteMetadata writes the given tools metadata for the specified streams to the given storage. 492 // streamMetadata contains all known metadata so that the correct index files can be written. 493 // Only product files for the specified streams are written. 494 func WriteMetadata(stor storage.Storage, streamMetadata map[string][]*ToolsMetadata, streams []string, writeMirrors ShouldWriteMirrors) error { 495 updated := time.Now() 496 index, legacyIndex, products, err := MarshalToolsMetadataJSON(streamMetadata, updated) 497 if err != nil { 498 return err 499 } 500 metadataInfo := []MetadataFile{ 501 {simplestreams.UnsignedIndex(currentStreamsVersion, IndexFileVersion), index}, 502 } 503 if legacyIndex != nil { 504 metadataInfo = append(metadataInfo, MetadataFile{ 505 simplestreams.UnsignedIndex(currentStreamsVersion, 1), legacyIndex, 506 }) 507 } 508 for _, stream := range streams { 509 if metadata, ok := products[stream]; ok { 510 // If metadata hasn't changed, do not overwrite. 511 unchanged, err := metadataUnchanged(stor, stream, metadata) 512 if err != nil { 513 return err 514 } 515 if unchanged { 516 logger.Infof("Metadata for stream %q unchanged", stream) 517 continue 518 } 519 // Metadata is different, so include it. 520 metadataInfo = append(metadataInfo, MetadataFile{ProductMetadataPath(stream), metadata}) 521 } 522 } 523 if writeMirrors { 524 streamsMirrorsMetadata := make(map[string][]simplestreams.MirrorReference) 525 for stream := range streamMetadata { 526 streamsMirrorsMetadata[ToolsContentId(stream)] = []simplestreams.MirrorReference{{ 527 Updated: updated.Format("20060102"), // YYYYMMDD 528 DataType: ContentDownload, 529 Format: simplestreams.MirrorFormat, 530 Path: simplestreams.MirrorFile, 531 }} 532 } 533 mirrorsMetadata := map[string]map[string][]simplestreams.MirrorReference{ 534 "mirrors": streamsMirrorsMetadata, 535 } 536 mirrorsInfo, err := json.MarshalIndent(&mirrorsMetadata, "", " ") 537 if err != nil { 538 return err 539 } 540 metadataInfo = append( 541 metadataInfo, MetadataFile{simplestreams.UnsignedMirror(currentStreamsVersion), mirrorsInfo}) 542 } 543 return writeMetadataFiles(stor, metadataInfo) 544 } 545 546 var writeMetadataFiles = func(stor storage.Storage, metadataInfo []MetadataFile) error { 547 for _, md := range metadataInfo { 548 filePath := path.Join(storage.BaseToolsPath, md.Path) 549 logger.Infof("Writing %s", filePath) 550 err := stor.Put(filePath, bytes.NewReader(md.Data), int64(len(md.Data))) 551 if err != nil { 552 return err 553 } 554 } 555 return nil 556 } 557 558 type ShouldWriteMirrors bool 559 560 const ( 561 WriteMirrors = ShouldWriteMirrors(true) 562 DoNotWriteMirrors = ShouldWriteMirrors(false) 563 ) 564 565 // MergeAndWriteMetadata reads the existing metadata from storage (if any), 566 // and merges it with metadata generated from the given tools list. The 567 // resulting metadata is written to storage. 568 func MergeAndWriteMetadata(stor storage.Storage, toolsDir, stream string, tools coretools.List, writeMirrors ShouldWriteMirrors) error { 569 existing, err := ReadAllMetadata(stor) 570 if err != nil { 571 return err 572 } 573 metadata := MetadataFromTools(tools, toolsDir) 574 if metadata, err = MergeMetadata(metadata, existing[stream]); err != nil { 575 return err 576 } 577 existing[stream] = metadata 578 return WriteMetadata(stor, existing, []string{stream}, writeMirrors) 579 } 580 581 // fetchToolsHash fetches the tools from storage and calculates 582 // its size in bytes and computes a SHA256 hash of its contents. 583 func fetchToolsHash(stor storage.StorageReader, stream string, ver version.Binary) (size int64, sha256hash hash.Hash, err error) { 584 r, err := storage.Get(stor, StorageName(ver, stream)) 585 if err != nil { 586 return 0, nil, err 587 } 588 defer r.Close() 589 sha256hash = sha256.New() 590 size, err = io.Copy(sha256hash, r) 591 return size, sha256hash, err 592 }