github.com/wallyworld/juju@v0.0.0-20161013125918-6cf1bc9d917a/environs/tools/simplestreams_test.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package tools_test 5 6 import ( 7 "bytes" 8 "encoding/json" 9 "flag" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "net/http" 14 "os" 15 "path/filepath" 16 "reflect" 17 "runtime" 18 "strings" 19 "testing" 20 21 jc "github.com/juju/testing/checkers" 22 "github.com/juju/utils" 23 "github.com/juju/utils/series" 24 "github.com/juju/version" 25 "gopkg.in/amz.v3/aws" 26 gc "gopkg.in/check.v1" 27 28 "github.com/juju/juju/environs/filestorage" 29 "github.com/juju/juju/environs/simplestreams" 30 sstesting "github.com/juju/juju/environs/simplestreams/testing" 31 "github.com/juju/juju/environs/storage" 32 "github.com/juju/juju/environs/tools" 33 toolstesting "github.com/juju/juju/environs/tools/testing" 34 "github.com/juju/juju/juju/keys" 35 coretesting "github.com/juju/juju/testing" 36 coretools "github.com/juju/juju/tools" 37 ) 38 39 var live = flag.Bool("live", false, "Include live simplestreams tests") 40 var vendor = flag.String("vendor", "", "The vendor representing the source of the simplestream data") 41 42 type liveTestData struct { 43 baseURL string 44 requireSigned bool 45 validCloudSpec simplestreams.CloudSpec 46 } 47 48 var liveURLs = map[string]liveTestData{ 49 "ec2": { 50 baseURL: tools.DefaultBaseURL, 51 requireSigned: true, 52 validCloudSpec: simplestreams.CloudSpec{"us-east-1", aws.Regions["us-east-1"].EC2Endpoint}, 53 }, 54 "canonistack": { 55 baseURL: "https://swift.canonistack.canonical.com/v1/AUTH_526ad877f3e3464589dc1145dfeaac60/juju-tools", 56 requireSigned: false, 57 validCloudSpec: simplestreams.CloudSpec{"lcy01", "https://keystone.canonistack.canonical.com:443/v1.0/"}, 58 }, 59 } 60 61 func setupSimpleStreamsTests(t *testing.T) { 62 if *live { 63 if *vendor == "" { 64 t.Fatal("missing vendor") 65 } 66 var ok bool 67 var testData liveTestData 68 if testData, ok = liveURLs[*vendor]; !ok { 69 keys := reflect.ValueOf(liveURLs).MapKeys() 70 t.Fatalf("Unknown vendor %s. Must be one of %s", *vendor, keys) 71 } 72 registerLiveSimpleStreamsTests(testData.baseURL, 73 tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 74 CloudSpec: testData.validCloudSpec, 75 Series: []string{series.HostSeries()}, 76 Arches: []string{"amd64"}, 77 Stream: "released", 78 }), testData.requireSigned) 79 } 80 registerSimpleStreamsTests() 81 } 82 83 func registerSimpleStreamsTests() { 84 gc.Suite(&simplestreamsSuite{ 85 LocalLiveSimplestreamsSuite: sstesting.LocalLiveSimplestreamsSuite{ 86 Source: simplestreams.NewURLDataSource("test", "test:", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false), 87 RequireSigned: false, 88 DataType: tools.ContentDownload, 89 StreamsVersion: tools.CurrentStreamsVersion, 90 ValidConstraint: tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 91 CloudSpec: simplestreams.CloudSpec{ 92 Region: "us-east-1", 93 Endpoint: "https://ec2.us-east-1.amazonaws.com", 94 }, 95 Series: []string{"precise"}, 96 Arches: []string{"amd64", "arm"}, 97 Stream: "released", 98 }), 99 }, 100 }) 101 gc.Suite(&signedSuite{}) 102 } 103 104 func registerLiveSimpleStreamsTests(baseURL string, validToolsConstraint simplestreams.LookupConstraint, requireSigned bool) { 105 gc.Suite(&sstesting.LocalLiveSimplestreamsSuite{ 106 Source: simplestreams.NewURLDataSource("test", baseURL, utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, requireSigned), 107 RequireSigned: requireSigned, 108 DataType: tools.ContentDownload, 109 StreamsVersion: tools.CurrentStreamsVersion, 110 ValidConstraint: validToolsConstraint, 111 }) 112 } 113 114 type simplestreamsSuite struct { 115 sstesting.LocalLiveSimplestreamsSuite 116 sstesting.TestDataSuite 117 } 118 119 func (s *simplestreamsSuite) SetUpSuite(c *gc.C) { 120 s.LocalLiveSimplestreamsSuite.SetUpSuite(c) 121 s.TestDataSuite.SetUpSuite(c) 122 } 123 124 func (s *simplestreamsSuite) TearDownSuite(c *gc.C) { 125 s.TestDataSuite.TearDownSuite(c) 126 s.LocalLiveSimplestreamsSuite.TearDownSuite(c) 127 } 128 129 var fetchTests = []struct { 130 region string 131 series string 132 version string 133 stream string 134 major int 135 minor int 136 arches []string 137 tools []*tools.ToolsMetadata 138 }{{ 139 series: "precise", 140 arches: []string{"amd64", "arm"}, 141 version: "1.13.0", 142 tools: []*tools.ToolsMetadata{ 143 { 144 Release: "precise", 145 Version: "1.13.0", 146 Arch: "amd64", 147 Size: 2973595, 148 Path: "tools/released/20130806/juju-1.13.0-precise-amd64.tgz", 149 FileType: "tar.gz", 150 SHA256: "447aeb6a934a5eaec4f703eda4ef2dde", 151 }, 152 }, 153 }, { 154 series: "raring", 155 arches: []string{"amd64", "arm"}, 156 version: "1.13.0", 157 tools: []*tools.ToolsMetadata{ 158 { 159 Release: "raring", 160 Version: "1.13.0", 161 Arch: "amd64", 162 Size: 2973173, 163 Path: "tools/released/20130806/juju-1.13.0-raring-amd64.tgz", 164 FileType: "tar.gz", 165 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 166 }, 167 }, 168 }, { 169 series: "raring", 170 arches: []string{"amd64", "arm"}, 171 version: "1.11.4", 172 tools: []*tools.ToolsMetadata{ 173 { 174 Release: "raring", 175 Version: "1.11.4", 176 Arch: "arm", 177 Size: 1950327, 178 Path: "tools/released/20130806/juju-1.11.4-raring-arm.tgz", 179 FileType: "tar.gz", 180 SHA256: "6472014e3255e3fe7fbd3550ef3f0a11", 181 }, 182 }, 183 }, { 184 series: "precise", 185 arches: []string{"amd64", "arm"}, 186 major: 2, 187 tools: []*tools.ToolsMetadata{ 188 { 189 Release: "precise", 190 Version: "2.0.1", 191 Arch: "arm", 192 Size: 1951096, 193 Path: "tools/released/20130806/juju-2.0.1-precise-arm.tgz", 194 FileType: "tar.gz", 195 SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", 196 }, 197 }, 198 }, { 199 series: "precise", 200 arches: []string{"amd64", "arm"}, 201 major: 1, 202 minor: 11, 203 tools: []*tools.ToolsMetadata{ 204 { 205 Release: "precise", 206 Version: "1.11.4", 207 Arch: "arm", 208 Size: 1951096, 209 Path: "tools/released/20130806/juju-1.11.4-precise-arm.tgz", 210 FileType: "tar.gz", 211 SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", 212 }, 213 { 214 Release: "precise", 215 Version: "1.11.5", 216 Arch: "arm", 217 Size: 2031281, 218 Path: "tools/released/20130803/juju-1.11.5-precise-arm.tgz", 219 FileType: "tar.gz", 220 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 221 }, 222 }, 223 }, { 224 series: "trusty", 225 arches: []string{"amd64"}, 226 version: "1.16.0", 227 stream: "testing", 228 tools: []*tools.ToolsMetadata{ 229 { 230 Release: "trusty", 231 Version: "1.16.0", 232 Arch: "amd64", 233 Size: 2973512, 234 Path: "tools/testing/20130806/juju-1.16.0-trusty-amd64.tgz", 235 FileType: "tar.gz", 236 SHA256: "447aeb6a934a5eaec4f703eda4ef2dac", 237 }, 238 }, 239 }} 240 241 func (s *simplestreamsSuite) TestFetch(c *gc.C) { 242 for i, t := range fetchTests { 243 c.Logf("test %d", i) 244 if t.stream == "" { 245 t.stream = "released" 246 } 247 var toolsConstraint *tools.ToolsConstraint 248 if t.version == "" { 249 toolsConstraint = tools.NewGeneralToolsConstraint(t.major, t.minor, simplestreams.LookupParams{ 250 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 251 Series: []string{t.series}, 252 Arches: t.arches, 253 Stream: t.stream, 254 }) 255 } else { 256 toolsConstraint = tools.NewVersionedToolsConstraint(version.MustParse(t.version), 257 simplestreams.LookupParams{ 258 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 259 Series: []string{t.series}, 260 Arches: t.arches, 261 Stream: t.stream, 262 }) 263 } 264 // Add invalid datasource and check later that resolveInfo is correct. 265 invalidSource := simplestreams.NewURLDataSource("invalid", "file://invalid", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, s.RequireSigned) 266 tools, resolveInfo, err := tools.Fetch( 267 []simplestreams.DataSource{invalidSource, s.Source}, toolsConstraint) 268 if !c.Check(err, jc.ErrorIsNil) { 269 continue 270 } 271 for _, tm := range t.tools { 272 tm.FullPath, err = s.Source.URL(tm.Path) 273 c.Assert(err, jc.ErrorIsNil) 274 } 275 c.Check(tools, gc.DeepEquals, t.tools) 276 c.Check(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{ 277 Source: "test", 278 Signed: s.RequireSigned, 279 IndexURL: "test:/streams/v1/index.json", 280 MirrorURL: "", 281 }) 282 } 283 } 284 285 func (s *simplestreamsSuite) TestFetchNoMatchingStream(c *gc.C) { 286 toolsConstraint := tools.NewGeneralToolsConstraint(2, -1, simplestreams.LookupParams{ 287 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 288 Series: []string{"precise"}, 289 Arches: []string{}, 290 Stream: "proposed", 291 }) 292 _, _, err := tools.Fetch( 293 []simplestreams.DataSource{s.Source}, toolsConstraint) 294 c.Assert(err, gc.ErrorMatches, `"content-download" data not found`) 295 } 296 297 func (s *simplestreamsSuite) TestFetchWithMirror(c *gc.C) { 298 toolsConstraint := tools.NewGeneralToolsConstraint(1, 13, simplestreams.LookupParams{ 299 CloudSpec: simplestreams.CloudSpec{"us-west-2", "https://ec2.us-west-2.amazonaws.com"}, 300 Series: []string{"precise"}, 301 Arches: []string{"amd64"}, 302 Stream: "released", 303 }) 304 toolsMetadata, resolveInfo, err := tools.Fetch( 305 []simplestreams.DataSource{s.Source}, toolsConstraint) 306 c.Assert(err, jc.ErrorIsNil) 307 c.Assert(len(toolsMetadata), gc.Equals, 1) 308 309 expectedMetadata := &tools.ToolsMetadata{ 310 Release: "precise", 311 Version: "1.13.0", 312 Arch: "amd64", 313 Size: 2973595, 314 Path: "mirrored-path/juju-1.13.0-precise-amd64.tgz", 315 FullPath: "test:/mirrored-path/juju-1.13.0-precise-amd64.tgz", 316 FileType: "tar.gz", 317 SHA256: "447aeb6a934a5eaec4f703eda4ef2dde", 318 } 319 c.Assert(err, jc.ErrorIsNil) 320 c.Assert(toolsMetadata[0], gc.DeepEquals, expectedMetadata) 321 c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{ 322 Source: "test", 323 Signed: s.RequireSigned, 324 IndexURL: "test:/streams/v1/index.json", 325 MirrorURL: "test:/", 326 }) 327 } 328 329 func assertMetadataMatches(c *gc.C, storageDir string, stream string, toolList coretools.List, metadata []*tools.ToolsMetadata) { 330 var expectedMetadata []*tools.ToolsMetadata = make([]*tools.ToolsMetadata, len(toolList)) 331 for i, tool := range toolList { 332 expectedMetadata[i] = &tools.ToolsMetadata{ 333 Release: tool.Version.Series, 334 Version: tool.Version.Number.String(), 335 Arch: tool.Version.Arch, 336 Size: tool.Size, 337 Path: fmt.Sprintf("%s/juju-%s.tgz", stream, tool.Version.String()), 338 FileType: "tar.gz", 339 SHA256: tool.SHA256, 340 } 341 } 342 c.Assert(metadata, gc.DeepEquals, expectedMetadata) 343 } 344 345 func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) { 346 toolsList := coretools.List{ 347 { 348 Version: version.MustParseBinary("1.2.3-precise-amd64"), 349 Size: 123, 350 SHA256: "abcd", 351 }, { 352 Version: version.MustParseBinary("2.0.1-raring-amd64"), 353 Size: 456, 354 SHA256: "xyz", 355 }, 356 } 357 expected := toolsList 358 359 // Add tools with an unknown series. Do not add an entry in the 360 // expected list as these tools should be ignored. 361 vers, err := version.ParseBinary("3.2.1-xuanhuaceratops-amd64") 362 c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError) 363 toolsList = append(toolsList, &coretools.Tools{ 364 Version: vers, 365 Size: 456, 366 SHA256: "wqe", 367 }) 368 369 dir := c.MkDir() 370 writer, err := filestorage.NewFileStorageWriter(dir) 371 c.Assert(err, jc.ErrorIsNil) 372 err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, tools.DoNotWriteMirrors) 373 c.Assert(err, jc.ErrorIsNil) 374 metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", false) 375 assertMetadataMatches(c, dir, "proposed", expected, metadata) 376 } 377 378 func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) { 379 var versionStrings = []string{ 380 "1.2.3-precise-amd64", 381 "2.0.1-raring-amd64", 382 } 383 dir := c.MkDir() 384 toolstesting.MakeTools(c, dir, "proposed", versionStrings) 385 386 toolsList := coretools.List{ 387 { 388 // If sha256/size is already known, do not recalculate 389 Version: version.MustParseBinary("1.2.3-precise-amd64"), 390 Size: 123, 391 SHA256: "abcd", 392 }, { 393 Version: version.MustParseBinary("2.0.1-raring-amd64"), 394 // The URL is not used for generating metadata. 395 URL: "bogus://", 396 }, 397 } 398 writer, err := filestorage.NewFileStorageWriter(dir) 399 c.Assert(err, jc.ErrorIsNil) 400 writeMirrors := tools.DoNotWriteMirrors 401 if withMirrors { 402 writeMirrors = tools.WriteMirrors 403 } 404 err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, writeMirrors) 405 c.Assert(err, jc.ErrorIsNil) 406 metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", withMirrors) 407 assertMetadataMatches(c, dir, "proposed", toolsList, metadata) 408 409 // No release stream generated so there will not be a legacy index file created. 410 _, err = writer.Get("tools/streams/v1/index.json") 411 c.Assert(err, gc.NotNil) 412 } 413 414 func (s *simplestreamsSuite) TestWriteMetadata(c *gc.C) { 415 s.assertWriteMetadata(c, false) 416 } 417 418 func (s *simplestreamsSuite) TestWriteMetadataWithMirrors(c *gc.C) { 419 s.assertWriteMetadata(c, true) 420 } 421 422 func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) { 423 dir := c.MkDir() 424 existingToolsList := coretools.List{ 425 { 426 Version: version.MustParseBinary("1.2.3-precise-amd64"), 427 Size: 123, 428 SHA256: "abc", 429 }, { 430 Version: version.MustParseBinary("2.0.1-raring-amd64"), 431 Size: 456, 432 SHA256: "xyz", 433 }, 434 } 435 writer, err := filestorage.NewFileStorageWriter(dir) 436 c.Assert(err, jc.ErrorIsNil) 437 err = tools.MergeAndWriteMetadata(writer, "testing", "testing", existingToolsList, tools.WriteMirrors) 438 c.Assert(err, jc.ErrorIsNil) 439 newToolsList := coretools.List{ 440 existingToolsList[0], 441 { 442 Version: version.MustParseBinary("2.1.0-raring-amd64"), 443 Size: 789, 444 SHA256: "def", 445 }, 446 } 447 err = tools.MergeAndWriteMetadata(writer, "testing", "testing", newToolsList, tools.WriteMirrors) 448 c.Assert(err, jc.ErrorIsNil) 449 requiredToolsList := append(existingToolsList, newToolsList[1]) 450 metadata := toolstesting.ParseMetadataFromDir(c, dir, "testing", true) 451 assertMetadataMatches(c, dir, "testing", requiredToolsList, metadata) 452 453 err = tools.MergeAndWriteMetadata(writer, "devel", "devel", newToolsList, tools.WriteMirrors) 454 c.Assert(err, jc.ErrorIsNil) 455 metadata = toolstesting.ParseMetadataFromDir(c, dir, "testing", true) 456 assertMetadataMatches(c, dir, "testing", requiredToolsList, metadata) 457 metadata = toolstesting.ParseMetadataFromDir(c, dir, "devel", true) 458 assertMetadataMatches(c, dir, "devel", newToolsList, metadata) 459 } 460 461 type productSpecSuite struct{} 462 463 var _ = gc.Suite(&productSpecSuite{}) 464 465 func (s *productSpecSuite) TestIndexIdNoStream(c *gc.C) { 466 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 467 Series: []string{"precise"}, 468 Arches: []string{"amd64"}, 469 }) 470 ids := toolsConstraint.IndexIds() 471 c.Assert(ids, gc.HasLen, 0) 472 } 473 474 func (s *productSpecSuite) TestIndexId(c *gc.C) { 475 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 476 Series: []string{"precise"}, 477 Arches: []string{"amd64"}, 478 Stream: "proposed", 479 }) 480 ids := toolsConstraint.IndexIds() 481 c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.juju:proposed:tools"}) 482 } 483 484 func (s *productSpecSuite) TestProductId(c *gc.C) { 485 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 486 Series: []string{"precise"}, 487 Arches: []string{"amd64"}, 488 }) 489 ids, err := toolsConstraint.ProductIds() 490 c.Assert(err, jc.ErrorIsNil) 491 c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.juju:12.04:amd64"}) 492 } 493 494 func (s *productSpecSuite) TestIdMultiArch(c *gc.C) { 495 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.11.3"), simplestreams.LookupParams{ 496 Series: []string{"precise"}, 497 Arches: []string{"amd64", "arm"}, 498 }) 499 ids, err := toolsConstraint.ProductIds() 500 c.Assert(err, jc.ErrorIsNil) 501 c.Assert(ids, gc.DeepEquals, []string{ 502 "com.ubuntu.juju:12.04:amd64", 503 "com.ubuntu.juju:12.04:arm"}) 504 } 505 506 func (s *productSpecSuite) TestIdMultiSeries(c *gc.C) { 507 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.11.3"), simplestreams.LookupParams{ 508 Series: []string{"precise", "raring"}, 509 Arches: []string{"amd64"}, 510 Stream: "released", 511 }) 512 ids, err := toolsConstraint.ProductIds() 513 c.Assert(err, jc.ErrorIsNil) 514 c.Assert(ids, gc.DeepEquals, []string{ 515 "com.ubuntu.juju:12.04:amd64", 516 "com.ubuntu.juju:13.04:amd64"}) 517 } 518 519 func (s *productSpecSuite) TestIdIgnoresInvalidSeries(c *gc.C) { 520 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.11.3"), simplestreams.LookupParams{ 521 Series: []string{"precise", "foobar"}, 522 Arches: []string{"amd64"}, 523 Stream: "released", 524 }) 525 ids, err := toolsConstraint.ProductIds() 526 c.Assert(err, jc.ErrorIsNil) 527 c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.juju:12.04:amd64"}) 528 } 529 530 func (s *productSpecSuite) TestIdWithMajorVersionOnly(c *gc.C) { 531 toolsConstraint := tools.NewGeneralToolsConstraint(1, -1, simplestreams.LookupParams{ 532 Series: []string{"precise"}, 533 Arches: []string{"amd64"}, 534 Stream: "released", 535 }) 536 ids, err := toolsConstraint.ProductIds() 537 c.Assert(err, jc.ErrorIsNil) 538 c.Assert(ids, gc.DeepEquals, []string{`com.ubuntu.juju:12.04:amd64`}) 539 } 540 541 func (s *productSpecSuite) TestIdWithMajorMinorVersion(c *gc.C) { 542 toolsConstraint := tools.NewGeneralToolsConstraint(1, 2, simplestreams.LookupParams{ 543 Series: []string{"precise"}, 544 Arches: []string{"amd64"}, 545 Stream: "released", 546 }) 547 ids, err := toolsConstraint.ProductIds() 548 c.Assert(err, jc.ErrorIsNil) 549 c.Assert(ids, gc.DeepEquals, []string{`com.ubuntu.juju:12.04:amd64`}) 550 } 551 552 func (s *productSpecSuite) TestLargeNumber(c *gc.C) { 553 json := `{ 554 "updated": "Fri, 30 Aug 2013 16:12:58 +0800", 555 "format": "products:1.0", 556 "products": { 557 "com.ubuntu.juju:1.10.0:amd64": { 558 "version": "1.10.0", 559 "arch": "amd64", 560 "versions": { 561 "20133008": { 562 "items": { 563 "1.10.0-precise-amd64": { 564 "release": "precise", 565 "version": "1.10.0", 566 "arch": "amd64", 567 "size": 9223372036854775807, 568 "path": "releases/juju-1.10.0-precise-amd64.tgz", 569 "ftype": "tar.gz", 570 "sha256": "" 571 } 572 } 573 } 574 } 575 } 576 } 577 }` 578 cloudMetadata, err := simplestreams.ParseCloudMetadata([]byte(json), "products:1.0", "", tools.ToolsMetadata{}) 579 c.Assert(err, jc.ErrorIsNil) 580 c.Assert(cloudMetadata.Products, gc.HasLen, 1) 581 product := cloudMetadata.Products["com.ubuntu.juju:1.10.0:amd64"] 582 c.Assert(product, gc.NotNil) 583 c.Assert(product.Items, gc.HasLen, 1) 584 version := product.Items["20133008"] 585 c.Assert(version, gc.NotNil) 586 c.Assert(version.Items, gc.HasLen, 1) 587 item := version.Items["1.10.0-precise-amd64"] 588 c.Assert(item, gc.NotNil) 589 c.Assert(item, gc.FitsTypeOf, &tools.ToolsMetadata{}) 590 c.Assert(item.(*tools.ToolsMetadata).Size, gc.Equals, int64(9223372036854775807)) 591 } 592 593 type metadataHelperSuite struct { 594 coretesting.BaseSuite 595 } 596 597 var _ = gc.Suite(&metadataHelperSuite{}) 598 599 func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) { 600 metadata := tools.MetadataFromTools(nil, "proposed") 601 c.Assert(metadata, gc.HasLen, 0) 602 603 toolsList := coretools.List{{ 604 Version: version.MustParseBinary("1.2.3-precise-amd64"), 605 Size: 123, 606 SHA256: "abc", 607 }, { 608 Version: version.MustParseBinary("2.0.1-raring-amd64"), 609 URL: "file:///tmp/proposed/juju-2.0.1-raring-amd64.tgz", 610 Size: 456, 611 SHA256: "xyz", 612 }} 613 metadata = tools.MetadataFromTools(toolsList, "proposed") 614 c.Assert(metadata, gc.HasLen, len(toolsList)) 615 for i, t := range toolsList { 616 md := metadata[i] 617 c.Assert(md.Release, gc.Equals, t.Version.Series) 618 c.Assert(md.Version, gc.Equals, t.Version.Number.String()) 619 c.Assert(md.Arch, gc.Equals, t.Version.Arch) 620 // FullPath is only filled out when reading tools using simplestreams. 621 // It's not needed elsewhere and requires a URL() call. 622 c.Assert(md.FullPath, gc.Equals, "") 623 c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version, "proposed")[len("tools/"):]) 624 c.Assert(md.FileType, gc.Equals, "tar.gz") 625 c.Assert(md.Size, gc.Equals, t.Size) 626 c.Assert(md.SHA256, gc.Equals, t.SHA256) 627 } 628 } 629 630 type countingStorage struct { 631 storage.StorageReader 632 counter int 633 } 634 635 func (c *countingStorage) Get(name string) (io.ReadCloser, error) { 636 c.counter++ 637 return c.StorageReader.Get(name) 638 } 639 640 func (*metadataHelperSuite) TestResolveMetadata(c *gc.C) { 641 var versionStrings = []string{"1.2.3-precise-amd64"} 642 dir := c.MkDir() 643 toolstesting.MakeTools(c, dir, "released", versionStrings) 644 toolsList := coretools.List{{ 645 Version: version.MustParseBinary(versionStrings[0]), 646 Size: 123, 647 SHA256: "abc", 648 }} 649 650 stor, err := filestorage.NewFileStorageReader(dir) 651 c.Assert(err, jc.ErrorIsNil) 652 err = tools.ResolveMetadata(stor, "released", nil) 653 c.Assert(err, jc.ErrorIsNil) 654 655 // We already have size/sha256, so ensure that storage isn't consulted. 656 countingStorage := &countingStorage{StorageReader: stor} 657 metadata := tools.MetadataFromTools(toolsList, "released") 658 err = tools.ResolveMetadata(countingStorage, "released", metadata) 659 c.Assert(err, jc.ErrorIsNil) 660 c.Assert(countingStorage.counter, gc.Equals, 0) 661 662 // Now clear size/sha256, and check that it is called, and 663 // the size/sha256 sum are updated. 664 metadata[0].Size = 0 665 metadata[0].SHA256 = "" 666 err = tools.ResolveMetadata(countingStorage, "released", metadata) 667 c.Assert(err, jc.ErrorIsNil) 668 c.Assert(countingStorage.counter, gc.Equals, 1) 669 c.Assert(metadata[0].Size, gc.Not(gc.Equals), 0) 670 c.Assert(metadata[0].SHA256, gc.Not(gc.Equals), "") 671 } 672 673 func (*metadataHelperSuite) TestResolveMetadataLegacyPPC64(c *gc.C) { 674 var versionStrings = []string{"1.2.3-precise-amd64", "1.2.3-precise-ppc64el"} 675 dir := c.MkDir() 676 toolstesting.MakeTools(c, dir, "released", versionStrings) 677 678 toolsList := coretools.List{ 679 { 680 Version: version.MustParseBinary(versionStrings[0]), 681 }, { 682 Version: version.MustParseBinary(versionStrings[1]), 683 }, { 684 Version: version.MustParseBinary("1.2.3-precise-ppc64"), 685 }, 686 } 687 toolsMetadata := tools.MetadataFromTools(toolsList, dir) 688 stor, err := filestorage.NewFileStorageReader(dir) 689 c.Assert(err, jc.ErrorIsNil) 690 err = tools.ResolveMetadata(stor, "released", toolsMetadata) 691 c.Assert(err, jc.ErrorIsNil) 692 c.Assert(toolsMetadata, gc.DeepEquals, []*tools.ToolsMetadata{ 693 { 694 Release: "precise", 695 Version: "1.2.3", 696 Arch: "amd64", 697 Size: 19, 698 FileType: "tar.gz", 699 SHA256: "dcdd65b962b804a3d63b108d670290ee95a867a97fe9b9f99b2b77b5c7173e59", 700 Path: fmt.Sprintf("%s/juju-1.2.3-precise-amd64.tgz", dir), 701 }, 702 { 703 Release: "precise", 704 Version: "1.2.3", 705 Arch: "ppc64el", 706 Size: 21, 707 FileType: "tar.gz", 708 SHA256: "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2", 709 Path: fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir), 710 }, 711 { 712 Release: "precise", 713 Version: "1.2.3", 714 Arch: "ppc64", 715 Size: 21, 716 FileType: "tar.gz", 717 SHA256: "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2", 718 Path: fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir), 719 }, 720 }) 721 } 722 723 func (*metadataHelperSuite) TestMergeMetadata(c *gc.C) { 724 md1 := &tools.ToolsMetadata{ 725 Release: "precise", 726 Version: "1.2.3", 727 Arch: "amd64", 728 Path: "path1", 729 } 730 md2 := &tools.ToolsMetadata{ 731 Release: "precise", 732 Version: "1.2.3", 733 Arch: "amd64", 734 Path: "path2", 735 } 736 md3 := &tools.ToolsMetadata{ 737 Release: "raring", 738 Version: "1.2.3", 739 Arch: "amd64", 740 Path: "path3", 741 } 742 743 withSize := func(md *tools.ToolsMetadata, size int64) *tools.ToolsMetadata { 744 clone := *md 745 clone.Size = size 746 return &clone 747 } 748 withSHA256 := func(md *tools.ToolsMetadata, sha256 string) *tools.ToolsMetadata { 749 clone := *md 750 clone.SHA256 = sha256 751 return &clone 752 } 753 754 type mdlist []*tools.ToolsMetadata 755 type test struct { 756 name string 757 lhs, rhs, merged []*tools.ToolsMetadata 758 err string 759 } 760 tests := []test{{ 761 name: "non-empty lhs, empty rhs", 762 lhs: mdlist{md1}, 763 rhs: nil, 764 merged: mdlist{md1}, 765 }, { 766 name: "empty lhs, non-empty rhs", 767 lhs: nil, 768 rhs: mdlist{md2}, 769 merged: mdlist{md2}, 770 }, { 771 name: "identical lhs, rhs", 772 lhs: mdlist{md1}, 773 rhs: mdlist{md1}, 774 merged: mdlist{md1}, 775 }, { 776 name: "same tools in lhs and rhs, neither have size: prefer lhs", 777 lhs: mdlist{md1}, 778 rhs: mdlist{md2}, 779 merged: mdlist{md1}, 780 }, { 781 name: "same tools in lhs and rhs, only lhs has a size: prefer lhs", 782 lhs: mdlist{withSize(md1, 123)}, 783 rhs: mdlist{md2}, 784 merged: mdlist{withSize(md1, 123)}, 785 }, { 786 name: "same tools in lhs and rhs, only rhs has a size: prefer rhs", 787 lhs: mdlist{md1}, 788 rhs: mdlist{withSize(md2, 123)}, 789 merged: mdlist{withSize(md2, 123)}, 790 }, { 791 name: "same tools in lhs and rhs, both have the same size: prefer lhs", 792 lhs: mdlist{withSize(md1, 123)}, 793 rhs: mdlist{withSize(md2, 123)}, 794 merged: mdlist{withSize(md1, 123)}, 795 }, { 796 name: "same tools in lhs and rhs, both have different sizes: error", 797 lhs: mdlist{withSize(md1, 123)}, 798 rhs: mdlist{withSize(md2, 456)}, 799 err: "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,456\\) sha256=\\(,\\)", 800 }, { 801 name: "same tools in lhs and rhs, both have same size but different sha256: error", 802 lhs: mdlist{withSHA256(withSize(md1, 123), "a")}, 803 rhs: mdlist{withSHA256(withSize(md2, 123), "b")}, 804 err: "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,123\\) sha256=\\(a,b\\)", 805 }, { 806 name: "lhs is a proper superset of rhs: union of lhs and rhs", 807 lhs: mdlist{md1, md3}, 808 rhs: mdlist{md1}, 809 merged: mdlist{md1, md3}, 810 }, { 811 name: "rhs is a proper superset of lhs: union of lhs and rhs", 812 lhs: mdlist{md1}, 813 rhs: mdlist{md1, md3}, 814 merged: mdlist{md1, md3}, 815 }} 816 for i, test := range tests { 817 c.Logf("test %d: %s", i, test.name) 818 merged, err := tools.MergeMetadata(test.lhs, test.rhs) 819 if test.err == "" { 820 c.Assert(err, jc.ErrorIsNil) 821 c.Assert(merged, gc.DeepEquals, test.merged) 822 } else { 823 c.Assert(err, gc.ErrorMatches, test.err) 824 c.Assert(merged, gc.IsNil) 825 } 826 } 827 } 828 829 func (*metadataHelperSuite) TestReadWriteMetadataSingleStream(c *gc.C) { 830 metadata := map[string][]*tools.ToolsMetadata{ 831 "released": {{ 832 Release: "precise", 833 Version: "1.2.3", 834 Arch: "amd64", 835 Path: "path1", 836 }, { 837 Release: "raring", 838 Version: "1.2.3", 839 Arch: "amd64", 840 Path: "path2", 841 }}, 842 } 843 844 stor, err := filestorage.NewFileStorageWriter(c.MkDir()) 845 c.Assert(err, jc.ErrorIsNil) 846 out, err := tools.ReadAllMetadata(stor) 847 c.Assert(err, jc.ErrorIsNil) // non-existence is not an error 848 c.Assert(out, gc.HasLen, 0) 849 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 850 c.Assert(err, jc.ErrorIsNil) 851 852 // Read back what was just written. 853 out, err = tools.ReadAllMetadata(stor) 854 c.Assert(err, jc.ErrorIsNil) 855 for _, outMetadata := range out { 856 for _, md := range outMetadata { 857 // FullPath is set by ReadAllMetadata. 858 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 859 md.FullPath = "" 860 } 861 } 862 c.Assert(out, jc.DeepEquals, metadata) 863 } 864 865 func (*metadataHelperSuite) writeMetadataMultipleStream(c *gc.C) (storage.StorageReader, map[string][]*tools.ToolsMetadata) { 866 metadata := map[string][]*tools.ToolsMetadata{ 867 "released": {{ 868 Release: "precise", 869 Version: "1.2.3", 870 Arch: "amd64", 871 Path: "path1", 872 }}, 873 "proposed": {{ 874 Release: "raring", 875 Version: "1.2.3", 876 Arch: "amd64", 877 Path: "path2", 878 }}, 879 } 880 881 stor, err := filestorage.NewFileStorageWriter(c.MkDir()) 882 c.Assert(err, jc.ErrorIsNil) 883 out, err := tools.ReadAllMetadata(stor) 884 c.Assert(out, gc.HasLen, 0) 885 c.Assert(err, jc.ErrorIsNil) // non-existence is not an error 886 err = tools.WriteMetadata(stor, metadata, []string{"released", "proposed"}, tools.DoNotWriteMirrors) 887 c.Assert(err, jc.ErrorIsNil) 888 return stor, metadata 889 } 890 891 func (s *metadataHelperSuite) TestReadWriteMetadataMultipleStream(c *gc.C) { 892 stor, metadata := s.writeMetadataMultipleStream(c) 893 // Read back what was just written. 894 out, err := tools.ReadAllMetadata(stor) 895 c.Assert(err, jc.ErrorIsNil) 896 for _, outMetadata := range out { 897 for _, md := range outMetadata { 898 // FullPath is set by ReadAllMetadata. 899 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 900 md.FullPath = "" 901 } 902 } 903 c.Assert(out, jc.DeepEquals, metadata) 904 } 905 906 func (s *metadataHelperSuite) TestWriteMetadataLegacyIndex(c *gc.C) { 907 stor, _ := s.writeMetadataMultipleStream(c) 908 // Read back the legacy index 909 rdr, err := stor.Get("tools/streams/v1/index.json") 910 defer rdr.Close() 911 c.Assert(err, jc.ErrorIsNil) 912 data, err := ioutil.ReadAll(rdr) 913 c.Assert(err, jc.ErrorIsNil) 914 var indices simplestreams.Indices 915 err = json.Unmarshal(data, &indices) 916 c.Assert(err, jc.ErrorIsNil) 917 c.Assert(indices.Indexes, gc.HasLen, 1) 918 indices.Updated = "" 919 c.Assert(indices.Indexes["com.ubuntu.juju:released:tools"], gc.NotNil) 920 indices.Indexes["com.ubuntu.juju:released:tools"].Updated = "" 921 expected := simplestreams.Indices{ 922 Format: "index:1.0", 923 Indexes: map[string]*simplestreams.IndexMetadata{ 924 "com.ubuntu.juju:released:tools": { 925 Format: "products:1.0", 926 DataType: "content-download", 927 ProductsFilePath: "streams/v1/com.ubuntu.juju-released-tools.json", 928 ProductIds: []string{"com.ubuntu.juju:12.04:amd64"}, 929 }, 930 }, 931 } 932 c.Assert(indices, jc.DeepEquals, expected) 933 } 934 935 func (s *metadataHelperSuite) TestReadWriteMetadataUnchanged(c *gc.C) { 936 metadata := map[string][]*tools.ToolsMetadata{ 937 "released": {{ 938 Release: "precise", 939 Version: "1.2.3", 940 Arch: "amd64", 941 Path: "path1", 942 }, { 943 Release: "raring", 944 Version: "1.2.3", 945 Arch: "amd64", 946 Path: "path2", 947 }}, 948 } 949 950 stor, err := filestorage.NewFileStorageWriter(c.MkDir()) 951 c.Assert(err, jc.ErrorIsNil) 952 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 953 c.Assert(err, jc.ErrorIsNil) 954 955 s.PatchValue(tools.WriteMetadataFiles, func(stor storage.Storage, metadataInfo []tools.MetadataFile) error { 956 // The product data is the same, we only write the indices. 957 c.Assert(metadataInfo, gc.HasLen, 2) 958 c.Assert(metadataInfo[0].Path, gc.Equals, "streams/v1/index2.json") 959 c.Assert(metadataInfo[1].Path, gc.Equals, "streams/v1/index.json") 960 return nil 961 }) 962 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 963 c.Assert(err, jc.ErrorIsNil) 964 } 965 966 func (*metadataHelperSuite) TestReadMetadataPrefersNewIndex(c *gc.C) { 967 if runtime.GOOS == "windows" { 968 c.Skip("Skipped for now because of introduced regression") 969 } 970 metadataDir := c.MkDir() 971 972 // Generate metadata and rename index to index.json 973 metadata := map[string][]*tools.ToolsMetadata{ 974 "proposed": {{ 975 Release: "precise", 976 Version: "1.2.3", 977 Arch: "amd64", 978 Path: "path1", 979 }}, 980 "released": {{ 981 Release: "trusty", 982 Version: "1.2.3", 983 Arch: "amd64", 984 Path: "path1", 985 }}, 986 } 987 stor, err := filestorage.NewFileStorageWriter(metadataDir) 988 c.Assert(err, jc.ErrorIsNil) 989 err = tools.WriteMetadata(stor, metadata, []string{"proposed", "released"}, tools.DoNotWriteMirrors) 990 c.Assert(err, jc.ErrorIsNil) 991 err = os.Rename( 992 filepath.Join(metadataDir, "tools", "streams", "v1", "index2.json"), 993 filepath.Join(metadataDir, "tools", "streams", "v1", "index.json"), 994 ) 995 c.Assert(err, jc.ErrorIsNil) 996 997 // Generate different metadata with index2.json 998 metadata = map[string][]*tools.ToolsMetadata{ 999 "released": {{ 1000 Release: "precise", 1001 Version: "1.2.3", 1002 Arch: "amd64", 1003 Path: "path1", 1004 }}, 1005 } 1006 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 1007 c.Assert(err, jc.ErrorIsNil) 1008 1009 // Read back all metadata, expecting to find metadata in index2.json. 1010 out, err := tools.ReadAllMetadata(stor) 1011 for _, outMetadata := range out { 1012 for _, md := range outMetadata { 1013 // FullPath is set by ReadAllMetadata. 1014 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 1015 md.FullPath = "" 1016 } 1017 } 1018 c.Assert(out, jc.DeepEquals, metadata) 1019 } 1020 1021 type signedSuite struct { 1022 coretesting.BaseSuite 1023 } 1024 1025 func (s *signedSuite) SetUpSuite(c *gc.C) { 1026 s.BaseSuite.SetUpSuite(c) 1027 var imageData = map[string]string{ 1028 "/unsigned/streams/v1/index.json": unsignedIndex, 1029 "/unsigned/streams/v1/tools_metadata.json": unsignedProduct, 1030 } 1031 1032 // Set up some signed data from the unsigned data. 1033 // Overwrite the product path to use the sjson suffix. 1034 rawUnsignedIndex := strings.Replace( 1035 unsignedIndex, "streams/v1/tools_metadata.json", "streams/v1/tools_metadata.sjson", -1) 1036 r := bytes.NewReader([]byte(rawUnsignedIndex)) 1037 signedData, err := simplestreams.Encode( 1038 r, sstesting.SignedMetadataPrivateKey, sstesting.PrivateKeyPassphrase) 1039 c.Assert(err, jc.ErrorIsNil) 1040 imageData["/signed/streams/v1/index.sjson"] = string(signedData) 1041 1042 // Replace the tools path in the unsigned data with a different one so we can test that the right 1043 // tools path is used. 1044 rawUnsignedProduct := strings.Replace( 1045 unsignedProduct, "juju-1.13.0", "juju-1.13.1", -1) 1046 r = bytes.NewReader([]byte(rawUnsignedProduct)) 1047 signedData, err = simplestreams.Encode( 1048 r, sstesting.SignedMetadataPrivateKey, sstesting.PrivateKeyPassphrase) 1049 c.Assert(err, jc.ErrorIsNil) 1050 imageData["/signed/streams/v1/tools_metadata.sjson"] = string(signedData) 1051 sstesting.SetRoundTripperFiles(imageData, map[string]int{"signedtest://unauth": http.StatusUnauthorized}) 1052 s.PatchValue(&keys.JujuPublicKey, sstesting.SignedMetadataPublicKey) 1053 } 1054 1055 func (s *signedSuite) TearDownSuite(c *gc.C) { 1056 sstesting.SetRoundTripperFiles(nil, nil) 1057 s.BaseSuite.TearDownSuite(c) 1058 } 1059 1060 func (s *signedSuite) TestSignedToolsMetadata(c *gc.C) { 1061 signedSource := simplestreams.NewURLSignedDataSource( 1062 "test", "signedtest://host/signed", sstesting.SignedMetadataPublicKey, 1063 utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, true) 1064 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 1065 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 1066 Series: []string{"precise"}, 1067 Arches: []string{"amd64"}, 1068 Stream: "released", 1069 }) 1070 toolsMetadata, resolveInfo, err := tools.Fetch( 1071 []simplestreams.DataSource{signedSource}, toolsConstraint) 1072 c.Assert(err, jc.ErrorIsNil) 1073 c.Assert(len(toolsMetadata), gc.Equals, 1) 1074 c.Assert(toolsMetadata[0].Path, gc.Equals, "tools/releases/20130806/juju-1.13.1-precise-amd64.tgz") 1075 c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{ 1076 Source: "test", 1077 Signed: true, 1078 IndexURL: "signedtest://host/signed/streams/v1/index.sjson", 1079 MirrorURL: "", 1080 }) 1081 } 1082 1083 var unsignedIndex = ` 1084 { 1085 "index": { 1086 "com.ubuntu.juju:released:tools": { 1087 "updated": "Mon, 05 Aug 2013 11:07:04 +0000", 1088 "datatype": "content-download", 1089 "format": "products:1.0", 1090 "products": [ 1091 "com.ubuntu.juju:12.04:amd64" 1092 ], 1093 "path": "streams/v1/tools_metadata.json" 1094 } 1095 }, 1096 "updated": "Wed, 01 May 2013 13:31:26 +0000", 1097 "format": "index:1.0" 1098 } 1099 ` 1100 var unsignedProduct = ` 1101 { 1102 "updated": "Wed, 01 May 2013 13:31:26 +0000", 1103 "content_id": "com.ubuntu.cloud:released:aws", 1104 "datatype": "content-download", 1105 "products": { 1106 "com.ubuntu.juju:12.04:amd64": { 1107 "arch": "amd64", 1108 "release": "precise", 1109 "versions": { 1110 "20130806": { 1111 "items": { 1112 "1130preciseamd64": { 1113 "version": "1.13.0", 1114 "size": 2973595, 1115 "path": "tools/releases/20130806/juju-1.13.0-precise-amd64.tgz", 1116 "ftype": "tar.gz", 1117 "sha256": "447aeb6a934a5eaec4f703eda4ef2dde" 1118 } 1119 } 1120 } 1121 } 1122 } 1123 }, 1124 "format": "products:1.0" 1125 } 1126 `