launchpad.net/~rogpeppe/juju-core/500-errgo-fix@v0.0.0-20140213181702-000000002356/environs/tools/simplestreams_test.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package tools_test 5 6 import ( 7 "bytes" 8 "flag" 9 "fmt" 10 "io" 11 "net/http" 12 "reflect" 13 "strings" 14 "testing" 15 16 "launchpad.net/goamz/aws" 17 gc "launchpad.net/gocheck" 18 19 "launchpad.net/juju-core/environs/filestorage" 20 "launchpad.net/juju-core/environs/jujutest" 21 "launchpad.net/juju-core/environs/simplestreams" 22 sstesting "launchpad.net/juju-core/environs/simplestreams/testing" 23 "launchpad.net/juju-core/environs/storage" 24 "launchpad.net/juju-core/environs/tools" 25 ttesting "launchpad.net/juju-core/environs/tools/testing" 26 "launchpad.net/juju-core/testing/testbase" 27 coretools "launchpad.net/juju-core/tools" 28 "launchpad.net/juju-core/version" 29 ) 30 31 var live = flag.Bool("live", false, "Include live simplestreams tests") 32 var vendor = flag.String("vendor", "", "The vendor representing the source of the simplestream data") 33 34 type liveTestData struct { 35 baseURL string 36 requireSigned bool 37 validCloudSpec simplestreams.CloudSpec 38 } 39 40 var liveUrls = map[string]liveTestData{ 41 "ec2": { 42 baseURL: tools.DefaultBaseURL, 43 requireSigned: true, 44 validCloudSpec: simplestreams.CloudSpec{"us-east-1", aws.Regions["us-east-1"].EC2Endpoint}, 45 }, 46 "canonistack": { 47 baseURL: "https://swift.canonistack.canonical.com/v1/AUTH_526ad877f3e3464589dc1145dfeaac60/juju-tools", 48 requireSigned: false, 49 validCloudSpec: simplestreams.CloudSpec{"lcy01", "https://keystone.canonistack.canonical.com:443/v2.0/"}, 50 }, 51 } 52 53 func setupSimpleStreamsTests(t *testing.T) { 54 if *live { 55 if *vendor == "" { 56 t.Fatal("missing vendor") 57 } 58 var ok bool 59 var testData liveTestData 60 if testData, ok = liveUrls[*vendor]; !ok { 61 keys := reflect.ValueOf(liveUrls).MapKeys() 62 t.Fatalf("Unknown vendor %s. Must be one of %s", *vendor, keys) 63 } 64 registerLiveSimpleStreamsTests(testData.baseURL, 65 tools.NewVersionedToolsConstraint("1.13.0", simplestreams.LookupParams{ 66 CloudSpec: testData.validCloudSpec, 67 Series: []string{version.Current.Series}, 68 Arches: []string{"amd64"}, 69 }), testData.requireSigned) 70 } 71 registerSimpleStreamsTests() 72 } 73 74 func registerSimpleStreamsTests() { 75 gc.Suite(&simplestreamsSuite{ 76 LocalLiveSimplestreamsSuite: sstesting.LocalLiveSimplestreamsSuite{ 77 Source: simplestreams.NewURLDataSource("test:", simplestreams.VerifySSLHostnames), 78 RequireSigned: false, 79 DataType: tools.ContentDownload, 80 ValidConstraint: tools.NewVersionedToolsConstraint("1.13.0", simplestreams.LookupParams{ 81 CloudSpec: simplestreams.CloudSpec{ 82 Region: "us-east-1", 83 Endpoint: "https://ec2.us-east-1.amazonaws.com", 84 }, 85 Series: []string{"precise"}, 86 Arches: []string{"amd64", "arm"}, 87 }), 88 }, 89 }) 90 gc.Suite(&signedSuite{}) 91 } 92 93 func registerLiveSimpleStreamsTests(baseURL string, validToolsConstraint simplestreams.LookupConstraint, requireSigned bool) { 94 gc.Suite(&sstesting.LocalLiveSimplestreamsSuite{ 95 Source: simplestreams.NewURLDataSource(baseURL, simplestreams.VerifySSLHostnames), 96 RequireSigned: requireSigned, 97 DataType: tools.ContentDownload, 98 ValidConstraint: validToolsConstraint, 99 }) 100 } 101 102 type simplestreamsSuite struct { 103 sstesting.LocalLiveSimplestreamsSuite 104 sstesting.TestDataSuite 105 } 106 107 func (s *simplestreamsSuite) SetUpSuite(c *gc.C) { 108 s.LocalLiveSimplestreamsSuite.SetUpSuite(c) 109 s.TestDataSuite.SetUpSuite(c) 110 } 111 112 func (s *simplestreamsSuite) TearDownSuite(c *gc.C) { 113 s.TestDataSuite.TearDownSuite(c) 114 s.LocalLiveSimplestreamsSuite.TearDownSuite(c) 115 } 116 117 var fetchTests = []struct { 118 region string 119 series string 120 version string 121 major int 122 minor int 123 released bool 124 arches []string 125 tools []*tools.ToolsMetadata 126 }{{ 127 series: "precise", 128 arches: []string{"amd64", "arm"}, 129 version: "1.13.0", 130 tools: []*tools.ToolsMetadata{ 131 { 132 Release: "precise", 133 Version: "1.13.0", 134 Arch: "amd64", 135 Size: 2973595, 136 Path: "tools/releases/20130806/juju-1.13.0-precise-amd64.tgz", 137 FileType: "tar.gz", 138 SHA256: "447aeb6a934a5eaec4f703eda4ef2dde", 139 }, 140 }, 141 }, { 142 series: "raring", 143 arches: []string{"amd64", "arm"}, 144 version: "1.13.0", 145 tools: []*tools.ToolsMetadata{ 146 { 147 Release: "raring", 148 Version: "1.13.0", 149 Arch: "amd64", 150 Size: 2973173, 151 Path: "tools/releases/20130806/juju-1.13.0-raring-amd64.tgz", 152 FileType: "tar.gz", 153 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 154 }, 155 }, 156 }, { 157 series: "raring", 158 arches: []string{"amd64", "arm"}, 159 version: "1.11.4", 160 tools: []*tools.ToolsMetadata{ 161 { 162 Release: "raring", 163 Version: "1.11.4", 164 Arch: "arm", 165 Size: 1950327, 166 Path: "tools/releases/20130806/juju-1.11.4-raring-arm.tgz", 167 FileType: "tar.gz", 168 SHA256: "6472014e3255e3fe7fbd3550ef3f0a11", 169 }, 170 }, 171 }, { 172 series: "precise", 173 arches: []string{"amd64", "arm"}, 174 major: 2, 175 tools: []*tools.ToolsMetadata{ 176 { 177 Release: "precise", 178 Version: "2.0.1", 179 Arch: "arm", 180 Size: 1951096, 181 Path: "tools/releases/20130806/juju-2.0.1-precise-arm.tgz", 182 FileType: "tar.gz", 183 SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", 184 }, 185 }, 186 }, { 187 series: "precise", 188 arches: []string{"amd64", "arm"}, 189 major: 1, 190 minor: 11, 191 tools: []*tools.ToolsMetadata{ 192 { 193 Release: "precise", 194 Version: "1.11.4", 195 Arch: "arm", 196 Size: 1951096, 197 Path: "tools/releases/20130806/juju-1.11.4-precise-arm.tgz", 198 FileType: "tar.gz", 199 SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", 200 }, 201 { 202 Release: "precise", 203 Version: "1.11.5", 204 Arch: "arm", 205 Size: 2031281, 206 Path: "tools/releases/20130803/juju-1.11.5-precise-arm.tgz", 207 FileType: "tar.gz", 208 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 209 }, 210 }, 211 }, { 212 series: "raring", 213 arches: []string{"amd64", "arm"}, 214 major: 1, 215 minor: -1, 216 released: true, 217 tools: []*tools.ToolsMetadata{ 218 { 219 Release: "raring", 220 Version: "1.14.0", 221 Arch: "amd64", 222 Size: 2973173, 223 Path: "tools/releases/20130806/juju-1.14.0-raring-amd64.tgz", 224 FileType: "tar.gz", 225 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 226 }, 227 }, 228 }} 229 230 func (s *simplestreamsSuite) TestFetch(c *gc.C) { 231 for i, t := range fetchTests { 232 c.Logf("test %d", i) 233 var toolsConstraint *tools.ToolsConstraint 234 if t.version == "" { 235 toolsConstraint = tools.NewGeneralToolsConstraint(t.major, t.minor, t.released, simplestreams.LookupParams{ 236 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 237 Series: []string{t.series}, 238 Arches: t.arches, 239 }) 240 } else { 241 toolsConstraint = tools.NewVersionedToolsConstraint(t.version, simplestreams.LookupParams{ 242 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 243 Series: []string{t.series}, 244 Arches: t.arches, 245 }) 246 } 247 tools, err := tools.Fetch( 248 []simplestreams.DataSource{s.Source}, simplestreams.DefaultIndexPath, toolsConstraint, s.RequireSigned) 249 if !c.Check(err, gc.IsNil) { 250 continue 251 } 252 for _, tm := range t.tools { 253 tm.FullPath, err = s.Source.URL(tm.Path) 254 c.Assert(err, gc.IsNil) 255 } 256 c.Check(tools, gc.DeepEquals, t.tools) 257 } 258 } 259 260 func (s *simplestreamsSuite) TestFetchWithMirror(c *gc.C) { 261 toolsConstraint := tools.NewGeneralToolsConstraint(1, 13, false, simplestreams.LookupParams{ 262 CloudSpec: simplestreams.CloudSpec{"us-west-2", "https://ec2.us-west-2.amazonaws.com"}, 263 Series: []string{"precise"}, 264 Arches: []string{"amd64"}, 265 }) 266 toolsMetadata, err := tools.Fetch( 267 []simplestreams.DataSource{s.Source}, simplestreams.DefaultIndexPath, toolsConstraint, s.RequireSigned) 268 c.Assert(err, gc.IsNil) 269 c.Assert(len(toolsMetadata), gc.Equals, 1) 270 271 expectedMetadata := &tools.ToolsMetadata{ 272 Release: "precise", 273 Version: "1.13.0", 274 Arch: "amd64", 275 Size: 2973595, 276 Path: "mirrored-path/juju-1.13.0-precise-amd64.tgz", 277 FullPath: "test:/mirrored-path/juju-1.13.0-precise-amd64.tgz", 278 FileType: "tar.gz", 279 SHA256: "447aeb6a934a5eaec4f703eda4ef2dde", 280 } 281 c.Assert(err, gc.IsNil) 282 c.Assert(toolsMetadata[0], gc.DeepEquals, expectedMetadata) 283 } 284 285 func assertMetadataMatches(c *gc.C, storageDir string, toolList coretools.List, metadata []*tools.ToolsMetadata) { 286 var expectedMetadata []*tools.ToolsMetadata = make([]*tools.ToolsMetadata, len(toolList)) 287 for i, tool := range toolList { 288 expectedMetadata[i] = &tools.ToolsMetadata{ 289 Release: tool.Version.Series, 290 Version: tool.Version.Number.String(), 291 Arch: tool.Version.Arch, 292 Size: tool.Size, 293 Path: fmt.Sprintf("releases/juju-%s.tgz", tool.Version.String()), 294 FileType: "tar.gz", 295 SHA256: tool.SHA256, 296 } 297 } 298 c.Assert(metadata, gc.DeepEquals, expectedMetadata) 299 } 300 301 func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) { 302 toolsList := coretools.List{ 303 { 304 Version: version.MustParseBinary("1.2.3-precise-amd64"), 305 Size: 123, 306 SHA256: "abcd", 307 }, { 308 Version: version.MustParseBinary("2.0.1-raring-amd64"), 309 Size: 456, 310 SHA256: "xyz", 311 }, 312 } 313 dir := c.MkDir() 314 writer, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir) 315 c.Assert(err, gc.IsNil) 316 err = tools.MergeAndWriteMetadata(writer, toolsList, tools.DoNotWriteMirrors) 317 c.Assert(err, gc.IsNil) 318 metadata := ttesting.ParseMetadataFromDir(c, dir, false) 319 assertMetadataMatches(c, dir, toolsList, metadata) 320 } 321 322 func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) { 323 var versionStrings = []string{ 324 "1.2.3-precise-amd64", 325 "2.0.1-raring-amd64", 326 } 327 dir := c.MkDir() 328 ttesting.MakeTools(c, dir, "releases", versionStrings) 329 330 toolsList := coretools.List{ 331 { 332 // If sha256/size is already known, do not recalculate 333 Version: version.MustParseBinary("1.2.3-precise-amd64"), 334 Size: 123, 335 SHA256: "abcd", 336 }, { 337 Version: version.MustParseBinary("2.0.1-raring-amd64"), 338 // The URL is not used for generating metadata. 339 URL: "bogus://", 340 }, 341 } 342 writer, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir) 343 c.Assert(err, gc.IsNil) 344 writeMirrors := tools.DoNotWriteMirrors 345 if withMirrors { 346 writeMirrors = tools.WriteMirrors 347 } 348 err = tools.MergeAndWriteMetadata(writer, toolsList, writeMirrors) 349 c.Assert(err, gc.IsNil) 350 metadata := ttesting.ParseMetadataFromDir(c, dir, withMirrors) 351 assertMetadataMatches(c, dir, toolsList, metadata) 352 } 353 354 func (s *simplestreamsSuite) TestWriteMetadata(c *gc.C) { 355 s.assertWriteMetadata(c, false) 356 } 357 358 func (s *simplestreamsSuite) TestWriteMetadataWithMirrors(c *gc.C) { 359 s.assertWriteMetadata(c, true) 360 } 361 362 func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) { 363 dir := c.MkDir() 364 existingToolsList := coretools.List{ 365 { 366 Version: version.MustParseBinary("1.2.3-precise-amd64"), 367 Size: 123, 368 SHA256: "abc", 369 }, { 370 Version: version.MustParseBinary("2.0.1-raring-amd64"), 371 Size: 456, 372 SHA256: "xyz", 373 }, 374 } 375 writer, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir) 376 c.Assert(err, gc.IsNil) 377 err = tools.MergeAndWriteMetadata(writer, existingToolsList, tools.DoNotWriteMirrors) 378 c.Assert(err, gc.IsNil) 379 newToolsList := coretools.List{ 380 existingToolsList[0], 381 { 382 Version: version.MustParseBinary("2.1.0-raring-amd64"), 383 Size: 789, 384 SHA256: "def", 385 }, 386 } 387 err = tools.MergeAndWriteMetadata(writer, newToolsList, tools.DoNotWriteMirrors) 388 c.Assert(err, gc.IsNil) 389 requiredToolsList := append(existingToolsList, newToolsList[1]) 390 metadata := ttesting.ParseMetadataFromDir(c, dir, false) 391 assertMetadataMatches(c, dir, requiredToolsList, metadata) 392 } 393 394 type productSpecSuite struct{} 395 396 var _ = gc.Suite(&productSpecSuite{}) 397 398 func (s *productSpecSuite) TestId(c *gc.C) { 399 toolsConstraint := tools.NewVersionedToolsConstraint("1.13.0", simplestreams.LookupParams{ 400 Series: []string{"precise"}, 401 Arches: []string{"amd64"}, 402 }) 403 ids, err := toolsConstraint.Ids() 404 c.Assert(err, gc.IsNil) 405 c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.juju:12.04:amd64"}) 406 } 407 408 func (s *productSpecSuite) TestIdMultiArch(c *gc.C) { 409 toolsConstraint := tools.NewVersionedToolsConstraint("1.11.3", simplestreams.LookupParams{ 410 Series: []string{"precise"}, 411 Arches: []string{"amd64", "arm"}, 412 }) 413 ids, err := toolsConstraint.Ids() 414 c.Assert(err, gc.IsNil) 415 c.Assert(ids, gc.DeepEquals, []string{ 416 "com.ubuntu.juju:12.04:amd64", 417 "com.ubuntu.juju:12.04:arm"}) 418 } 419 420 func (s *productSpecSuite) TestIdMultiSeries(c *gc.C) { 421 toolsConstraint := tools.NewVersionedToolsConstraint("1.11.3", simplestreams.LookupParams{ 422 Series: []string{"precise", "raring"}, 423 Arches: []string{"amd64"}, 424 }) 425 ids, err := toolsConstraint.Ids() 426 c.Assert(err, gc.IsNil) 427 c.Assert(ids, gc.DeepEquals, []string{ 428 "com.ubuntu.juju:12.04:amd64", 429 "com.ubuntu.juju:13.04:amd64"}) 430 } 431 432 func (s *productSpecSuite) TestIdWithMajorVersionOnly(c *gc.C) { 433 toolsConstraint := tools.NewGeneralToolsConstraint(1, -1, false, simplestreams.LookupParams{ 434 Series: []string{"precise"}, 435 Arches: []string{"amd64"}, 436 }) 437 ids, err := toolsConstraint.Ids() 438 c.Assert(err, gc.IsNil) 439 c.Assert(ids, gc.DeepEquals, []string{`com.ubuntu.juju:12.04:amd64`}) 440 } 441 442 func (s *productSpecSuite) TestIdWithMajorMinorVersion(c *gc.C) { 443 toolsConstraint := tools.NewGeneralToolsConstraint(1, 2, false, simplestreams.LookupParams{ 444 Series: []string{"precise"}, 445 Arches: []string{"amd64"}, 446 }) 447 ids, err := toolsConstraint.Ids() 448 c.Assert(err, gc.IsNil) 449 c.Assert(ids, gc.DeepEquals, []string{`com.ubuntu.juju:12.04:amd64`}) 450 } 451 452 func (s *productSpecSuite) TestLargeNumber(c *gc.C) { 453 json := `{ 454 "updated": "Fri, 30 Aug 2013 16:12:58 +0800", 455 "format": "products:1.0", 456 "products": { 457 "com.ubuntu.juju:1.10.0:amd64": { 458 "version": "1.10.0", 459 "arch": "amd64", 460 "versions": { 461 "20133008": { 462 "items": { 463 "1.10.0-precise-amd64": { 464 "release": "precise", 465 "version": "1.10.0", 466 "arch": "amd64", 467 "size": 9223372036854775807, 468 "path": "releases/juju-1.10.0-precise-amd64.tgz", 469 "ftype": "tar.gz", 470 "sha256": "" 471 } 472 } 473 } 474 } 475 } 476 } 477 }` 478 cloudMetadata, err := simplestreams.ParseCloudMetadata([]byte(json), "products:1.0", "", tools.ToolsMetadata{}) 479 c.Assert(err, gc.IsNil) 480 c.Assert(cloudMetadata.Products, gc.HasLen, 1) 481 product := cloudMetadata.Products["com.ubuntu.juju:1.10.0:amd64"] 482 c.Assert(product, gc.NotNil) 483 c.Assert(product.Items, gc.HasLen, 1) 484 version := product.Items["20133008"] 485 c.Assert(version, gc.NotNil) 486 c.Assert(version.Items, gc.HasLen, 1) 487 item := version.Items["1.10.0-precise-amd64"] 488 c.Assert(item, gc.NotNil) 489 c.Assert(item, gc.FitsTypeOf, &tools.ToolsMetadata{}) 490 c.Assert(item.(*tools.ToolsMetadata).Size, gc.Equals, int64(9223372036854775807)) 491 } 492 493 type metadataHelperSuite struct { 494 testbase.LoggingSuite 495 } 496 497 var _ = gc.Suite(&metadataHelperSuite{}) 498 499 func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) { 500 metadata := tools.MetadataFromTools(nil) 501 c.Assert(metadata, gc.HasLen, 0) 502 503 toolsList := coretools.List{{ 504 Version: version.MustParseBinary("1.2.3-precise-amd64"), 505 Size: 123, 506 SHA256: "abc", 507 }, { 508 Version: version.MustParseBinary("2.0.1-raring-amd64"), 509 URL: "file:///tmp/releases/juju-2.0.1-raring-amd64.tgz", 510 Size: 456, 511 SHA256: "xyz", 512 }} 513 metadata = tools.MetadataFromTools(toolsList) 514 c.Assert(metadata, gc.HasLen, len(toolsList)) 515 for i, t := range toolsList { 516 md := metadata[i] 517 c.Assert(md.Release, gc.Equals, t.Version.Series) 518 c.Assert(md.Version, gc.Equals, t.Version.Number.String()) 519 c.Assert(md.Arch, gc.Equals, t.Version.Arch) 520 // FullPath is only filled out when reading tools using simplestreams. 521 // It's not needed elsewhere and requires a URL() call. 522 c.Assert(md.FullPath, gc.Equals, "") 523 c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version)[len("tools/"):]) 524 c.Assert(md.FileType, gc.Equals, "tar.gz") 525 c.Assert(md.Size, gc.Equals, t.Size) 526 c.Assert(md.SHA256, gc.Equals, t.SHA256) 527 } 528 } 529 530 type countingStorage struct { 531 storage.StorageReader 532 counter int 533 } 534 535 func (c *countingStorage) Get(name string) (io.ReadCloser, error) { 536 c.counter++ 537 return c.StorageReader.Get(name) 538 } 539 540 func (*metadataHelperSuite) TestResolveMetadata(c *gc.C) { 541 var versionStrings = []string{"1.2.3-precise-amd64"} 542 dir := c.MkDir() 543 ttesting.MakeTools(c, dir, "releases", versionStrings) 544 toolsList := coretools.List{{ 545 Version: version.MustParseBinary(versionStrings[0]), 546 Size: 123, 547 SHA256: "abc", 548 }} 549 550 stor, err := filestorage.NewFileStorageReader(dir) 551 c.Assert(err, gc.IsNil) 552 err = tools.ResolveMetadata(stor, nil) 553 c.Assert(err, gc.IsNil) 554 555 // We already have size/sha256, so ensure that storage isn't consulted. 556 countingStorage := &countingStorage{StorageReader: stor} 557 metadata := tools.MetadataFromTools(toolsList) 558 err = tools.ResolveMetadata(countingStorage, metadata) 559 c.Assert(err, gc.IsNil) 560 c.Assert(countingStorage.counter, gc.Equals, 0) 561 562 // Now clear size/sha256, and check that it is called, and 563 // the size/sha256 sum are updated. 564 metadata[0].Size = 0 565 metadata[0].SHA256 = "" 566 err = tools.ResolveMetadata(countingStorage, metadata) 567 c.Assert(err, gc.IsNil) 568 c.Assert(countingStorage.counter, gc.Equals, 1) 569 c.Assert(metadata[0].Size, gc.Not(gc.Equals), 0) 570 c.Assert(metadata[0].SHA256, gc.Not(gc.Equals), "") 571 } 572 573 func (*metadataHelperSuite) TestMergeMetadata(c *gc.C) { 574 md1 := &tools.ToolsMetadata{ 575 Release: "precise", 576 Version: "1.2.3", 577 Arch: "amd64", 578 Path: "path1", 579 } 580 md2 := &tools.ToolsMetadata{ 581 Release: "precise", 582 Version: "1.2.3", 583 Arch: "amd64", 584 Path: "path2", 585 } 586 md3 := &tools.ToolsMetadata{ 587 Release: "raring", 588 Version: "1.2.3", 589 Arch: "amd64", 590 Path: "path3", 591 } 592 593 withSize := func(md *tools.ToolsMetadata, size int64) *tools.ToolsMetadata { 594 clone := *md 595 clone.Size = size 596 return &clone 597 } 598 withSHA256 := func(md *tools.ToolsMetadata, sha256 string) *tools.ToolsMetadata { 599 clone := *md 600 clone.SHA256 = sha256 601 return &clone 602 } 603 604 type mdlist []*tools.ToolsMetadata 605 type test struct { 606 name string 607 lhs, rhs, merged []*tools.ToolsMetadata 608 err string 609 } 610 tests := []test{{ 611 name: "non-empty lhs, empty rhs", 612 lhs: mdlist{md1}, 613 rhs: nil, 614 merged: mdlist{md1}, 615 }, { 616 name: "empty lhs, non-empty rhs", 617 lhs: nil, 618 rhs: mdlist{md2}, 619 merged: mdlist{md2}, 620 }, { 621 name: "identical lhs, rhs", 622 lhs: mdlist{md1}, 623 rhs: mdlist{md1}, 624 merged: mdlist{md1}, 625 }, { 626 name: "same tools in lhs and rhs, neither have size: prefer lhs", 627 lhs: mdlist{md1}, 628 rhs: mdlist{md2}, 629 merged: mdlist{md1}, 630 }, { 631 name: "same tools in lhs and rhs, only lhs has a size: prefer lhs", 632 lhs: mdlist{withSize(md1, 123)}, 633 rhs: mdlist{md2}, 634 merged: mdlist{withSize(md1, 123)}, 635 }, { 636 name: "same tools in lhs and rhs, only rhs has a size: prefer rhs", 637 lhs: mdlist{md1}, 638 rhs: mdlist{withSize(md2, 123)}, 639 merged: mdlist{withSize(md2, 123)}, 640 }, { 641 name: "same tools in lhs and rhs, both have the same size: prefer lhs", 642 lhs: mdlist{withSize(md1, 123)}, 643 rhs: mdlist{withSize(md2, 123)}, 644 merged: mdlist{withSize(md1, 123)}, 645 }, { 646 name: "same tools in lhs and rhs, both have different sizes: error", 647 lhs: mdlist{withSize(md1, 123)}, 648 rhs: mdlist{withSize(md2, 456)}, 649 err: "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,456\\) sha256=\\(,\\)", 650 }, { 651 name: "same tools in lhs and rhs, both have same size but different sha256: error", 652 lhs: mdlist{withSHA256(withSize(md1, 123), "a")}, 653 rhs: mdlist{withSHA256(withSize(md2, 123), "b")}, 654 err: "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,123\\) sha256=\\(a,b\\)", 655 }, { 656 name: "lhs is a proper superset of rhs: union of lhs and rhs", 657 lhs: mdlist{md1, md3}, 658 rhs: mdlist{md1}, 659 merged: mdlist{md1, md3}, 660 }, { 661 name: "rhs is a proper superset of lhs: union of lhs and rhs", 662 lhs: mdlist{md1}, 663 rhs: mdlist{md1, md3}, 664 merged: mdlist{md1, md3}, 665 }} 666 for i, test := range tests { 667 c.Logf("test %d: %s", i, test.name) 668 merged, err := tools.MergeMetadata(test.lhs, test.rhs) 669 if test.err == "" { 670 c.Assert(err, gc.IsNil) 671 c.Assert(merged, gc.DeepEquals, test.merged) 672 } else { 673 c.Assert(err, gc.ErrorMatches, test.err) 674 c.Assert(merged, gc.IsNil) 675 } 676 } 677 } 678 679 func (*metadataHelperSuite) TestReadWriteMetadata(c *gc.C) { 680 metadata := []*tools.ToolsMetadata{{ 681 Release: "precise", 682 Version: "1.2.3", 683 Arch: "amd64", 684 Path: "path1", 685 }, { 686 Release: "raring", 687 Version: "1.2.3", 688 Arch: "amd64", 689 Path: "path2", 690 }} 691 692 stor, err := filestorage.NewFileStorageWriter(c.MkDir(), filestorage.UseDefaultTmpDir) 693 c.Assert(err, gc.IsNil) 694 out, err := tools.ReadMetadata(stor) 695 c.Assert(out, gc.HasLen, 0) 696 c.Assert(err, gc.IsNil) // non-existence is not an error 697 err = tools.WriteMetadata(stor, metadata, tools.DoNotWriteMirrors) 698 c.Assert(err, gc.IsNil) 699 out, err = tools.ReadMetadata(stor) 700 for _, md := range out { 701 // FullPath is set by ReadMetadata. 702 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 703 md.FullPath = "" 704 } 705 c.Assert(out, gc.DeepEquals, metadata) 706 } 707 708 type signedSuite struct { 709 origKey string 710 } 711 712 var testRoundTripper *jujutest.ProxyRoundTripper 713 714 func init() { 715 testRoundTripper = &jujutest.ProxyRoundTripper{} 716 simplestreams.RegisterProtocol("signedtest", testRoundTripper) 717 } 718 719 func (s *signedSuite) SetUpSuite(c *gc.C) { 720 var imageData = map[string]string{ 721 "/unsigned/streams/v1/index.json": unsignedIndex, 722 "/unsigned/streams/v1/tools_metadata.json": unsignedProduct, 723 } 724 725 // Set up some signed data from the unsigned data. 726 // Overwrite the product path to use the sjson suffix. 727 rawUnsignedIndex := strings.Replace( 728 unsignedIndex, "streams/v1/tools_metadata.json", "streams/v1/tools_metadata.sjson", -1) 729 r := bytes.NewReader([]byte(rawUnsignedIndex)) 730 signedData, err := simplestreams.Encode( 731 r, sstesting.SignedMetadataPrivateKey, sstesting.PrivateKeyPassphrase) 732 c.Assert(err, gc.IsNil) 733 imageData["/signed/streams/v1/index.sjson"] = string(signedData) 734 735 // Replace the tools path in the unsigned data with a different one so we can test that the right 736 // tools path is used. 737 rawUnsignedProduct := strings.Replace( 738 unsignedProduct, "juju-1.13.0", "juju-1.13.1", -1) 739 r = bytes.NewReader([]byte(rawUnsignedProduct)) 740 signedData, err = simplestreams.Encode( 741 r, sstesting.SignedMetadataPrivateKey, sstesting.PrivateKeyPassphrase) 742 c.Assert(err, gc.IsNil) 743 imageData["/signed/streams/v1/tools_metadata.sjson"] = string(signedData) 744 testRoundTripper.Sub = jujutest.NewCannedRoundTripper( 745 imageData, map[string]int{"signedtest://unauth": http.StatusUnauthorized}) 746 s.origKey = tools.SetSigningPublicKey(sstesting.SignedMetadataPublicKey) 747 } 748 749 func (s *signedSuite) TearDownSuite(c *gc.C) { 750 testRoundTripper.Sub = nil 751 tools.SetSigningPublicKey(s.origKey) 752 } 753 754 func (s *signedSuite) TestSignedToolsMetadata(c *gc.C) { 755 signedSource := simplestreams.NewURLDataSource("signedtest://host/signed", simplestreams.VerifySSLHostnames) 756 toolsConstraint := tools.NewVersionedToolsConstraint("1.13.0", simplestreams.LookupParams{ 757 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 758 Series: []string{"precise"}, 759 Arches: []string{"amd64"}, 760 }) 761 toolsMetadata, err := tools.Fetch( 762 []simplestreams.DataSource{signedSource}, simplestreams.DefaultIndexPath, toolsConstraint, true) 763 c.Assert(err, gc.IsNil) 764 c.Assert(len(toolsMetadata), gc.Equals, 1) 765 c.Assert(toolsMetadata[0].Path, gc.Equals, "tools/releases/20130806/juju-1.13.1-precise-amd64.tgz") 766 } 767 768 var unsignedIndex = ` 769 { 770 "index": { 771 "com.ubuntu.juju:released:tools": { 772 "updated": "Mon, 05 Aug 2013 11:07:04 +0000", 773 "datatype": "content-download", 774 "format": "products:1.0", 775 "products": [ 776 "com.ubuntu.juju:12.04:amd64" 777 ], 778 "path": "streams/v1/tools_metadata.json" 779 } 780 }, 781 "updated": "Wed, 01 May 2013 13:31:26 +0000", 782 "format": "index:1.0" 783 } 784 ` 785 var unsignedProduct = ` 786 { 787 "updated": "Wed, 01 May 2013 13:31:26 +0000", 788 "content_id": "com.ubuntu.cloud:released:aws", 789 "datatype": "content-download", 790 "products": { 791 "com.ubuntu.juju:12.04:amd64": { 792 "arch": "amd64", 793 "release": "precise", 794 "versions": { 795 "20130806": { 796 "items": { 797 "1130preciseamd64": { 798 "version": "1.13.0", 799 "size": 2973595, 800 "path": "tools/releases/20130806/juju-1.13.0-precise-amd64.tgz", 801 "ftype": "tar.gz", 802 "sha256": "447aeb6a934a5eaec4f703eda4ef2dde" 803 } 804 } 805 } 806 } 807 } 808 }, 809 "format": "products:1.0" 810 } 811 `