github.com/Pankov404/juju@v0.0.0-20150703034450-be266991dceb/environs/tools/simplestreams_test.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package tools_test 5 6 import ( 7 "bytes" 8 "encoding/json" 9 "flag" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "net/http" 14 "os" 15 "path/filepath" 16 "reflect" 17 "runtime" 18 "strings" 19 "testing" 20 21 jc "github.com/juju/testing/checkers" 22 "github.com/juju/utils" 23 "gopkg.in/amz.v3/aws" 24 gc "gopkg.in/check.v1" 25 26 "github.com/juju/juju/environs/filestorage" 27 "github.com/juju/juju/environs/jujutest" 28 "github.com/juju/juju/environs/simplestreams" 29 sstesting "github.com/juju/juju/environs/simplestreams/testing" 30 "github.com/juju/juju/environs/storage" 31 "github.com/juju/juju/environs/tools" 32 toolstesting "github.com/juju/juju/environs/tools/testing" 33 coretesting "github.com/juju/juju/testing" 34 coretools "github.com/juju/juju/tools" 35 "github.com/juju/juju/version" 36 ) 37 38 var live = flag.Bool("live", false, "Include live simplestreams tests") 39 var vendor = flag.String("vendor", "", "The vendor representing the source of the simplestream data") 40 41 type liveTestData struct { 42 baseURL string 43 requireSigned bool 44 validCloudSpec simplestreams.CloudSpec 45 } 46 47 var liveUrls = map[string]liveTestData{ 48 "ec2": { 49 baseURL: tools.DefaultBaseURL, 50 requireSigned: true, 51 validCloudSpec: simplestreams.CloudSpec{"us-east-1", aws.Regions["us-east-1"].EC2Endpoint}, 52 }, 53 "canonistack": { 54 baseURL: "https://swift.canonistack.canonical.com/v1/AUTH_526ad877f3e3464589dc1145dfeaac60/juju-tools", 55 requireSigned: false, 56 validCloudSpec: simplestreams.CloudSpec{"lcy01", "https://keystone.canonistack.canonical.com:443/v1.0/"}, 57 }, 58 } 59 60 func setupSimpleStreamsTests(t *testing.T) { 61 if *live { 62 if *vendor == "" { 63 t.Fatal("missing vendor") 64 } 65 var ok bool 66 var testData liveTestData 67 if testData, ok = liveUrls[*vendor]; !ok { 68 keys := reflect.ValueOf(liveUrls).MapKeys() 69 t.Fatalf("Unknown vendor %s. Must be one of %s", *vendor, keys) 70 } 71 registerLiveSimpleStreamsTests(testData.baseURL, 72 tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 73 CloudSpec: testData.validCloudSpec, 74 Series: []string{version.Current.Series}, 75 Arches: []string{"amd64"}, 76 Stream: "released", 77 }), testData.requireSigned) 78 } 79 registerSimpleStreamsTests() 80 } 81 82 func registerSimpleStreamsTests() { 83 gc.Suite(&simplestreamsSuite{ 84 LocalLiveSimplestreamsSuite: sstesting.LocalLiveSimplestreamsSuite{ 85 Source: simplestreams.NewURLDataSource("test", "test:", utils.VerifySSLHostnames), 86 RequireSigned: false, 87 DataType: tools.ContentDownload, 88 StreamsVersion: tools.CurrentStreamsVersion, 89 ValidConstraint: tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 90 CloudSpec: simplestreams.CloudSpec{ 91 Region: "us-east-1", 92 Endpoint: "https://ec2.us-east-1.amazonaws.com", 93 }, 94 Series: []string{"precise"}, 95 Arches: []string{"amd64", "arm"}, 96 Stream: "released", 97 }), 98 }, 99 }) 100 gc.Suite(&signedSuite{}) 101 } 102 103 func registerLiveSimpleStreamsTests(baseURL string, validToolsConstraint simplestreams.LookupConstraint, requireSigned bool) { 104 gc.Suite(&sstesting.LocalLiveSimplestreamsSuite{ 105 Source: simplestreams.NewURLDataSource("test", baseURL, utils.VerifySSLHostnames), 106 RequireSigned: requireSigned, 107 DataType: tools.ContentDownload, 108 StreamsVersion: tools.CurrentStreamsVersion, 109 ValidConstraint: validToolsConstraint, 110 }) 111 } 112 113 type simplestreamsSuite struct { 114 sstesting.LocalLiveSimplestreamsSuite 115 sstesting.TestDataSuite 116 } 117 118 func (s *simplestreamsSuite) SetUpSuite(c *gc.C) { 119 s.LocalLiveSimplestreamsSuite.SetUpSuite(c) 120 s.TestDataSuite.SetUpSuite(c) 121 } 122 123 func (s *simplestreamsSuite) TearDownSuite(c *gc.C) { 124 s.TestDataSuite.TearDownSuite(c) 125 s.LocalLiveSimplestreamsSuite.TearDownSuite(c) 126 } 127 128 var fetchTests = []struct { 129 region string 130 series string 131 version string 132 stream string 133 major int 134 minor int 135 arches []string 136 tools []*tools.ToolsMetadata 137 }{{ 138 series: "precise", 139 arches: []string{"amd64", "arm"}, 140 version: "1.13.0", 141 tools: []*tools.ToolsMetadata{ 142 { 143 Release: "precise", 144 Version: "1.13.0", 145 Arch: "amd64", 146 Size: 2973595, 147 Path: "tools/released/20130806/juju-1.13.0-precise-amd64.tgz", 148 FileType: "tar.gz", 149 SHA256: "447aeb6a934a5eaec4f703eda4ef2dde", 150 }, 151 }, 152 }, { 153 series: "raring", 154 arches: []string{"amd64", "arm"}, 155 version: "1.13.0", 156 tools: []*tools.ToolsMetadata{ 157 { 158 Release: "raring", 159 Version: "1.13.0", 160 Arch: "amd64", 161 Size: 2973173, 162 Path: "tools/released/20130806/juju-1.13.0-raring-amd64.tgz", 163 FileType: "tar.gz", 164 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 165 }, 166 }, 167 }, { 168 series: "raring", 169 arches: []string{"amd64", "arm"}, 170 version: "1.11.4", 171 tools: []*tools.ToolsMetadata{ 172 { 173 Release: "raring", 174 Version: "1.11.4", 175 Arch: "arm", 176 Size: 1950327, 177 Path: "tools/released/20130806/juju-1.11.4-raring-arm.tgz", 178 FileType: "tar.gz", 179 SHA256: "6472014e3255e3fe7fbd3550ef3f0a11", 180 }, 181 }, 182 }, { 183 series: "precise", 184 arches: []string{"amd64", "arm"}, 185 major: 2, 186 tools: []*tools.ToolsMetadata{ 187 { 188 Release: "precise", 189 Version: "2.0.1", 190 Arch: "arm", 191 Size: 1951096, 192 Path: "tools/released/20130806/juju-2.0.1-precise-arm.tgz", 193 FileType: "tar.gz", 194 SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", 195 }, 196 }, 197 }, { 198 series: "precise", 199 arches: []string{"amd64", "arm"}, 200 major: 1, 201 minor: 11, 202 tools: []*tools.ToolsMetadata{ 203 { 204 Release: "precise", 205 Version: "1.11.4", 206 Arch: "arm", 207 Size: 1951096, 208 Path: "tools/released/20130806/juju-1.11.4-precise-arm.tgz", 209 FileType: "tar.gz", 210 SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", 211 }, 212 { 213 Release: "precise", 214 Version: "1.11.5", 215 Arch: "arm", 216 Size: 2031281, 217 Path: "tools/released/20130803/juju-1.11.5-precise-arm.tgz", 218 FileType: "tar.gz", 219 SHA256: "df07ac5e1fb4232d4e9aa2effa57918a", 220 }, 221 }, 222 }, { 223 series: "trusty", 224 arches: []string{"amd64"}, 225 version: "1.16.0", 226 stream: "testing", 227 tools: []*tools.ToolsMetadata{ 228 { 229 Release: "trusty", 230 Version: "1.16.0", 231 Arch: "amd64", 232 Size: 2973512, 233 Path: "tools/testing/20130806/juju-1.16.0-trusty-amd64.tgz", 234 FileType: "tar.gz", 235 SHA256: "447aeb6a934a5eaec4f703eda4ef2dac", 236 }, 237 }, 238 }} 239 240 func (s *simplestreamsSuite) TestFetch(c *gc.C) { 241 for i, t := range fetchTests { 242 c.Logf("test %d", i) 243 if t.stream == "" { 244 t.stream = "released" 245 } 246 var toolsConstraint *tools.ToolsConstraint 247 if t.version == "" { 248 toolsConstraint = tools.NewGeneralToolsConstraint(t.major, t.minor, simplestreams.LookupParams{ 249 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 250 Series: []string{t.series}, 251 Arches: t.arches, 252 Stream: t.stream, 253 }) 254 } else { 255 toolsConstraint = tools.NewVersionedToolsConstraint(version.MustParse(t.version), 256 simplestreams.LookupParams{ 257 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 258 Series: []string{t.series}, 259 Arches: t.arches, 260 Stream: t.stream, 261 }) 262 } 263 // Add invalid datasource and check later that resolveInfo is correct. 264 invalidSource := simplestreams.NewURLDataSource("invalid", "file://invalid", utils.VerifySSLHostnames) 265 tools, resolveInfo, err := tools.Fetch( 266 []simplestreams.DataSource{invalidSource, s.Source}, toolsConstraint, s.RequireSigned) 267 if !c.Check(err, jc.ErrorIsNil) { 268 continue 269 } 270 for _, tm := range t.tools { 271 tm.FullPath, err = s.Source.URL(tm.Path) 272 c.Assert(err, jc.ErrorIsNil) 273 } 274 c.Check(tools, gc.DeepEquals, t.tools) 275 c.Check(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{ 276 Source: "test", 277 Signed: s.RequireSigned, 278 IndexURL: "test:/streams/v1/index.json", 279 MirrorURL: "", 280 }) 281 } 282 } 283 284 func (s *simplestreamsSuite) TestFetchNoMatchingStream(c *gc.C) { 285 toolsConstraint := tools.NewGeneralToolsConstraint(2, -1, simplestreams.LookupParams{ 286 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 287 Series: []string{"precise"}, 288 Arches: []string{}, 289 Stream: "proposed", 290 }) 291 _, _, err := tools.Fetch( 292 []simplestreams.DataSource{s.Source}, toolsConstraint, s.RequireSigned) 293 c.Assert(err, gc.ErrorMatches, `"content-download" data not found`) 294 } 295 296 func (s *simplestreamsSuite) TestFetchWithMirror(c *gc.C) { 297 toolsConstraint := tools.NewGeneralToolsConstraint(1, 13, simplestreams.LookupParams{ 298 CloudSpec: simplestreams.CloudSpec{"us-west-2", "https://ec2.us-west-2.amazonaws.com"}, 299 Series: []string{"precise"}, 300 Arches: []string{"amd64"}, 301 Stream: "released", 302 }) 303 toolsMetadata, resolveInfo, err := tools.Fetch( 304 []simplestreams.DataSource{s.Source}, toolsConstraint, s.RequireSigned) 305 c.Assert(err, jc.ErrorIsNil) 306 c.Assert(len(toolsMetadata), gc.Equals, 1) 307 308 expectedMetadata := &tools.ToolsMetadata{ 309 Release: "precise", 310 Version: "1.13.0", 311 Arch: "amd64", 312 Size: 2973595, 313 Path: "mirrored-path/juju-1.13.0-precise-amd64.tgz", 314 FullPath: "test:/mirrored-path/juju-1.13.0-precise-amd64.tgz", 315 FileType: "tar.gz", 316 SHA256: "447aeb6a934a5eaec4f703eda4ef2dde", 317 } 318 c.Assert(err, jc.ErrorIsNil) 319 c.Assert(toolsMetadata[0], gc.DeepEquals, expectedMetadata) 320 c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{ 321 Source: "test", 322 Signed: s.RequireSigned, 323 IndexURL: "test:/streams/v1/index.json", 324 MirrorURL: "test:/", 325 }) 326 } 327 328 func assertMetadataMatches(c *gc.C, storageDir string, stream string, toolList coretools.List, metadata []*tools.ToolsMetadata) { 329 var expectedMetadata []*tools.ToolsMetadata = make([]*tools.ToolsMetadata, len(toolList)) 330 for i, tool := range toolList { 331 expectedMetadata[i] = &tools.ToolsMetadata{ 332 Release: tool.Version.Series, 333 Version: tool.Version.Number.String(), 334 Arch: tool.Version.Arch, 335 Size: tool.Size, 336 Path: fmt.Sprintf("%s/juju-%s.tgz", stream, tool.Version.String()), 337 FileType: "tar.gz", 338 SHA256: tool.SHA256, 339 } 340 } 341 c.Assert(metadata, gc.DeepEquals, expectedMetadata) 342 } 343 344 func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) { 345 toolsList := coretools.List{ 346 { 347 Version: version.MustParseBinary("1.2.3-precise-amd64"), 348 Size: 123, 349 SHA256: "abcd", 350 }, { 351 Version: version.MustParseBinary("2.0.1-raring-amd64"), 352 Size: 456, 353 SHA256: "xyz", 354 }, 355 } 356 expected := toolsList 357 358 // Add tools with an unknown series. Do not add an entry in the 359 // expected list as these tools should be ignored. 360 vers, err := version.ParseBinary("3.2.1-xuanhuaceratops-amd64") 361 c.Assert(err, jc.Satisfies, version.IsUnknownOSForSeriesError) 362 toolsList = append(toolsList, &coretools.Tools{ 363 Version: vers, 364 Size: 456, 365 SHA256: "wqe", 366 }) 367 368 dir := c.MkDir() 369 writer, err := filestorage.NewFileStorageWriter(dir) 370 c.Assert(err, jc.ErrorIsNil) 371 err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, tools.DoNotWriteMirrors) 372 c.Assert(err, jc.ErrorIsNil) 373 metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", false) 374 assertMetadataMatches(c, dir, "proposed", expected, metadata) 375 } 376 377 func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) { 378 var versionStrings = []string{ 379 "1.2.3-precise-amd64", 380 "2.0.1-raring-amd64", 381 } 382 dir := c.MkDir() 383 toolstesting.MakeTools(c, dir, "proposed", versionStrings) 384 385 toolsList := coretools.List{ 386 { 387 // If sha256/size is already known, do not recalculate 388 Version: version.MustParseBinary("1.2.3-precise-amd64"), 389 Size: 123, 390 SHA256: "abcd", 391 }, { 392 Version: version.MustParseBinary("2.0.1-raring-amd64"), 393 // The URL is not used for generating metadata. 394 URL: "bogus://", 395 }, 396 } 397 writer, err := filestorage.NewFileStorageWriter(dir) 398 c.Assert(err, jc.ErrorIsNil) 399 writeMirrors := tools.DoNotWriteMirrors 400 if withMirrors { 401 writeMirrors = tools.WriteMirrors 402 } 403 err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, writeMirrors) 404 c.Assert(err, jc.ErrorIsNil) 405 metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", withMirrors) 406 assertMetadataMatches(c, dir, "proposed", toolsList, metadata) 407 408 // No release stream generated so there will not be a legacy index file created. 409 _, err = writer.Get("tools/streams/v1/index.json") 410 c.Assert(err, gc.NotNil) 411 } 412 413 func (s *simplestreamsSuite) TestWriteMetadata(c *gc.C) { 414 s.assertWriteMetadata(c, false) 415 } 416 417 func (s *simplestreamsSuite) TestWriteMetadataWithMirrors(c *gc.C) { 418 s.assertWriteMetadata(c, true) 419 } 420 421 func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) { 422 dir := c.MkDir() 423 existingToolsList := coretools.List{ 424 { 425 Version: version.MustParseBinary("1.2.3-precise-amd64"), 426 Size: 123, 427 SHA256: "abc", 428 }, { 429 Version: version.MustParseBinary("2.0.1-raring-amd64"), 430 Size: 456, 431 SHA256: "xyz", 432 }, 433 } 434 writer, err := filestorage.NewFileStorageWriter(dir) 435 c.Assert(err, jc.ErrorIsNil) 436 err = tools.MergeAndWriteMetadata(writer, "testing", "testing", existingToolsList, tools.WriteMirrors) 437 c.Assert(err, jc.ErrorIsNil) 438 newToolsList := coretools.List{ 439 existingToolsList[0], 440 { 441 Version: version.MustParseBinary("2.1.0-raring-amd64"), 442 Size: 789, 443 SHA256: "def", 444 }, 445 } 446 err = tools.MergeAndWriteMetadata(writer, "testing", "testing", newToolsList, tools.WriteMirrors) 447 c.Assert(err, jc.ErrorIsNil) 448 requiredToolsList := append(existingToolsList, newToolsList[1]) 449 metadata := toolstesting.ParseMetadataFromDir(c, dir, "testing", true) 450 assertMetadataMatches(c, dir, "testing", requiredToolsList, metadata) 451 452 err = tools.MergeAndWriteMetadata(writer, "devel", "devel", newToolsList, tools.WriteMirrors) 453 c.Assert(err, jc.ErrorIsNil) 454 metadata = toolstesting.ParseMetadataFromDir(c, dir, "testing", true) 455 assertMetadataMatches(c, dir, "testing", requiredToolsList, metadata) 456 metadata = toolstesting.ParseMetadataFromDir(c, dir, "devel", true) 457 assertMetadataMatches(c, dir, "devel", newToolsList, metadata) 458 } 459 460 type productSpecSuite struct{} 461 462 var _ = gc.Suite(&productSpecSuite{}) 463 464 func (s *productSpecSuite) TestIndexIdNoStream(c *gc.C) { 465 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 466 Series: []string{"precise"}, 467 Arches: []string{"amd64"}, 468 }) 469 ids := toolsConstraint.IndexIds() 470 c.Assert(ids, gc.HasLen, 0) 471 } 472 473 func (s *productSpecSuite) TestIndexId(c *gc.C) { 474 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 475 Series: []string{"precise"}, 476 Arches: []string{"amd64"}, 477 Stream: "proposed", 478 }) 479 ids := toolsConstraint.IndexIds() 480 c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.juju:proposed:tools"}) 481 } 482 483 func (s *productSpecSuite) TestProductId(c *gc.C) { 484 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 485 Series: []string{"precise"}, 486 Arches: []string{"amd64"}, 487 }) 488 ids, err := toolsConstraint.ProductIds() 489 c.Assert(err, jc.ErrorIsNil) 490 c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.juju:12.04:amd64"}) 491 } 492 493 func (s *productSpecSuite) TestIdMultiArch(c *gc.C) { 494 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.11.3"), simplestreams.LookupParams{ 495 Series: []string{"precise"}, 496 Arches: []string{"amd64", "arm"}, 497 }) 498 ids, err := toolsConstraint.ProductIds() 499 c.Assert(err, jc.ErrorIsNil) 500 c.Assert(ids, gc.DeepEquals, []string{ 501 "com.ubuntu.juju:12.04:amd64", 502 "com.ubuntu.juju:12.04:arm"}) 503 } 504 505 func (s *productSpecSuite) TestIdMultiSeries(c *gc.C) { 506 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.11.3"), simplestreams.LookupParams{ 507 Series: []string{"precise", "raring"}, 508 Arches: []string{"amd64"}, 509 Stream: "released", 510 }) 511 ids, err := toolsConstraint.ProductIds() 512 c.Assert(err, jc.ErrorIsNil) 513 c.Assert(ids, gc.DeepEquals, []string{ 514 "com.ubuntu.juju:12.04:amd64", 515 "com.ubuntu.juju:13.04:amd64"}) 516 } 517 518 func (s *productSpecSuite) TestIdWithMajorVersionOnly(c *gc.C) { 519 toolsConstraint := tools.NewGeneralToolsConstraint(1, -1, simplestreams.LookupParams{ 520 Series: []string{"precise"}, 521 Arches: []string{"amd64"}, 522 Stream: "released", 523 }) 524 ids, err := toolsConstraint.ProductIds() 525 c.Assert(err, jc.ErrorIsNil) 526 c.Assert(ids, gc.DeepEquals, []string{`com.ubuntu.juju:12.04:amd64`}) 527 } 528 529 func (s *productSpecSuite) TestIdWithMajorMinorVersion(c *gc.C) { 530 toolsConstraint := tools.NewGeneralToolsConstraint(1, 2, simplestreams.LookupParams{ 531 Series: []string{"precise"}, 532 Arches: []string{"amd64"}, 533 Stream: "released", 534 }) 535 ids, err := toolsConstraint.ProductIds() 536 c.Assert(err, jc.ErrorIsNil) 537 c.Assert(ids, gc.DeepEquals, []string{`com.ubuntu.juju:12.04:amd64`}) 538 } 539 540 func (s *productSpecSuite) TestLargeNumber(c *gc.C) { 541 json := `{ 542 "updated": "Fri, 30 Aug 2013 16:12:58 +0800", 543 "format": "products:1.0", 544 "products": { 545 "com.ubuntu.juju:1.10.0:amd64": { 546 "version": "1.10.0", 547 "arch": "amd64", 548 "versions": { 549 "20133008": { 550 "items": { 551 "1.10.0-precise-amd64": { 552 "release": "precise", 553 "version": "1.10.0", 554 "arch": "amd64", 555 "size": 9223372036854775807, 556 "path": "releases/juju-1.10.0-precise-amd64.tgz", 557 "ftype": "tar.gz", 558 "sha256": "" 559 } 560 } 561 } 562 } 563 } 564 } 565 }` 566 cloudMetadata, err := simplestreams.ParseCloudMetadata([]byte(json), "products:1.0", "", tools.ToolsMetadata{}) 567 c.Assert(err, jc.ErrorIsNil) 568 c.Assert(cloudMetadata.Products, gc.HasLen, 1) 569 product := cloudMetadata.Products["com.ubuntu.juju:1.10.0:amd64"] 570 c.Assert(product, gc.NotNil) 571 c.Assert(product.Items, gc.HasLen, 1) 572 version := product.Items["20133008"] 573 c.Assert(version, gc.NotNil) 574 c.Assert(version.Items, gc.HasLen, 1) 575 item := version.Items["1.10.0-precise-amd64"] 576 c.Assert(item, gc.NotNil) 577 c.Assert(item, gc.FitsTypeOf, &tools.ToolsMetadata{}) 578 c.Assert(item.(*tools.ToolsMetadata).Size, gc.Equals, int64(9223372036854775807)) 579 } 580 581 type metadataHelperSuite struct { 582 coretesting.BaseSuite 583 } 584 585 var _ = gc.Suite(&metadataHelperSuite{}) 586 587 func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) { 588 metadata := tools.MetadataFromTools(nil, "proposed") 589 c.Assert(metadata, gc.HasLen, 0) 590 591 toolsList := coretools.List{{ 592 Version: version.MustParseBinary("1.2.3-precise-amd64"), 593 Size: 123, 594 SHA256: "abc", 595 }, { 596 Version: version.MustParseBinary("2.0.1-raring-amd64"), 597 URL: "file:///tmp/proposed/juju-2.0.1-raring-amd64.tgz", 598 Size: 456, 599 SHA256: "xyz", 600 }} 601 metadata = tools.MetadataFromTools(toolsList, "proposed") 602 c.Assert(metadata, gc.HasLen, len(toolsList)) 603 for i, t := range toolsList { 604 md := metadata[i] 605 c.Assert(md.Release, gc.Equals, t.Version.Series) 606 c.Assert(md.Version, gc.Equals, t.Version.Number.String()) 607 c.Assert(md.Arch, gc.Equals, t.Version.Arch) 608 // FullPath is only filled out when reading tools using simplestreams. 609 // It's not needed elsewhere and requires a URL() call. 610 c.Assert(md.FullPath, gc.Equals, "") 611 c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version, "proposed")[len("tools/"):]) 612 c.Assert(md.FileType, gc.Equals, "tar.gz") 613 c.Assert(md.Size, gc.Equals, t.Size) 614 c.Assert(md.SHA256, gc.Equals, t.SHA256) 615 } 616 } 617 618 type countingStorage struct { 619 storage.StorageReader 620 counter int 621 } 622 623 func (c *countingStorage) Get(name string) (io.ReadCloser, error) { 624 c.counter++ 625 return c.StorageReader.Get(name) 626 } 627 628 func (*metadataHelperSuite) TestResolveMetadata(c *gc.C) { 629 var versionStrings = []string{"1.2.3-precise-amd64"} 630 dir := c.MkDir() 631 toolstesting.MakeTools(c, dir, "released", versionStrings) 632 toolsList := coretools.List{{ 633 Version: version.MustParseBinary(versionStrings[0]), 634 Size: 123, 635 SHA256: "abc", 636 }} 637 638 stor, err := filestorage.NewFileStorageReader(dir) 639 c.Assert(err, jc.ErrorIsNil) 640 err = tools.ResolveMetadata(stor, "released", nil) 641 c.Assert(err, jc.ErrorIsNil) 642 643 // We already have size/sha256, so ensure that storage isn't consulted. 644 countingStorage := &countingStorage{StorageReader: stor} 645 metadata := tools.MetadataFromTools(toolsList, "released") 646 err = tools.ResolveMetadata(countingStorage, "released", metadata) 647 c.Assert(err, jc.ErrorIsNil) 648 c.Assert(countingStorage.counter, gc.Equals, 0) 649 650 // Now clear size/sha256, and check that it is called, and 651 // the size/sha256 sum are updated. 652 metadata[0].Size = 0 653 metadata[0].SHA256 = "" 654 err = tools.ResolveMetadata(countingStorage, "released", metadata) 655 c.Assert(err, jc.ErrorIsNil) 656 c.Assert(countingStorage.counter, gc.Equals, 1) 657 c.Assert(metadata[0].Size, gc.Not(gc.Equals), 0) 658 c.Assert(metadata[0].SHA256, gc.Not(gc.Equals), "") 659 } 660 661 func (*metadataHelperSuite) TestResolveMetadataLegacyPPC64(c *gc.C) { 662 var versionStrings = []string{"1.2.3-precise-amd64", "1.2.3-precise-ppc64el"} 663 dir := c.MkDir() 664 toolstesting.MakeTools(c, dir, "released", versionStrings) 665 666 toolsList := coretools.List{ 667 { 668 Version: version.MustParseBinary(versionStrings[0]), 669 }, { 670 Version: version.MustParseBinary(versionStrings[1]), 671 }, { 672 Version: version.MustParseBinary("1.2.3-precise-ppc64"), 673 }, 674 } 675 toolsMetadata := tools.MetadataFromTools(toolsList, dir) 676 stor, err := filestorage.NewFileStorageReader(dir) 677 c.Assert(err, jc.ErrorIsNil) 678 err = tools.ResolveMetadata(stor, "released", toolsMetadata) 679 c.Assert(err, jc.ErrorIsNil) 680 c.Assert(toolsMetadata, gc.DeepEquals, []*tools.ToolsMetadata{ 681 { 682 Release: "precise", 683 Version: "1.2.3", 684 Arch: "amd64", 685 Size: 19, 686 FileType: "tar.gz", 687 SHA256: "dcdd65b962b804a3d63b108d670290ee95a867a97fe9b9f99b2b77b5c7173e59", 688 Path: fmt.Sprintf("%s/juju-1.2.3-precise-amd64.tgz", dir), 689 }, 690 { 691 Release: "precise", 692 Version: "1.2.3", 693 Arch: "ppc64el", 694 Size: 21, 695 FileType: "tar.gz", 696 SHA256: "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2", 697 Path: fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir), 698 }, 699 { 700 Release: "precise", 701 Version: "1.2.3", 702 Arch: "ppc64", 703 Size: 21, 704 FileType: "tar.gz", 705 SHA256: "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2", 706 Path: fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir), 707 }, 708 }) 709 } 710 711 func (*metadataHelperSuite) TestMergeMetadata(c *gc.C) { 712 md1 := &tools.ToolsMetadata{ 713 Release: "precise", 714 Version: "1.2.3", 715 Arch: "amd64", 716 Path: "path1", 717 } 718 md2 := &tools.ToolsMetadata{ 719 Release: "precise", 720 Version: "1.2.3", 721 Arch: "amd64", 722 Path: "path2", 723 } 724 md3 := &tools.ToolsMetadata{ 725 Release: "raring", 726 Version: "1.2.3", 727 Arch: "amd64", 728 Path: "path3", 729 } 730 731 withSize := func(md *tools.ToolsMetadata, size int64) *tools.ToolsMetadata { 732 clone := *md 733 clone.Size = size 734 return &clone 735 } 736 withSHA256 := func(md *tools.ToolsMetadata, sha256 string) *tools.ToolsMetadata { 737 clone := *md 738 clone.SHA256 = sha256 739 return &clone 740 } 741 742 type mdlist []*tools.ToolsMetadata 743 type test struct { 744 name string 745 lhs, rhs, merged []*tools.ToolsMetadata 746 err string 747 } 748 tests := []test{{ 749 name: "non-empty lhs, empty rhs", 750 lhs: mdlist{md1}, 751 rhs: nil, 752 merged: mdlist{md1}, 753 }, { 754 name: "empty lhs, non-empty rhs", 755 lhs: nil, 756 rhs: mdlist{md2}, 757 merged: mdlist{md2}, 758 }, { 759 name: "identical lhs, rhs", 760 lhs: mdlist{md1}, 761 rhs: mdlist{md1}, 762 merged: mdlist{md1}, 763 }, { 764 name: "same tools in lhs and rhs, neither have size: prefer lhs", 765 lhs: mdlist{md1}, 766 rhs: mdlist{md2}, 767 merged: mdlist{md1}, 768 }, { 769 name: "same tools in lhs and rhs, only lhs has a size: prefer lhs", 770 lhs: mdlist{withSize(md1, 123)}, 771 rhs: mdlist{md2}, 772 merged: mdlist{withSize(md1, 123)}, 773 }, { 774 name: "same tools in lhs and rhs, only rhs has a size: prefer rhs", 775 lhs: mdlist{md1}, 776 rhs: mdlist{withSize(md2, 123)}, 777 merged: mdlist{withSize(md2, 123)}, 778 }, { 779 name: "same tools in lhs and rhs, both have the same size: prefer lhs", 780 lhs: mdlist{withSize(md1, 123)}, 781 rhs: mdlist{withSize(md2, 123)}, 782 merged: mdlist{withSize(md1, 123)}, 783 }, { 784 name: "same tools in lhs and rhs, both have different sizes: error", 785 lhs: mdlist{withSize(md1, 123)}, 786 rhs: mdlist{withSize(md2, 456)}, 787 err: "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,456\\) sha256=\\(,\\)", 788 }, { 789 name: "same tools in lhs and rhs, both have same size but different sha256: error", 790 lhs: mdlist{withSHA256(withSize(md1, 123), "a")}, 791 rhs: mdlist{withSHA256(withSize(md2, 123), "b")}, 792 err: "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,123\\) sha256=\\(a,b\\)", 793 }, { 794 name: "lhs is a proper superset of rhs: union of lhs and rhs", 795 lhs: mdlist{md1, md3}, 796 rhs: mdlist{md1}, 797 merged: mdlist{md1, md3}, 798 }, { 799 name: "rhs is a proper superset of lhs: union of lhs and rhs", 800 lhs: mdlist{md1}, 801 rhs: mdlist{md1, md3}, 802 merged: mdlist{md1, md3}, 803 }} 804 for i, test := range tests { 805 c.Logf("test %d: %s", i, test.name) 806 merged, err := tools.MergeMetadata(test.lhs, test.rhs) 807 if test.err == "" { 808 c.Assert(err, jc.ErrorIsNil) 809 c.Assert(merged, gc.DeepEquals, test.merged) 810 } else { 811 c.Assert(err, gc.ErrorMatches, test.err) 812 c.Assert(merged, gc.IsNil) 813 } 814 } 815 } 816 817 func (*metadataHelperSuite) TestReadWriteMetadataSingleStream(c *gc.C) { 818 metadata := map[string][]*tools.ToolsMetadata{ 819 "released": {{ 820 Release: "precise", 821 Version: "1.2.3", 822 Arch: "amd64", 823 Path: "path1", 824 }, { 825 Release: "raring", 826 Version: "1.2.3", 827 Arch: "amd64", 828 Path: "path2", 829 }}, 830 } 831 832 stor, err := filestorage.NewFileStorageWriter(c.MkDir()) 833 c.Assert(err, jc.ErrorIsNil) 834 out, err := tools.ReadAllMetadata(stor) 835 c.Assert(err, jc.ErrorIsNil) // non-existence is not an error 836 c.Assert(out, gc.HasLen, 0) 837 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 838 c.Assert(err, jc.ErrorIsNil) 839 840 // Read back what was just written. 841 out, err = tools.ReadAllMetadata(stor) 842 c.Assert(err, jc.ErrorIsNil) 843 for _, outMetadata := range out { 844 for _, md := range outMetadata { 845 // FullPath is set by ReadAllMetadata. 846 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 847 md.FullPath = "" 848 } 849 } 850 c.Assert(out, jc.DeepEquals, metadata) 851 } 852 853 func (*metadataHelperSuite) writeMetadataMultipleStream(c *gc.C) (storage.StorageReader, map[string][]*tools.ToolsMetadata) { 854 metadata := map[string][]*tools.ToolsMetadata{ 855 "released": {{ 856 Release: "precise", 857 Version: "1.2.3", 858 Arch: "amd64", 859 Path: "path1", 860 }}, 861 "proposed": {{ 862 Release: "raring", 863 Version: "1.2.3", 864 Arch: "amd64", 865 Path: "path2", 866 }}, 867 } 868 869 stor, err := filestorage.NewFileStorageWriter(c.MkDir()) 870 c.Assert(err, jc.ErrorIsNil) 871 out, err := tools.ReadAllMetadata(stor) 872 c.Assert(out, gc.HasLen, 0) 873 c.Assert(err, jc.ErrorIsNil) // non-existence is not an error 874 err = tools.WriteMetadata(stor, metadata, []string{"released", "proposed"}, tools.DoNotWriteMirrors) 875 c.Assert(err, jc.ErrorIsNil) 876 return stor, metadata 877 } 878 879 func (s *metadataHelperSuite) TestReadWriteMetadataMultipleStream(c *gc.C) { 880 stor, metadata := s.writeMetadataMultipleStream(c) 881 // Read back what was just written. 882 out, err := tools.ReadAllMetadata(stor) 883 c.Assert(err, jc.ErrorIsNil) 884 for _, outMetadata := range out { 885 for _, md := range outMetadata { 886 // FullPath is set by ReadAllMetadata. 887 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 888 md.FullPath = "" 889 } 890 } 891 c.Assert(out, jc.DeepEquals, metadata) 892 } 893 894 func (s *metadataHelperSuite) TestWriteMetadataLegacyIndex(c *gc.C) { 895 stor, _ := s.writeMetadataMultipleStream(c) 896 // Read back the legacy index 897 rdr, err := stor.Get("tools/streams/v1/index.json") 898 defer rdr.Close() 899 c.Assert(err, jc.ErrorIsNil) 900 data, err := ioutil.ReadAll(rdr) 901 c.Assert(err, jc.ErrorIsNil) 902 var indices simplestreams.Indices 903 err = json.Unmarshal(data, &indices) 904 c.Assert(err, jc.ErrorIsNil) 905 c.Assert(indices.Indexes, gc.HasLen, 1) 906 indices.Updated = "" 907 c.Assert(indices.Indexes["com.ubuntu.juju:released:tools"], gc.NotNil) 908 indices.Indexes["com.ubuntu.juju:released:tools"].Updated = "" 909 expected := simplestreams.Indices{ 910 Format: "index:1.0", 911 Indexes: map[string]*simplestreams.IndexMetadata{ 912 "com.ubuntu.juju:released:tools": { 913 Format: "products:1.0", 914 DataType: "content-download", 915 ProductsFilePath: "streams/v1/com.ubuntu.juju-released-tools.json", 916 ProductIds: []string{"com.ubuntu.juju:12.04:amd64"}, 917 }, 918 }, 919 } 920 c.Assert(indices, jc.DeepEquals, expected) 921 } 922 923 func (s *metadataHelperSuite) TestReadWriteMetadataUnchanged(c *gc.C) { 924 metadata := map[string][]*tools.ToolsMetadata{ 925 "released": {{ 926 Release: "precise", 927 Version: "1.2.3", 928 Arch: "amd64", 929 Path: "path1", 930 }, { 931 Release: "raring", 932 Version: "1.2.3", 933 Arch: "amd64", 934 Path: "path2", 935 }}, 936 } 937 938 stor, err := filestorage.NewFileStorageWriter(c.MkDir()) 939 c.Assert(err, jc.ErrorIsNil) 940 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 941 c.Assert(err, jc.ErrorIsNil) 942 943 s.PatchValue(tools.WriteMetadataFiles, func(stor storage.Storage, metadataInfo []tools.MetadataFile) error { 944 // The product data is the same, we only write the indices. 945 c.Assert(metadataInfo, gc.HasLen, 2) 946 c.Assert(metadataInfo[0].Path, gc.Equals, "streams/v1/index2.json") 947 c.Assert(metadataInfo[1].Path, gc.Equals, "streams/v1/index.json") 948 return nil 949 }) 950 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 951 c.Assert(err, jc.ErrorIsNil) 952 } 953 954 func (*metadataHelperSuite) TestReadMetadataPrefersNewIndex(c *gc.C) { 955 if runtime.GOOS == "windows" { 956 c.Skip("Skipped for now because of introduced regression") 957 } 958 metadataDir := c.MkDir() 959 960 // Generate metadata and rename index to index.json 961 metadata := map[string][]*tools.ToolsMetadata{ 962 "proposed": {{ 963 Release: "precise", 964 Version: "1.2.3", 965 Arch: "amd64", 966 Path: "path1", 967 }}, 968 "released": {{ 969 Release: "trusty", 970 Version: "1.2.3", 971 Arch: "amd64", 972 Path: "path1", 973 }}, 974 } 975 stor, err := filestorage.NewFileStorageWriter(metadataDir) 976 c.Assert(err, jc.ErrorIsNil) 977 err = tools.WriteMetadata(stor, metadata, []string{"proposed", "released"}, tools.DoNotWriteMirrors) 978 c.Assert(err, jc.ErrorIsNil) 979 err = os.Rename( 980 filepath.Join(metadataDir, "tools", "streams", "v1", "index2.json"), 981 filepath.Join(metadataDir, "tools", "streams", "v1", "index.json"), 982 ) 983 c.Assert(err, jc.ErrorIsNil) 984 985 // Generate different metadata with index2.json 986 metadata = map[string][]*tools.ToolsMetadata{ 987 "released": {{ 988 Release: "precise", 989 Version: "1.2.3", 990 Arch: "amd64", 991 Path: "path1", 992 }}, 993 } 994 err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors) 995 c.Assert(err, jc.ErrorIsNil) 996 997 // Read back all metadata, expecting to find metadata in index2.json. 998 out, err := tools.ReadAllMetadata(stor) 999 for _, outMetadata := range out { 1000 for _, md := range outMetadata { 1001 // FullPath is set by ReadAllMetadata. 1002 c.Assert(md.FullPath, gc.Not(gc.Equals), "") 1003 md.FullPath = "" 1004 } 1005 } 1006 c.Assert(out, jc.DeepEquals, metadata) 1007 } 1008 1009 type signedSuite struct { 1010 origKey string 1011 } 1012 1013 var testRoundTripper *jujutest.ProxyRoundTripper 1014 1015 func init() { 1016 testRoundTripper = &jujutest.ProxyRoundTripper{} 1017 testRoundTripper.RegisterForScheme("signedtest") 1018 } 1019 1020 func (s *signedSuite) SetUpSuite(c *gc.C) { 1021 var imageData = map[string]string{ 1022 "/unsigned/streams/v1/index.json": unsignedIndex, 1023 "/unsigned/streams/v1/tools_metadata.json": unsignedProduct, 1024 } 1025 1026 // Set up some signed data from the unsigned data. 1027 // Overwrite the product path to use the sjson suffix. 1028 rawUnsignedIndex := strings.Replace( 1029 unsignedIndex, "streams/v1/tools_metadata.json", "streams/v1/tools_metadata.sjson", -1) 1030 r := bytes.NewReader([]byte(rawUnsignedIndex)) 1031 signedData, err := simplestreams.Encode( 1032 r, sstesting.SignedMetadataPrivateKey, sstesting.PrivateKeyPassphrase) 1033 c.Assert(err, jc.ErrorIsNil) 1034 imageData["/signed/streams/v1/index.sjson"] = string(signedData) 1035 1036 // Replace the tools path in the unsigned data with a different one so we can test that the right 1037 // tools path is used. 1038 rawUnsignedProduct := strings.Replace( 1039 unsignedProduct, "juju-1.13.0", "juju-1.13.1", -1) 1040 r = bytes.NewReader([]byte(rawUnsignedProduct)) 1041 signedData, err = simplestreams.Encode( 1042 r, sstesting.SignedMetadataPrivateKey, sstesting.PrivateKeyPassphrase) 1043 c.Assert(err, jc.ErrorIsNil) 1044 imageData["/signed/streams/v1/tools_metadata.sjson"] = string(signedData) 1045 testRoundTripper.Sub = jujutest.NewCannedRoundTripper( 1046 imageData, map[string]int{"signedtest://unauth": http.StatusUnauthorized}) 1047 s.origKey = tools.SetSigningPublicKey(sstesting.SignedMetadataPublicKey) 1048 } 1049 1050 func (s *signedSuite) TearDownSuite(c *gc.C) { 1051 testRoundTripper.Sub = nil 1052 tools.SetSigningPublicKey(s.origKey) 1053 } 1054 1055 func (s *signedSuite) TestSignedToolsMetadata(c *gc.C) { 1056 signedSource := simplestreams.NewURLDataSource("test", "signedtest://host/signed", utils.VerifySSLHostnames) 1057 toolsConstraint := tools.NewVersionedToolsConstraint(version.MustParse("1.13.0"), simplestreams.LookupParams{ 1058 CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"}, 1059 Series: []string{"precise"}, 1060 Arches: []string{"amd64"}, 1061 Stream: "released", 1062 }) 1063 toolsMetadata, resolveInfo, err := tools.Fetch( 1064 []simplestreams.DataSource{signedSource}, toolsConstraint, true) 1065 c.Assert(err, jc.ErrorIsNil) 1066 c.Assert(len(toolsMetadata), gc.Equals, 1) 1067 c.Assert(toolsMetadata[0].Path, gc.Equals, "tools/releases/20130806/juju-1.13.1-precise-amd64.tgz") 1068 c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{ 1069 Source: "test", 1070 Signed: true, 1071 IndexURL: "signedtest://host/signed/streams/v1/index.sjson", 1072 MirrorURL: "", 1073 }) 1074 } 1075 1076 var unsignedIndex = ` 1077 { 1078 "index": { 1079 "com.ubuntu.juju:released:tools": { 1080 "updated": "Mon, 05 Aug 2013 11:07:04 +0000", 1081 "datatype": "content-download", 1082 "format": "products:1.0", 1083 "products": [ 1084 "com.ubuntu.juju:12.04:amd64" 1085 ], 1086 "path": "streams/v1/tools_metadata.json" 1087 } 1088 }, 1089 "updated": "Wed, 01 May 2013 13:31:26 +0000", 1090 "format": "index:1.0" 1091 } 1092 ` 1093 var unsignedProduct = ` 1094 { 1095 "updated": "Wed, 01 May 2013 13:31:26 +0000", 1096 "content_id": "com.ubuntu.cloud:released:aws", 1097 "datatype": "content-download", 1098 "products": { 1099 "com.ubuntu.juju:12.04:amd64": { 1100 "arch": "amd64", 1101 "release": "precise", 1102 "versions": { 1103 "20130806": { 1104 "items": { 1105 "1130preciseamd64": { 1106 "version": "1.13.0", 1107 "size": 2973595, 1108 "path": "tools/releases/20130806/juju-1.13.0-precise-amd64.tgz", 1109 "ftype": "tar.gz", 1110 "sha256": "447aeb6a934a5eaec4f703eda4ef2dde" 1111 } 1112 } 1113 } 1114 } 1115 } 1116 }, 1117 "format": "products:1.0" 1118 } 1119 `