launchpad.net/~rogpeppe/juju-core/500-errgo-fix@v0.0.0-20140213181702-000000002356/provider/ec2/local_test.go (about) 1 // Copyright 2011, 2012, 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package ec2_test 5 6 import ( 7 "fmt" 8 "regexp" 9 "sort" 10 "strings" 11 12 "launchpad.net/goamz/aws" 13 amzec2 "launchpad.net/goamz/ec2" 14 "launchpad.net/goamz/ec2/ec2test" 15 "launchpad.net/goamz/s3" 16 "launchpad.net/goamz/s3/s3test" 17 gc "launchpad.net/gocheck" 18 "launchpad.net/goyaml" 19 20 "launchpad.net/juju-core/constraints" 21 "launchpad.net/juju-core/environs" 22 "launchpad.net/juju-core/environs/bootstrap" 23 "launchpad.net/juju-core/environs/config" 24 "launchpad.net/juju-core/environs/configstore" 25 "launchpad.net/juju-core/environs/imagemetadata" 26 "launchpad.net/juju-core/environs/jujutest" 27 "launchpad.net/juju-core/environs/simplestreams" 28 envtesting "launchpad.net/juju-core/environs/testing" 29 "launchpad.net/juju-core/environs/tools" 30 "launchpad.net/juju-core/instance" 31 "launchpad.net/juju-core/juju/testing" 32 "launchpad.net/juju-core/provider/ec2" 33 coretesting "launchpad.net/juju-core/testing" 34 jc "launchpad.net/juju-core/testing/checkers" 35 "launchpad.net/juju-core/testing/testbase" 36 "launchpad.net/juju-core/utils" 37 "launchpad.net/juju-core/utils/ssh" 38 "launchpad.net/juju-core/version" 39 ) 40 41 type ProviderSuite struct { 42 testbase.LoggingSuite 43 } 44 45 var _ = gc.Suite(&ProviderSuite{}) 46 47 func (s *ProviderSuite) TestMetadata(c *gc.C) { 48 metadataContent := map[string]string{ 49 "/2011-01-01/meta-data/public-hostname": "public.dummy.address.invalid", 50 "/2011-01-01/meta-data/local-hostname": "private.dummy.address.invalid", 51 } 52 ec2.UseTestMetadata(metadataContent) 53 defer ec2.UseTestMetadata(nil) 54 55 p, err := environs.Provider("ec2") 56 c.Assert(err, gc.IsNil) 57 58 addr, err := p.PublicAddress() 59 c.Assert(err, gc.IsNil) 60 c.Assert(addr, gc.Equals, "public.dummy.address.invalid") 61 62 addr, err = p.PrivateAddress() 63 c.Assert(err, gc.IsNil) 64 c.Assert(addr, gc.Equals, "private.dummy.address.invalid") 65 } 66 67 func (t *ProviderSuite) assertGetImageMetadataSources(c *gc.C, stream, officialSourcePath string) { 68 // Make an env configured with the stream. 69 envAttrs := localConfigAttrs 70 if stream != "" { 71 envAttrs = envAttrs.Merge(coretesting.Attrs{ 72 "image-stream": stream, 73 }) 74 } 75 cfg, err := config.New(config.NoDefaults, envAttrs) 76 c.Assert(err, gc.IsNil) 77 env, err := environs.Prepare(cfg, configstore.NewMem()) 78 c.Assert(err, gc.IsNil) 79 c.Assert(env, gc.NotNil) 80 81 sources, err := imagemetadata.GetMetadataSources(env) 82 c.Assert(err, gc.IsNil) 83 c.Assert(len(sources), gc.Equals, 2) 84 var urls = make([]string, len(sources)) 85 for i, source := range sources { 86 url, err := source.URL("") 87 c.Assert(err, gc.IsNil) 88 urls[i] = url 89 } 90 // The control bucket URL contains the bucket name. 91 c.Check(strings.Contains(urls[0], ec2.ControlBucketName(env)+"/images"), jc.IsTrue) 92 c.Assert(urls[1], gc.Equals, fmt.Sprintf("http://cloud-images.ubuntu.com/%s/", officialSourcePath)) 93 } 94 95 func (t *ProviderSuite) TestGetImageMetadataSources(c *gc.C) { 96 t.assertGetImageMetadataSources(c, "", "releases") 97 t.assertGetImageMetadataSources(c, "released", "releases") 98 t.assertGetImageMetadataSources(c, "daily", "daily") 99 } 100 101 var localConfigAttrs = coretesting.FakeConfig().Merge(coretesting.Attrs{ 102 "name": "sample", 103 "type": "ec2", 104 "region": "test", 105 "control-bucket": "test-bucket", 106 "access-key": "x", 107 "secret-key": "x", 108 "agent-version": version.Current.Number.String(), 109 }) 110 111 func registerLocalTests() { 112 // N.B. Make sure the region we use here 113 // has entries in the images/query txt files. 114 aws.Regions["test"] = aws.Region{ 115 Name: "test", 116 } 117 118 gc.Suite(&localServerSuite{}) 119 gc.Suite(&localLiveSuite{}) 120 gc.Suite(&localNonUSEastSuite{}) 121 } 122 123 // localLiveSuite runs tests from LiveTests using a fake 124 // EC2 server that runs within the test process itself. 125 type localLiveSuite struct { 126 LiveTests 127 srv localServer 128 restoreEC2Patching func() 129 } 130 131 func (t *localLiveSuite) SetUpSuite(c *gc.C) { 132 t.TestConfig = localConfigAttrs 133 t.restoreEC2Patching = patchEC2ForTesting() 134 t.srv.startServer(c) 135 t.LiveTests.SetUpSuite(c) 136 } 137 138 func (t *localLiveSuite) TearDownSuite(c *gc.C) { 139 t.LiveTests.TearDownSuite(c) 140 t.srv.stopServer(c) 141 t.restoreEC2Patching() 142 } 143 144 func (t *LiveTests) TestStartInstanceOnUnknownPlatform(c *gc.C) { 145 c.Skip("broken under ec2 - see https://bugs.launchpad.net/juju-core/+bug/1233278") 146 } 147 148 // localServer represents a fake EC2 server running within 149 // the test process itself. 150 type localServer struct { 151 ec2srv *ec2test.Server 152 s3srv *s3test.Server 153 config *s3test.Config 154 } 155 156 func (srv *localServer) startServer(c *gc.C) { 157 var err error 158 srv.ec2srv, err = ec2test.NewServer() 159 if err != nil { 160 c.Fatalf("cannot start ec2 test server: %v", err) 161 } 162 srv.s3srv, err = s3test.NewServer(srv.config) 163 if err != nil { 164 c.Fatalf("cannot start s3 test server: %v", err) 165 } 166 aws.Regions["test"] = aws.Region{ 167 Name: "test", 168 EC2Endpoint: srv.ec2srv.URL(), 169 S3Endpoint: srv.s3srv.URL(), 170 S3LocationConstraint: true, 171 } 172 s3inst := s3.New(aws.Auth{}, aws.Regions["test"]) 173 storage := ec2.BucketStorage(s3inst.Bucket("juju-dist")) 174 envtesting.UploadFakeTools(c, storage) 175 srv.addSpice(c) 176 } 177 178 // addSpice adds some "spice" to the local server 179 // by adding state that may cause tests to fail. 180 func (srv *localServer) addSpice(c *gc.C) { 181 states := []amzec2.InstanceState{ 182 ec2test.ShuttingDown, 183 ec2test.Terminated, 184 ec2test.Stopped, 185 } 186 for _, state := range states { 187 srv.ec2srv.NewInstances(1, "m1.small", "ami-a7f539ce", state, nil) 188 } 189 } 190 191 func (srv *localServer) stopServer(c *gc.C) { 192 srv.ec2srv.Quit() 193 srv.s3srv.Quit() 194 // Clear out the region because the server address is 195 // no longer valid. 196 delete(aws.Regions, "test") 197 } 198 199 // localServerSuite contains tests that run against a fake EC2 server 200 // running within the test process itself. These tests can test things that 201 // would be unreasonably slow or expensive to test on a live Amazon server. 202 // It starts a new local ec2test server for each test. The server is 203 // accessed by using the "test" region, which is changed to point to the 204 // network address of the local server. 205 type localServerSuite struct { 206 jujutest.Tests 207 srv localServer 208 restoreEC2Patching func() 209 } 210 211 func (t *localServerSuite) SetUpSuite(c *gc.C) { 212 t.TestConfig = localConfigAttrs 213 t.restoreEC2Patching = patchEC2ForTesting() 214 t.Tests.SetUpSuite(c) 215 } 216 217 func (t *localServerSuite) TearDownSuite(c *gc.C) { 218 t.Tests.TearDownSuite(c) 219 t.restoreEC2Patching() 220 } 221 222 func (t *localServerSuite) SetUpTest(c *gc.C) { 223 t.srv.startServer(c) 224 t.Tests.SetUpTest(c) 225 } 226 227 func (t *localServerSuite) TearDownTest(c *gc.C) { 228 t.Tests.TearDownTest(c) 229 t.srv.stopServer(c) 230 } 231 232 func bootstrapContext(c *gc.C) environs.BootstrapContext { 233 return envtesting.NewBootstrapContext(coretesting.Context(c)) 234 } 235 236 func (t *localServerSuite) TestPrecheck(c *gc.C) { 237 env := t.Prepare(c) 238 prechecker, ok := env.(environs.Prechecker) 239 c.Assert(ok, jc.IsTrue) 240 var cons constraints.Value 241 err := prechecker.PrecheckInstance("precise", cons) 242 c.Check(err, gc.IsNil) 243 err = prechecker.PrecheckContainer("precise", instance.LXC) 244 c.Check(err, gc.ErrorMatches, "ec2 provider does not support containers") 245 } 246 247 func (t *localServerSuite) TestBootstrapInstanceUserDataAndState(c *gc.C) { 248 env := t.Prepare(c) 249 envtesting.UploadFakeTools(c, env.Storage()) 250 err := bootstrap.Bootstrap(bootstrapContext(c), env, constraints.Value{}) 251 c.Assert(err, gc.IsNil) 252 253 // check that the state holds the id of the bootstrap machine. 254 bootstrapState, err := bootstrap.LoadState(env.Storage()) 255 c.Assert(err, gc.IsNil) 256 c.Assert(bootstrapState.StateInstances, gc.HasLen, 1) 257 258 expectedHardware := instance.MustParseHardware("arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M") 259 insts, err := env.AllInstances() 260 c.Assert(err, gc.IsNil) 261 c.Assert(insts, gc.HasLen, 1) 262 c.Check(insts[0].Id(), gc.Equals, bootstrapState.StateInstances[0]) 263 c.Check(expectedHardware, gc.DeepEquals, bootstrapState.Characteristics[0]) 264 265 // check that the user data is configured to start zookeeper 266 // and the machine and provisioning agents. 267 // check that the user data is configured to only configure 268 // authorized SSH keys and set the log output; everything 269 // else happens after the machine is brought up. 270 inst := t.srv.ec2srv.Instance(string(insts[0].Id())) 271 c.Assert(inst, gc.NotNil) 272 bootstrapDNS, err := insts[0].DNSName() 273 c.Assert(err, gc.IsNil) 274 c.Assert(bootstrapDNS, gc.Not(gc.Equals), "") 275 userData, err := utils.Gunzip(inst.UserData) 276 c.Assert(err, gc.IsNil) 277 c.Logf("first instance: UserData: %q", userData) 278 var userDataMap map[interface{}]interface{} 279 err = goyaml.Unmarshal(userData, &userDataMap) 280 c.Assert(err, gc.IsNil) 281 c.Assert(userDataMap, jc.DeepEquals, map[interface{}]interface{}{ 282 "output": map[interface{}]interface{}{ 283 "all": "| tee -a /var/log/cloud-init-output.log", 284 }, 285 "ssh_authorized_keys": splitAuthKeys(env.Config().AuthorizedKeys()), 286 "runcmd": []interface{}{ 287 "set -xe", 288 "install -D -m 644 /dev/null '/var/lib/juju/nonce.txt'", 289 "printf '%s\\n' 'user-admin:bootstrap' > '/var/lib/juju/nonce.txt'", 290 }, 291 }) 292 293 // check that a new instance will be started with a machine agent 294 inst1, hc := testing.AssertStartInstance(c, env, "1") 295 c.Check(*hc.Arch, gc.Equals, "amd64") 296 c.Check(*hc.Mem, gc.Equals, uint64(1740)) 297 c.Check(*hc.CpuCores, gc.Equals, uint64(1)) 298 c.Assert(*hc.CpuPower, gc.Equals, uint64(100)) 299 inst = t.srv.ec2srv.Instance(string(inst1.Id())) 300 c.Assert(inst, gc.NotNil) 301 userData, err = utils.Gunzip(inst.UserData) 302 c.Assert(err, gc.IsNil) 303 c.Logf("second instance: UserData: %q", userData) 304 userDataMap = nil 305 err = goyaml.Unmarshal(userData, &userDataMap) 306 c.Assert(err, gc.IsNil) 307 CheckPackage(c, userDataMap, "git", true) 308 CheckPackage(c, userDataMap, "mongodb-server", false) 309 CheckScripts(c, userDataMap, "jujud bootstrap-state", false) 310 CheckScripts(c, userDataMap, "/var/lib/juju/agents/machine-1/agent.conf", true) 311 // TODO check for provisioning agent 312 313 err = env.Destroy() 314 c.Assert(err, gc.IsNil) 315 316 _, err = bootstrap.LoadState(env.Storage()) 317 c.Assert(err, gc.NotNil) 318 } 319 320 // splitAuthKeys splits the given authorized keys 321 // into the form expected to be found in the 322 // user data. 323 func splitAuthKeys(keys string) []interface{} { 324 slines := strings.FieldsFunc(keys, func(r rune) bool { 325 return r == '\n' 326 }) 327 var lines []interface{} 328 for _, line := range slines { 329 lines = append(lines, ssh.EnsureJujuComment(strings.TrimSpace(line))) 330 } 331 return lines 332 } 333 334 func (t *localServerSuite) TestInstanceStatus(c *gc.C) { 335 env := t.Prepare(c) 336 envtesting.UploadFakeTools(c, env.Storage()) 337 err := bootstrap.Bootstrap(bootstrapContext(c), env, constraints.Value{}) 338 c.Assert(err, gc.IsNil) 339 t.srv.ec2srv.SetInitialInstanceState(ec2test.Terminated) 340 inst, _ := testing.AssertStartInstance(c, env, "1") 341 c.Assert(err, gc.IsNil) 342 c.Assert(inst.Status(), gc.Equals, "terminated") 343 } 344 345 func (t *localServerSuite) TestStartInstanceHardwareCharacteristics(c *gc.C) { 346 env := t.Prepare(c) 347 envtesting.UploadFakeTools(c, env.Storage()) 348 err := bootstrap.Bootstrap(bootstrapContext(c), env, constraints.Value{}) 349 c.Assert(err, gc.IsNil) 350 _, hc := testing.AssertStartInstance(c, env, "1") 351 c.Check(*hc.Arch, gc.Equals, "amd64") 352 c.Check(*hc.Mem, gc.Equals, uint64(1740)) 353 c.Check(*hc.CpuCores, gc.Equals, uint64(1)) 354 c.Assert(*hc.CpuPower, gc.Equals, uint64(100)) 355 } 356 357 func (t *localServerSuite) TestAddresses(c *gc.C) { 358 env := t.Prepare(c) 359 envtesting.UploadFakeTools(c, env.Storage()) 360 err := bootstrap.Bootstrap(bootstrapContext(c), env, constraints.Value{}) 361 c.Assert(err, gc.IsNil) 362 inst, _ := testing.AssertStartInstance(c, env, "1") 363 c.Assert(err, gc.IsNil) 364 addrs, err := inst.Addresses() 365 c.Assert(err, gc.IsNil) 366 // Expected values use Address type but really contain a regexp for 367 // the value rather than a valid ip or hostname. 368 expected := []instance.Address{{ 369 Value: "*.testing.invalid", 370 Type: instance.HostName, 371 NetworkScope: instance.NetworkPublic, 372 }, { 373 Value: "*.internal.invalid", 374 Type: instance.HostName, 375 NetworkScope: instance.NetworkCloudLocal, 376 }, { 377 Value: "8.0.0.*", 378 Type: instance.Ipv4Address, 379 NetworkScope: instance.NetworkPublic, 380 }, { 381 Value: "127.0.0.*", 382 Type: instance.Ipv4Address, 383 NetworkScope: instance.NetworkCloudLocal, 384 }} 385 c.Assert(addrs, gc.HasLen, len(expected)) 386 for i, addr := range addrs { 387 c.Check(addr.Value, gc.Matches, expected[i].Value) 388 c.Check(addr.Type, gc.Equals, expected[i].Type) 389 c.Check(addr.NetworkScope, gc.Equals, expected[i].NetworkScope) 390 } 391 } 392 393 func (t *localServerSuite) TestValidateImageMetadata(c *gc.C) { 394 env := t.Prepare(c) 395 params, err := env.(simplestreams.MetadataValidator).MetadataLookupParams("test") 396 c.Assert(err, gc.IsNil) 397 params.Series = "precise" 398 params.Endpoint = "https://ec2.endpoint.com" 399 params.Sources, err = imagemetadata.GetMetadataSources(env) 400 c.Assert(err, gc.IsNil) 401 image_ids, err := imagemetadata.ValidateImageMetadata(params) 402 c.Assert(err, gc.IsNil) 403 sort.Strings(image_ids) 404 c.Assert(image_ids, gc.DeepEquals, []string{"ami-00000033", "ami-00000034", "ami-00000035"}) 405 } 406 407 func (t *localServerSuite) TestGetToolsMetadataSources(c *gc.C) { 408 env := t.Prepare(c) 409 sources, err := tools.GetMetadataSources(env) 410 c.Assert(err, gc.IsNil) 411 c.Assert(len(sources), gc.Equals, 1) 412 url, err := sources[0].URL("") 413 // The control bucket URL contains the bucket name. 414 c.Assert(strings.Contains(url, ec2.ControlBucketName(env)+"/tools"), jc.IsTrue) 415 } 416 417 // localNonUSEastSuite is similar to localServerSuite but the S3 mock server 418 // behaves as if it is not in the us-east region. 419 type localNonUSEastSuite struct { 420 testbase.LoggingSuite 421 restoreEC2Patching func() 422 srv localServer 423 env environs.Environ 424 } 425 426 func (t *localNonUSEastSuite) SetUpSuite(c *gc.C) { 427 t.LoggingSuite.SetUpSuite(c) 428 t.restoreEC2Patching = patchEC2ForTesting() 429 } 430 431 func (t *localNonUSEastSuite) TearDownSuite(c *gc.C) { 432 t.restoreEC2Patching() 433 t.LoggingSuite.TearDownSuite(c) 434 } 435 436 func (t *localNonUSEastSuite) SetUpTest(c *gc.C) { 437 t.LoggingSuite.SetUpTest(c) 438 t.srv.config = &s3test.Config{ 439 Send409Conflict: true, 440 } 441 t.srv.startServer(c) 442 443 cfg, err := config.New(config.NoDefaults, localConfigAttrs) 444 c.Assert(err, gc.IsNil) 445 env, err := environs.Prepare(cfg, configstore.NewMem()) 446 c.Assert(err, gc.IsNil) 447 t.env = env 448 } 449 450 func (t *localNonUSEastSuite) TearDownTest(c *gc.C) { 451 t.srv.stopServer(c) 452 t.LoggingSuite.TearDownTest(c) 453 } 454 455 func patchEC2ForTesting() func() { 456 ec2.UseTestImageData(ec2.TestImagesData) 457 ec2.UseTestInstanceTypeData(ec2.TestInstanceTypeCosts) 458 ec2.UseTestRegionData(ec2.TestRegions) 459 restoreTimeouts := envtesting.PatchAttemptStrategies(ec2.ShortAttempt, ec2.StorageAttempt) 460 restoreFinishBootstrap := envtesting.DisableFinishBootstrap() 461 return func() { 462 restoreFinishBootstrap() 463 restoreTimeouts() 464 ec2.UseTestImageData(nil) 465 ec2.UseTestInstanceTypeData(nil) 466 ec2.UseTestRegionData(nil) 467 } 468 } 469 470 // If match is true, CheckScripts checks that at least one script started 471 // by the cloudinit data matches the given regexp pattern, otherwise it 472 // checks that no script matches. It's exported so it can be used by tests 473 // defined in ec2_test. 474 func CheckScripts(c *gc.C, userDataMap map[interface{}]interface{}, pattern string, match bool) { 475 scripts0 := userDataMap["runcmd"] 476 if scripts0 == nil { 477 c.Errorf("cloudinit has no entry for runcmd") 478 return 479 } 480 scripts := scripts0.([]interface{}) 481 re := regexp.MustCompile(pattern) 482 found := false 483 for _, s0 := range scripts { 484 s := s0.(string) 485 if re.MatchString(s) { 486 found = true 487 } 488 } 489 switch { 490 case match && !found: 491 c.Errorf("script %q not found in %q", pattern, scripts) 492 case !match && found: 493 c.Errorf("script %q found but not expected in %q", pattern, scripts) 494 } 495 } 496 497 // CheckPackage checks that the cloudinit will or won't install the given 498 // package, depending on the value of match. It's exported so it can be 499 // used by tests defined outside the ec2 package. 500 func CheckPackage(c *gc.C, userDataMap map[interface{}]interface{}, pkg string, match bool) { 501 pkgs0 := userDataMap["packages"] 502 if pkgs0 == nil { 503 if match { 504 c.Errorf("cloudinit has no entry for packages") 505 } 506 return 507 } 508 509 pkgs := pkgs0.([]interface{}) 510 511 found := false 512 for _, p0 := range pkgs { 513 p := p0.(string) 514 if p == pkg { 515 found = true 516 } 517 } 518 switch { 519 case match && !found: 520 c.Errorf("package %q not found in %v", pkg, pkgs) 521 case !match && found: 522 c.Errorf("%q found but not expected in %v", pkg, pkgs) 523 } 524 }