github.com/wallyworld/juju@v0.0.0-20161013125918-6cf1bc9d917a/apiserver/apiserver.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package apiserver 5 6 import ( 7 "crypto/tls" 8 "crypto/x509" 9 "net" 10 "net/http" 11 "strings" 12 "sync" 13 "sync/atomic" 14 15 "github.com/bmizerany/pat" 16 "github.com/juju/errors" 17 "github.com/juju/loggo" 18 "github.com/juju/utils" 19 "github.com/juju/utils/clock" 20 "golang.org/x/crypto/acme" 21 "golang.org/x/crypto/acme/autocert" 22 "golang.org/x/net/websocket" 23 "gopkg.in/juju/names.v2" 24 "gopkg.in/macaroon-bakery.v1/httpbakery" 25 "gopkg.in/tomb.v1" 26 27 "github.com/juju/juju/apiserver/authentication" 28 "github.com/juju/juju/apiserver/common" 29 "github.com/juju/juju/apiserver/common/apihttp" 30 "github.com/juju/juju/apiserver/observer" 31 "github.com/juju/juju/apiserver/params" 32 "github.com/juju/juju/rpc" 33 "github.com/juju/juju/rpc/jsoncodec" 34 "github.com/juju/juju/state" 35 ) 36 37 var logger = loggo.GetLogger("juju.apiserver") 38 39 // loginRateLimit defines how many concurrent Login requests we will 40 // accept 41 const loginRateLimit = 10 42 43 // Server holds the server side of the API. 44 type Server struct { 45 tomb tomb.Tomb 46 clock clock.Clock 47 pingClock clock.Clock 48 wg sync.WaitGroup 49 state *state.State 50 statePool *state.StatePool 51 lis net.Listener 52 tag names.Tag 53 dataDir string 54 logDir string 55 limiter utils.Limiter 56 validator LoginValidator 57 adminAPIFactories map[int]adminAPIFactory 58 modelUUID string 59 authCtxt *authContext 60 lastConnectionID uint64 61 newObserver observer.ObserverFactory 62 connCount int64 63 certChanged <-chan params.StateServingInfo 64 tlsConfig *tls.Config 65 allowModelAccess bool 66 67 // mu guards the fields below it. 68 mu sync.Mutex 69 70 // cert holds the current certificate used for tls.Config. 71 cert *tls.Certificate 72 73 // certDNSNames holds the DNS names associated with cert. 74 certDNSNames []string 75 } 76 77 // LoginValidator functions are used to decide whether login requests 78 // are to be allowed. The validator is called before credentials are 79 // checked. 80 type LoginValidator func(params.LoginRequest) error 81 82 // ServerConfig holds parameters required to set up an API server. 83 type ServerConfig struct { 84 Clock clock.Clock 85 PingClock clock.Clock 86 Cert string 87 Key string 88 Tag names.Tag 89 DataDir string 90 LogDir string 91 Validator LoginValidator 92 CertChanged <-chan params.StateServingInfo 93 94 // AutocertDNSName holds the DNS name for which 95 // official TLS certificates will be obtained. If this is 96 // empty, no certificates will be requested. 97 AutocertDNSName string 98 99 // AutocertURL holds the URL from which official 100 // TLS certificates will be obtained. By default, 101 // acme.LetsEncryptURL will be used. 102 AutocertURL string 103 104 // AllowModelAccess holds whether users will be allowed to 105 // access models that they have access rights to even when 106 // they don't have access to the controller. 107 AllowModelAccess bool 108 109 // NewObserver is a function which will return an observer. This 110 // is used per-connection to instantiate a new observer to be 111 // notified of key events during API requests. 112 NewObserver observer.ObserverFactory 113 114 // StatePool only exists to support testing. 115 StatePool *state.StatePool 116 } 117 118 func (c *ServerConfig) Validate() error { 119 if c.Clock == nil { 120 return errors.NotValidf("missing Clock") 121 } 122 if c.NewObserver == nil { 123 return errors.NotValidf("missing NewObserver") 124 } 125 126 return nil 127 } 128 129 func (c *ServerConfig) pingClock() clock.Clock { 130 if c.PingClock == nil { 131 return c.Clock 132 } 133 return c.PingClock 134 } 135 136 // NewServer serves the given state by accepting requests on the given 137 // listener, using the given certificate and key (in PEM format) for 138 // authentication. 139 // 140 // The Server will close the listener when it exits, even if returns an error. 141 func NewServer(s *state.State, lis net.Listener, cfg ServerConfig) (*Server, error) { 142 if err := cfg.Validate(); err != nil { 143 return nil, errors.Trace(err) 144 } 145 146 // Important note: 147 // Do not manipulate the state within NewServer as the API 148 // server needs to run before mongo upgrades have happened and 149 // any state manipulation may be be relying on features of the 150 // database added by upgrades. Here be dragons. 151 srv, err := newServer(s, lis, cfg) 152 if err != nil { 153 // There is no running server around to close the listener. 154 lis.Close() 155 return nil, errors.Trace(err) 156 } 157 return srv, nil 158 } 159 160 func newServer(s *state.State, lis net.Listener, cfg ServerConfig) (_ *Server, err error) { 161 stPool := cfg.StatePool 162 if stPool == nil { 163 stPool = state.NewStatePool(s) 164 } 165 166 srv := &Server{ 167 clock: cfg.Clock, 168 pingClock: cfg.pingClock(), 169 lis: lis, 170 newObserver: cfg.NewObserver, 171 state: s, 172 statePool: stPool, 173 tag: cfg.Tag, 174 dataDir: cfg.DataDir, 175 logDir: cfg.LogDir, 176 limiter: utils.NewLimiter(loginRateLimit), 177 validator: cfg.Validator, 178 adminAPIFactories: map[int]adminAPIFactory{ 179 3: newAdminAPIV3, 180 }, 181 certChanged: cfg.CertChanged, 182 allowModelAccess: cfg.AllowModelAccess, 183 } 184 185 srv.tlsConfig = srv.newTLSConfig(cfg) 186 srv.lis = tls.NewListener(lis, srv.tlsConfig) 187 188 srv.authCtxt, err = newAuthContext(s) 189 if err != nil { 190 return nil, errors.Trace(err) 191 } 192 if err := srv.updateCertificate(cfg.Cert, cfg.Key); err != nil { 193 return nil, errors.Annotatef(err, "cannot set initial certificate") 194 } 195 go srv.run() 196 return srv, nil 197 } 198 199 func (srv *Server) newTLSConfig(cfg ServerConfig) *tls.Config { 200 tlsConfig := utils.SecureTLSConfig() 201 if cfg.AutocertDNSName == "" { 202 // No official DNS name, no certificate. 203 tlsConfig.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { 204 cert, _ := srv.localCertificate(clientHello.ServerName) 205 return cert, nil 206 } 207 return tlsConfig 208 } 209 m := autocert.Manager{ 210 Prompt: autocert.AcceptTOS, 211 Cache: srv.state.AutocertCache(), 212 HostPolicy: autocert.HostWhitelist(cfg.AutocertDNSName), 213 } 214 if cfg.AutocertURL != "" { 215 m.Client = &acme.Client{ 216 DirectoryURL: cfg.AutocertURL, 217 } 218 } 219 tlsConfig.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { 220 logger.Infof("getting certificate for server name %q", clientHello.ServerName) 221 // Get the locally created certificate and whether it's appropriate 222 // for the SNI name. If not, we'll try to get an acme cert and 223 // fall back to the local certificate if that fails. 224 cert, shouldUse := srv.localCertificate(clientHello.ServerName) 225 if shouldUse { 226 return cert, nil 227 } 228 acmeCert, err := m.GetCertificate(clientHello) 229 if err == nil { 230 return acmeCert, nil 231 } 232 logger.Errorf("cannot get autocert certificate for %q: %v", clientHello.ServerName, err) 233 return cert, nil 234 } 235 return tlsConfig 236 } 237 238 func (srv *Server) ConnectionCount() int64 { 239 return atomic.LoadInt64(&srv.connCount) 240 } 241 242 // Dead returns a channel that signals when the server has exited. 243 func (srv *Server) Dead() <-chan struct{} { 244 return srv.tomb.Dead() 245 } 246 247 // Stop stops the server and returns when all running requests 248 // have completed. 249 func (srv *Server) Stop() error { 250 srv.tomb.Kill(nil) 251 return srv.tomb.Wait() 252 } 253 254 // Kill implements worker.Worker.Kill. 255 func (srv *Server) Kill() { 256 srv.tomb.Kill(nil) 257 } 258 259 // Wait implements worker.Worker.Wait. 260 func (srv *Server) Wait() error { 261 return srv.tomb.Wait() 262 } 263 264 func (srv *Server) run() { 265 logger.Infof("listening on %q", srv.lis.Addr()) 266 267 defer func() { 268 addr := srv.lis.Addr().String() // Addr not valid after close 269 err := srv.lis.Close() 270 logger.Infof("closed listening socket %q with final error: %v", addr, err) 271 272 // Break deadlocks caused by leadership BlockUntil... calls. 273 srv.statePool.KillWorkers() 274 srv.state.KillWorkers() 275 276 srv.wg.Wait() // wait for any outstanding requests to complete. 277 srv.tomb.Done() 278 srv.statePool.Close() 279 srv.state.Close() 280 }() 281 282 srv.wg.Add(1) 283 go func() { 284 defer srv.wg.Done() 285 srv.tomb.Kill(srv.mongoPinger()) 286 }() 287 288 srv.wg.Add(1) 289 go func() { 290 defer srv.wg.Done() 291 srv.tomb.Kill(srv.expireLocalLoginInteractions()) 292 }() 293 294 srv.wg.Add(1) 295 go func() { 296 defer srv.wg.Done() 297 srv.tomb.Kill(srv.processCertChanges()) 298 }() 299 300 srv.wg.Add(1) 301 go func() { 302 defer srv.wg.Done() 303 srv.tomb.Kill(srv.processModelRemovals()) 304 }() 305 306 // for pat based handlers, they are matched in-order of being 307 // registered, first match wins. So more specific ones have to be 308 // registered first. 309 mux := pat.New() 310 for _, endpoint := range srv.endpoints() { 311 registerEndpoint(endpoint, mux) 312 } 313 314 go func() { 315 logger.Debugf("Starting API http server on address %q", srv.lis.Addr()) 316 httpSrv := &http.Server{ 317 Handler: mux, 318 TLSConfig: srv.tlsConfig, 319 } 320 err := httpSrv.Serve(srv.lis) 321 // Normally logging an error at debug level would be grounds for a beating, 322 // however in this case the error is *expected* to be non nil, and does not 323 // affect the operation of the apiserver, but for completeness log it anyway. 324 logger.Debugf("API http server exited, final error was: %v", err) 325 }() 326 327 <-srv.tomb.Dying() 328 } 329 330 func (srv *Server) endpoints() []apihttp.Endpoint { 331 httpCtxt := httpContext{ 332 srv: srv, 333 } 334 335 endpoints := common.ResolveAPIEndpoints(srv.newHandlerArgs) 336 337 // TODO(ericsnow) Add the following to the registry instead. 338 339 add := func(pattern string, handler http.Handler) { 340 // TODO: We can switch from all methods to specific ones for entries 341 // where we only want to support specific request methods. However, our 342 // tests currently assert that errors come back as application/json and 343 // pat only does "text/plain" responses. 344 for _, method := range common.DefaultHTTPMethods { 345 endpoints = append(endpoints, apihttp.Endpoint{ 346 Pattern: pattern, 347 Method: method, 348 Handler: handler, 349 }) 350 } 351 } 352 353 strictCtxt := httpCtxt 354 strictCtxt.strictValidation = true 355 strictCtxt.controllerModelOnly = true 356 357 mainAPIHandler := srv.trackRequests(http.HandlerFunc(srv.apiHandler)) 358 logSinkHandler := srv.trackRequests(newLogSinkHandler(httpCtxt, srv.logDir)) 359 logStreamHandler := srv.trackRequests(newLogStreamEndpointHandler(strictCtxt)) 360 debugLogHandler := srv.trackRequests(newDebugLogDBHandler(httpCtxt)) 361 362 add("/model/:modeluuid/logsink", logSinkHandler) 363 add("/model/:modeluuid/logstream", logStreamHandler) 364 add("/model/:modeluuid/log", debugLogHandler) 365 366 charmsHandler := &charmsHandler{ 367 ctxt: httpCtxt, 368 dataDir: srv.dataDir, 369 } 370 charmsServer := &CharmsHTTPHandler{ 371 PostHandler: charmsHandler.ServePost, 372 GetHandler: charmsHandler.ServeGet, 373 } 374 add("/model/:modeluuid/charms", charmsServer) 375 add("/model/:modeluuid/tools", 376 &toolsUploadHandler{ 377 ctxt: httpCtxt, 378 }, 379 ) 380 add("/model/:modeluuid/tools/:version", 381 &toolsDownloadHandler{ 382 ctxt: httpCtxt, 383 }, 384 ) 385 add("/model/:modeluuid/backups", 386 &backupHandler{ 387 ctxt: strictCtxt, 388 }, 389 ) 390 add("/model/:modeluuid/api", mainAPIHandler) 391 392 endpoints = append(endpoints, guiEndpoints("/gui/:modeluuid/", srv.dataDir, httpCtxt)...) 393 add("/gui-archive", &guiArchiveHandler{ 394 ctxt: httpCtxt, 395 }) 396 add("/gui-version", &guiVersionHandler{ 397 ctxt: httpCtxt, 398 }) 399 400 // For backwards compatibility we register all the old paths 401 add("/log", debugLogHandler) 402 403 add("/charms", charmsServer) 404 add("/tools", 405 &toolsUploadHandler{ 406 ctxt: httpCtxt, 407 }, 408 ) 409 add("/tools/:version", 410 &toolsDownloadHandler{ 411 ctxt: httpCtxt, 412 }, 413 ) 414 add("/register", 415 ®isterUserHandler{ 416 ctxt: httpCtxt, 417 }, 418 ) 419 add("/api", mainAPIHandler) 420 // Serve the API at / (only) for backward compatiblity. Note that the 421 // pat muxer special-cases / so that it does not serve all 422 // possible endpoints, but only / itself. 423 add("/", mainAPIHandler) 424 425 // Add HTTP handlers for local-user macaroon authentication. 426 localLoginHandlers := &localLoginHandlers{srv.authCtxt, srv.state} 427 dischargeMux := http.NewServeMux() 428 httpbakery.AddDischargeHandler( 429 dischargeMux, 430 localUserIdentityLocationPath, 431 localLoginHandlers.authCtxt.localUserThirdPartyBakeryService, 432 localLoginHandlers.checkThirdPartyCaveat, 433 ) 434 dischargeMux.Handle( 435 localUserIdentityLocationPath+"/login", 436 makeHandler(handleJSON(localLoginHandlers.serveLogin)), 437 ) 438 dischargeMux.Handle( 439 localUserIdentityLocationPath+"/wait", 440 makeHandler(handleJSON(localLoginHandlers.serveWait)), 441 ) 442 add(localUserIdentityLocationPath+"/discharge", dischargeMux) 443 add(localUserIdentityLocationPath+"/publickey", dischargeMux) 444 add(localUserIdentityLocationPath+"/login", dischargeMux) 445 add(localUserIdentityLocationPath+"/wait", dischargeMux) 446 447 return endpoints 448 } 449 450 func (srv *Server) expireLocalLoginInteractions() error { 451 for { 452 select { 453 case <-srv.tomb.Dying(): 454 return tomb.ErrDying 455 case <-srv.clock.After(authentication.LocalLoginInteractionTimeout): 456 now := srv.authCtxt.clock.Now() 457 srv.authCtxt.localUserInteractions.Expire(now) 458 } 459 } 460 } 461 462 func (srv *Server) newHandlerArgs(spec apihttp.HandlerConstraints) apihttp.NewHandlerArgs { 463 ctxt := httpContext{ 464 srv: srv, 465 strictValidation: spec.StrictValidation, 466 controllerModelOnly: spec.ControllerModelOnly, 467 } 468 469 var args apihttp.NewHandlerArgs 470 switch spec.AuthKind { 471 case names.UserTagKind: 472 args.Connect = ctxt.stateForRequestAuthenticatedUser 473 case names.UnitTagKind: 474 args.Connect = ctxt.stateForRequestAuthenticatedAgent 475 case "": 476 logger.Tracef(`no access level specified; proceeding with "unauthenticated"`) 477 args.Connect = func(req *http.Request) (*state.State, state.Entity, error) { 478 st, err := ctxt.stateForRequestUnauthenticated(req) 479 return st, nil, err 480 } 481 default: 482 logger.Infof(`unrecognized access level %q; proceeding with "unauthenticated"`, spec.AuthKind) 483 args.Connect = func(req *http.Request) (*state.State, state.Entity, error) { 484 st, err := ctxt.stateForRequestUnauthenticated(req) 485 return st, nil, err 486 } 487 } 488 return args 489 } 490 491 // trackRequests wraps a http.Handler, incrementing and decrementing 492 // the apiserver's WaitGroup and blocking request when the apiserver 493 // is shutting down. 494 // 495 // Note: It is only safe to use trackRequests with API handlers which 496 // are interruptible (i.e. they pay attention to the apiserver tomb) 497 // or are guaranteed to be short-lived. If it's used with long running 498 // API handlers which don't watch the apiserver's tomb, apiserver 499 // shutdown will be blocked until the API handler returns. 500 func (srv *Server) trackRequests(handler http.Handler) http.Handler { 501 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 502 // Care must be taken to not increment the waitgroup count 503 // after the listener has closed. 504 // 505 // First we check to see if the tomb has not yet been killed 506 // because the closure of the listener depends on the tomb being 507 // killed to trigger the defer block in srv.run. 508 select { 509 case <-srv.tomb.Dying(): 510 // This request was accepted before the listener was closed 511 // but after the tomb was killed. As we're in the process of 512 // shutting down, do not consider this request as in progress, 513 // just send a 503 and return. 514 http.Error(w, "apiserver shutdown in progress", 503) 515 default: 516 // If we get here then the tomb was not killed therefore the 517 // listener is still open. It is safe to increment the 518 // wg counter as wg.Wait in srv.run has not yet been called. 519 srv.wg.Add(1) 520 defer srv.wg.Done() 521 handler.ServeHTTP(w, r) 522 } 523 }) 524 } 525 526 func registerEndpoint(ep apihttp.Endpoint, mux *pat.PatternServeMux) { 527 mux.Add(ep.Method, ep.Pattern, ep.Handler) 528 if ep.Method == "GET" { 529 mux.Add("HEAD", ep.Pattern, ep.Handler) 530 } 531 } 532 533 func (srv *Server) apiHandler(w http.ResponseWriter, req *http.Request) { 534 addCount := func(delta int64) { 535 atomic.AddInt64(&srv.connCount, delta) 536 } 537 538 addCount(1) 539 defer addCount(-1) 540 541 connectionID := atomic.AddUint64(&srv.lastConnectionID, 1) 542 543 apiObserver := srv.newObserver() 544 apiObserver.Join(req, connectionID) 545 defer apiObserver.Leave() 546 547 wsServer := websocket.Server{ 548 Handler: func(conn *websocket.Conn) { 549 modelUUID := req.URL.Query().Get(":modeluuid") 550 logger.Tracef("got a request for model %q", modelUUID) 551 if err := srv.serveConn(conn, modelUUID, apiObserver, req.Host); err != nil { 552 logger.Errorf("error serving RPCs: %v", err) 553 } 554 }, 555 } 556 wsServer.ServeHTTP(w, req) 557 } 558 559 func (srv *Server) serveConn(wsConn *websocket.Conn, modelUUID string, apiObserver observer.Observer, host string) error { 560 codec := jsoncodec.NewWebsocket(wsConn) 561 562 conn := rpc.NewConn(codec, apiObserver) 563 564 // Note that we don't overwrite modelUUID here because 565 // newAPIHandler treats an empty modelUUID as signifying 566 // the API version used. 567 resolvedModelUUID, err := validateModelUUID(validateArgs{ 568 statePool: srv.statePool, 569 modelUUID: modelUUID, 570 }) 571 var ( 572 st *state.State 573 h *apiHandler 574 ) 575 if err == nil { 576 st, err = srv.statePool.Get(resolvedModelUUID) 577 } 578 579 if err == nil { 580 defer func() { 581 err := srv.statePool.Release(resolvedModelUUID) 582 if err != nil { 583 logger.Errorf("error releasing %v back into the state pool:", err) 584 } 585 }() 586 h, err = newAPIHandler(srv, st, conn, modelUUID, host) 587 } 588 589 if err != nil { 590 conn.ServeRoot(&errRoot{errors.Trace(err)}, serverError) 591 } else { 592 adminAPIs := make(map[int]interface{}) 593 for apiVersion, factory := range srv.adminAPIFactories { 594 adminAPIs[apiVersion] = factory(srv, h, apiObserver) 595 } 596 conn.ServeRoot(newAnonRoot(h, adminAPIs), serverError) 597 } 598 conn.Start() 599 select { 600 case <-conn.Dead(): 601 case <-srv.tomb.Dying(): 602 } 603 return conn.Close() 604 } 605 606 func (srv *Server) mongoPinger() error { 607 session := srv.state.MongoSession().Copy() 608 defer session.Close() 609 for { 610 if err := session.Ping(); err != nil { 611 logger.Infof("got error pinging mongo: %v", err) 612 return errors.Annotate(err, "error pinging mongo") 613 } 614 select { 615 case <-srv.clock.After(mongoPingInterval): 616 case <-srv.tomb.Dying(): 617 return tomb.ErrDying 618 } 619 } 620 } 621 622 // localCertificate returns the local server certificate and reports 623 // whether it should be used to serve a connection addressed to the 624 // given server name. 625 func (srv *Server) localCertificate(serverName string) (*tls.Certificate, bool) { 626 srv.mu.Lock() 627 defer srv.mu.Unlock() 628 if net.ParseIP(serverName) != nil { 629 // IP address connections always use the local certificate. 630 return srv.cert, true 631 } 632 if !strings.Contains(serverName, ".") { 633 // If the server name doesn't contain a period there's no 634 // way we can obtain a certificate for it. 635 // This applies to the common case where "juju-apiserver" is 636 // used as the server name. 637 return srv.cert, true 638 } 639 // Perhaps the server name is explicitly mentioned by the server certificate. 640 for _, name := range srv.certDNSNames { 641 if name == serverName { 642 return srv.cert, true 643 } 644 } 645 return srv.cert, false 646 } 647 648 // processCertChanges receives new certificate information and 649 // calls a method to update the listener's certificate. 650 func (srv *Server) processCertChanges() error { 651 for { 652 select { 653 case info := <-srv.certChanged: 654 if info.Cert == "" { 655 break 656 } 657 logger.Infof("received API server certificate") 658 if err := srv.updateCertificate(info.Cert, info.PrivateKey); err != nil { 659 logger.Errorf("cannot update certificate: %v", err) 660 } 661 case <-srv.tomb.Dying(): 662 return tomb.ErrDying 663 } 664 } 665 } 666 667 // updateCertificate updates the current CA certificate and key 668 // from the given cert and key. 669 func (srv *Server) updateCertificate(cert, key string) error { 670 srv.mu.Lock() 671 defer srv.mu.Unlock() 672 tlsCert, err := tls.X509KeyPair([]byte(cert), []byte(key)) 673 if err != nil { 674 return errors.Annotatef(err, "cannot create new TLS certificate") 675 } 676 x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) 677 if err != nil { 678 return errors.Annotatef(err, "parsing x509 cert") 679 } 680 var addr []string 681 for _, ip := range x509Cert.IPAddresses { 682 addr = append(addr, ip.String()) 683 } 684 logger.Infof("new certificate addresses: %v", strings.Join(addr, ", ")) 685 srv.cert = &tlsCert 686 srv.certDNSNames = x509Cert.DNSNames 687 return nil 688 } 689 690 func serverError(err error) error { 691 if err := common.ServerError(err); err != nil { 692 return err 693 } 694 return nil 695 } 696 697 func (srv *Server) processModelRemovals() error { 698 w := srv.state.WatchModels() 699 defer w.Stop() 700 for { 701 select { 702 case <-srv.tomb.Dying(): 703 return tomb.ErrDying 704 case modelUUIDs := <-w.Changes(): 705 for _, modelUUID := range modelUUIDs { 706 model, err := srv.state.GetModel(names.NewModelTag(modelUUID)) 707 gone := errors.IsNotFound(err) 708 dead := err == nil && model.Life() == state.Dead 709 if err != nil && !gone { 710 return errors.Trace(err) 711 } 712 if !dead && !gone { 713 continue 714 } 715 716 logger.Debugf("removing model %v from the state pool", modelUUID) 717 // Model's gone away - ensure that it gets removed 718 // from from the state pool once people are finished 719 // with it. 720 err = srv.statePool.Remove(modelUUID) 721 if err != nil { 722 return errors.Trace(err) 723 } 724 } 725 } 726 } 727 }