github.com/juju/juju@v0.0.0-20240430160146-1752b71fcf00/apiserver/admin_test.go (about)

     1  // Copyright 2012-2014 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package apiserver_test
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"fmt"
    10  	"io"
    11  	"math"
    12  	"net"
    13  	"net/http"
    14  	"net/url"
    15  	"strconv"
    16  	"time"
    17  
    18  	"github.com/go-macaroon-bakery/macaroon-bakery/v3/bakery"
    19  	"github.com/go-macaroon-bakery/macaroon-bakery/v3/httpbakery"
    20  	"github.com/juju/clock/testclock"
    21  	"github.com/juju/collections/set"
    22  	"github.com/juju/errors"
    23  	"github.com/juju/loggo"
    24  	"github.com/juju/names/v5"
    25  	jc "github.com/juju/testing/checkers"
    26  	"github.com/juju/utils/v3"
    27  	gc "gopkg.in/check.v1"
    28  
    29  	"github.com/juju/juju/api"
    30  	apimachiner "github.com/juju/juju/api/agent/machiner"
    31  	apiclient "github.com/juju/juju/api/client/client"
    32  	machineclient "github.com/juju/juju/api/client/machinemanager"
    33  	"github.com/juju/juju/api/client/modelconfig"
    34  	apitesting "github.com/juju/juju/api/testing"
    35  	"github.com/juju/juju/apiserver/common"
    36  	"github.com/juju/juju/apiserver/facades/client/controller"
    37  	servertesting "github.com/juju/juju/apiserver/testing"
    38  	"github.com/juju/juju/apiserver/testserver"
    39  	corecontroller "github.com/juju/juju/controller"
    40  	"github.com/juju/juju/core/auditlog"
    41  	"github.com/juju/juju/core/constraints"
    42  	"github.com/juju/juju/core/migration"
    43  	"github.com/juju/juju/core/model"
    44  	"github.com/juju/juju/core/network"
    45  	"github.com/juju/juju/core/permission"
    46  	jujutesting "github.com/juju/juju/juju/testing"
    47  	"github.com/juju/juju/rpc"
    48  	"github.com/juju/juju/rpc/params"
    49  	"github.com/juju/juju/state"
    50  	"github.com/juju/juju/testing"
    51  	coretesting "github.com/juju/juju/testing"
    52  	"github.com/juju/juju/testing/factory"
    53  	jujuversion "github.com/juju/juju/version"
    54  )
    55  
    56  const (
    57  	clientFacadeVersion           = 6
    58  	machineManagerFacadeVersion   = 9
    59  	userManagerFacadeVersion      = 3
    60  	sshClientFacadeVersion        = 4
    61  	pingerFacadeVersion           = 1
    62  	modelManagerFacadeVersion     = 9
    63  	highAvailabilityFacadeVersion = 2
    64  )
    65  
    66  type baseLoginSuite struct {
    67  	jujutesting.JujuConnSuite
    68  	mgmtSpace *state.Space
    69  }
    70  
    71  func (s *baseLoginSuite) SetUpTest(c *gc.C) {
    72  	if s.ControllerConfigAttrs == nil {
    73  		s.ControllerConfigAttrs = make(map[string]interface{})
    74  	}
    75  
    76  	s.JujuConnSuite.SetUpTest(c)
    77  	loggo.GetLogger("juju.apiserver").SetLogLevel(loggo.TRACE)
    78  
    79  	var err error
    80  	s.mgmtSpace, err = s.State.AddSpace("mgmt01", "", nil, false)
    81  	c.Assert(err, jc.ErrorIsNil)
    82  
    83  	err = s.State.UpdateControllerConfig(map[string]interface{}{corecontroller.JujuManagementSpace: "mgmt01"}, nil)
    84  	c.Assert(err, jc.ErrorIsNil)
    85  }
    86  
    87  // loginSuite is built on statetesting.StateSuite not JujuConnSuite.
    88  // It also uses a testing clock, not a wall clock.
    89  type loginSuite struct {
    90  	baseSuite
    91  }
    92  
    93  var _ = gc.Suite(&loginSuite{})
    94  
    95  func (s *loginSuite) SetUpTest(c *gc.C) {
    96  	s.Clock = testclock.NewDilatedWallClock(time.Second)
    97  	s.baseSuite.SetUpTest(c)
    98  }
    99  
   100  func (s *loginSuite) TestLoginWithInvalidTag(c *gc.C) {
   101  	info := s.newServer(c).Info
   102  	st := s.openAPIWithoutLogin(c, info)
   103  
   104  	request := &params.LoginRequest{
   105  		AuthTag:       "bar",
   106  		Credentials:   "password",
   107  		ClientVersion: jujuversion.Current.String(),
   108  	}
   109  
   110  	var response params.LoginResult
   111  	err := st.APICall("Admin", 3, "", "Login", request, &response)
   112  	c.Assert(err, gc.ErrorMatches, `.*"bar" is not a valid tag.*`)
   113  }
   114  
   115  func (s *loginSuite) TestBadLogin(c *gc.C) {
   116  	info := s.newServer(c).Info
   117  
   118  	for i, t := range []struct {
   119  		tag      names.Tag
   120  		password string
   121  		err      error
   122  		code     string
   123  	}{{
   124  		tag:      s.Owner,
   125  		password: "wrong password",
   126  		err: &rpc.RequestError{
   127  			Message: "invalid entity name or password",
   128  			Code:    "unauthorized access",
   129  		},
   130  		code: params.CodeUnauthorized,
   131  	}, {
   132  		tag:      names.NewUserTag("unknown"),
   133  		password: "password",
   134  		err: &rpc.RequestError{
   135  			Message: "invalid entity name or password",
   136  			Code:    "unauthorized access",
   137  		},
   138  		code: params.CodeUnauthorized,
   139  	}} {
   140  		c.Logf("test %d; entity %q; password %q", i, t.tag, t.password)
   141  		func() {
   142  			// Open the API without logging in, so we can perform
   143  			// operations on the connection before calling Login.
   144  			st := s.openAPIWithoutLogin(c, info)
   145  
   146  			_, err := apimachiner.NewState(st).Machine(names.NewMachineTag("0"))
   147  			c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   148  				Message: `unknown object type "Machiner"`,
   149  				Code:    "not implemented",
   150  			})
   151  
   152  			// Since these are user login tests, the nonce is empty.
   153  			err = st.Login(t.tag, t.password, "", nil)
   154  			c.Assert(errors.Cause(err), gc.DeepEquals, t.err)
   155  			c.Assert(params.ErrCode(err), gc.Equals, t.code)
   156  
   157  			_, err = apimachiner.NewState(st).Machine(names.NewMachineTag("0"))
   158  			c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   159  				Message: `unknown object type "Machiner"`,
   160  				Code:    "not implemented",
   161  			})
   162  		}()
   163  	}
   164  }
   165  
   166  func (s *loginSuite) TestLoginAsDeactivatedUser(c *gc.C) {
   167  	info := s.newServer(c).Info
   168  
   169  	st := s.openAPIWithoutLogin(c, info)
   170  	password := "password"
   171  	u := s.Factory.MakeUser(c, &factory.UserParams{Password: password, Disabled: true})
   172  
   173  	_, err := apiclient.NewClient(st, coretesting.NoopLogger{}).Status(nil)
   174  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   175  		Message: `unknown object type "Client"`,
   176  		Code:    "not implemented",
   177  	})
   178  
   179  	// Since these are user login tests, the nonce is empty.
   180  	err = st.Login(u.Tag(), password, "", nil)
   181  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   182  		Message: fmt.Sprintf("user %q is disabled", u.Tag().Id()),
   183  		Code:    "unauthorized access",
   184  	})
   185  
   186  	_, err = apiclient.NewClient(st, coretesting.NoopLogger{}).Status(nil)
   187  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   188  		Message: `unknown object type "Client"`,
   189  		Code:    "not implemented",
   190  	})
   191  }
   192  
   193  func (s *loginSuite) TestLoginAsDeletedUser(c *gc.C) {
   194  	info := s.newServer(c).Info
   195  
   196  	st := s.openAPIWithoutLogin(c, info)
   197  	password := "password"
   198  	u := s.Factory.MakeUser(c, &factory.UserParams{Password: password})
   199  
   200  	_, err := apiclient.NewClient(st, coretesting.NoopLogger{}).Status(nil)
   201  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   202  		Message: `unknown object type "Client"`,
   203  		Code:    "not implemented",
   204  	})
   205  
   206  	err = s.State.RemoveUser(u.UserTag())
   207  	c.Assert(err, jc.ErrorIsNil)
   208  
   209  	// Since these are user login tests, the nonce is empty.
   210  	err = st.Login(u.Tag(), password, "", nil)
   211  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   212  		Message: fmt.Sprintf("user %q is permanently deleted", u.Tag().Id()),
   213  		Code:    "unauthorized access",
   214  	})
   215  
   216  	_, err = apiclient.NewClient(st, coretesting.NoopLogger{}).Status(nil)
   217  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   218  		Message: `unknown object type "Client"`,
   219  		Code:    "not implemented",
   220  	})
   221  }
   222  
   223  func (s *loginSuite) setupManagementSpace(c *gc.C) *state.Space {
   224  	mgmtSpace, err := s.State.AddSpace("mgmt01", "", nil, false)
   225  	c.Assert(err, jc.ErrorIsNil)
   226  
   227  	err = s.State.UpdateControllerConfig(map[string]interface{}{corecontroller.JujuManagementSpace: "mgmt01"}, nil)
   228  	c.Assert(err, jc.ErrorIsNil)
   229  	return mgmtSpace
   230  }
   231  
   232  func (s *loginSuite) addController(c *gc.C) (state.ControllerNode, string) {
   233  	node, err := s.State.AddControllerNode()
   234  	c.Assert(err, jc.ErrorIsNil)
   235  	password, err := utils.RandomPassword()
   236  	c.Assert(err, jc.ErrorIsNil)
   237  	err = node.SetPassword(password)
   238  	c.Assert(err, jc.ErrorIsNil)
   239  	return node, password
   240  }
   241  
   242  func (s *loginSuite) TestControllerAgentLogin(c *gc.C) {
   243  	// The agent login tests also check the management space.
   244  	mgmtSpace := s.setupManagementSpace(c)
   245  	info := s.newServer(c).Info
   246  
   247  	node, password := s.addController(c)
   248  	info.Tag = node.Tag()
   249  	info.Password = password
   250  	info.Nonce = "fake_nonce"
   251  
   252  	s.assertAgentLogin(c, info, mgmtSpace)
   253  }
   254  
   255  func (s *loginSuite) TestLoginAddressesForAgents(c *gc.C) {
   256  	// The agent login tests also check the management space.
   257  	mgmtSpace := s.setupManagementSpace(c)
   258  
   259  	info := s.newServer(c).Info
   260  	machine := s.infoForNewMachine(c, info)
   261  
   262  	s.assertAgentLogin(c, machine, mgmtSpace)
   263  }
   264  
   265  func (s *loginSuite) loginHostPorts(
   266  	c *gc.C, info *api.Info,
   267  ) (connectedAddr string, hostPorts []network.MachineHostPorts) {
   268  	st, err := api.Open(info, fastDialOpts)
   269  	c.Assert(err, jc.ErrorIsNil)
   270  	defer st.Close()
   271  	return st.Addr(), st.APIHostPorts()
   272  }
   273  
   274  func (s *loginSuite) assertAgentLogin(c *gc.C, info *api.Info, mgmtSpace *state.Space) {
   275  	err := s.State.SetAPIHostPorts(nil)
   276  	c.Assert(err, jc.ErrorIsNil)
   277  
   278  	// Initially just the address we connect with is returned by the helper
   279  	// because there are no APIHostPorts in state.
   280  	connectedAddr, hostPorts := s.loginHostPorts(c, info)
   281  	connectedAddrHost, connectedAddrPortString, err := net.SplitHostPort(connectedAddr)
   282  	c.Assert(err, jc.ErrorIsNil)
   283  
   284  	connectedAddrPort, err := strconv.Atoi(connectedAddrPortString)
   285  	c.Assert(err, jc.ErrorIsNil)
   286  
   287  	connectedAddrHostPorts := []network.MachineHostPorts{
   288  		network.NewMachineHostPorts(connectedAddrPort, connectedAddrHost),
   289  	}
   290  	c.Assert(hostPorts, gc.DeepEquals, connectedAddrHostPorts)
   291  
   292  	// After storing APIHostPorts in state, Login should return the list
   293  	// filtered for agents along with the address we connected to.
   294  	server1Addresses := network.SpaceAddresses{
   295  		network.NewSpaceAddress("server-1", network.WithScope(network.ScopePublic)),
   296  		network.NewSpaceAddress("10.0.0.1", network.WithScope(network.ScopeCloudLocal)),
   297  	}
   298  	server1Addresses[1].SpaceID = mgmtSpace.Id()
   299  
   300  	server2Addresses := network.SpaceAddresses{
   301  		network.NewSpaceAddress("::1", network.WithScope(network.ScopeMachineLocal)),
   302  	}
   303  
   304  	err = s.State.SetAPIHostPorts([]network.SpaceHostPorts{
   305  		network.SpaceAddressesWithPort(server1Addresses, 123),
   306  		network.SpaceAddressesWithPort(server2Addresses, 456),
   307  	})
   308  	c.Assert(err, jc.ErrorIsNil)
   309  
   310  	_, hostPorts = s.loginHostPorts(c, info)
   311  
   312  	// The login method is called with a machine tag, so we expect the
   313  	// first return slice to only have the address in the management space.
   314  	expectedAPIHostPorts := []network.MachineHostPorts{
   315  		{{MachineAddress: server1Addresses[1].MachineAddress, NetPort: 123}},
   316  		{{MachineAddress: server2Addresses[0].MachineAddress, NetPort: 456}},
   317  	}
   318  	// Prepended as before with the connection address.
   319  	expectedAPIHostPorts = append(connectedAddrHostPorts, expectedAPIHostPorts...)
   320  	c.Assert(hostPorts, gc.DeepEquals, expectedAPIHostPorts)
   321  }
   322  
   323  func (s *loginSuite) TestLoginAddressesForClients(c *gc.C) {
   324  	mgmtSpace := s.setupManagementSpace(c)
   325  
   326  	info := s.newServer(c).Info
   327  	info = s.infoForNewUser(c, info)
   328  
   329  	server1Addresses := network.SpaceAddresses{
   330  		network.NewSpaceAddress("server-1", network.WithScope(network.ScopePublic)),
   331  		network.NewSpaceAddress("10.0.0.1", network.WithScope(network.ScopeCloudLocal)),
   332  	}
   333  	server1Addresses[1].SpaceID = mgmtSpace.Id()
   334  
   335  	server2Addresses := network.SpaceAddresses{
   336  		network.NewSpaceAddress("::1", network.WithScope(network.ScopeMachineLocal)),
   337  	}
   338  
   339  	newAPIHostPorts := []network.SpaceHostPorts{
   340  		network.SpaceAddressesWithPort(server1Addresses, 123),
   341  		network.SpaceAddressesWithPort(server2Addresses, 456),
   342  	}
   343  	err := s.State.SetAPIHostPorts(newAPIHostPorts)
   344  	c.Assert(err, jc.ErrorIsNil)
   345  
   346  	exp := []network.MachineHostPorts{
   347  		{
   348  			{
   349  				MachineAddress: network.NewMachineAddress("server-1", network.WithScope(network.ScopePublic)),
   350  				NetPort:        123,
   351  			},
   352  			{
   353  				MachineAddress: network.NewMachineAddress("10.0.0.1", network.WithScope(network.ScopeCloudLocal)),
   354  				NetPort:        123,
   355  			},
   356  		}, {
   357  			{
   358  				MachineAddress: network.NewMachineAddress("::1", network.WithScope(network.ScopeMachineLocal)),
   359  				NetPort:        456,
   360  			},
   361  		},
   362  	}
   363  
   364  	_, hostPorts := s.loginHostPorts(c, info)
   365  	// Ignoring the address used to login, the returned API addresses should not
   366  	// Have management space filtering applied.
   367  	c.Assert(hostPorts[1:], gc.DeepEquals, exp)
   368  }
   369  
   370  func (s *loginSuite) setupRateLimiting(c *gc.C) {
   371  	// Instead of the defaults, we'll be explicit in our ratelimit setup.
   372  	err := s.State.UpdateControllerConfig(
   373  		map[string]interface{}{
   374  			corecontroller.AgentRateLimitMax:  1,
   375  			corecontroller.AgentRateLimitRate: (60 * time.Second).String(),
   376  		}, nil)
   377  	c.Assert(err, jc.ErrorIsNil)
   378  }
   379  
   380  func (s *loginSuite) infoForNewMachine(c *gc.C, info *api.Info) *api.Info {
   381  	// Make a copy
   382  	newInfo := *info
   383  
   384  	machine, password := s.Factory.MakeMachineReturningPassword(
   385  		c, &factory.MachineParams{Nonce: "fake_nonce"})
   386  
   387  	newInfo.Tag = machine.Tag()
   388  	newInfo.Password = password
   389  	newInfo.Nonce = "fake_nonce"
   390  	return &newInfo
   391  }
   392  
   393  func (s *loginSuite) infoForNewUser(c *gc.C, info *api.Info) *api.Info {
   394  	// Make a copy
   395  	newInfo := *info
   396  
   397  	password := "shhh..."
   398  	user := s.Factory.MakeUser(c, &factory.UserParams{
   399  		Password: password,
   400  	})
   401  
   402  	newInfo.Tag = user.Tag()
   403  	newInfo.Password = password
   404  	return &newInfo
   405  }
   406  
   407  func (s *loginSuite) TestRateLimitAgents(c *gc.C) {
   408  	s.setupRateLimiting(c)
   409  
   410  	server := s.newServer(c)
   411  	info := server.Info
   412  
   413  	c.Assert(server.APIServer.Report(), jc.DeepEquals, map[string]interface{}{
   414  		"agent-ratelimit-max":  1,
   415  		"agent-ratelimit-rate": 60 * time.Second,
   416  	})
   417  
   418  	// First agent connection is fine.
   419  	machine1 := s.infoForNewMachine(c, info)
   420  	conn1, err := api.Open(machine1, fastDialOpts)
   421  	c.Assert(err, jc.ErrorIsNil)
   422  	defer conn1.Close()
   423  
   424  	// Second machine in the same minute gets told to go away and try again.
   425  	machine2 := s.infoForNewMachine(c, info)
   426  	_, err = api.Open(machine2, fastDialOpts)
   427  	c.Assert(err, gc.ErrorMatches, `try again \(try again\)`)
   428  
   429  	// If we wait a minute and try again, it is fine.
   430  	s.Clock.Advance(time.Minute)
   431  	conn2, err := api.Open(machine2, fastDialOpts)
   432  	c.Assert(err, jc.ErrorIsNil)
   433  	defer conn2.Close()
   434  
   435  	// And the next one is limited.
   436  	machine3 := s.infoForNewMachine(c, info)
   437  	_, err = api.Open(machine3, fastDialOpts)
   438  	c.Assert(err, gc.ErrorMatches, `try again \(try again\)`)
   439  }
   440  
   441  func (s *loginSuite) TestRateLimitNotApplicableToUsers(c *gc.C) {
   442  	s.setupRateLimiting(c)
   443  	info := s.newServer(c).Info
   444  
   445  	// First agent connection is fine.
   446  	machine1 := s.infoForNewMachine(c, info)
   447  	conn1, err := api.Open(machine1, fastDialOpts)
   448  	c.Assert(err, jc.ErrorIsNil)
   449  	defer conn1.Close()
   450  
   451  	// User connections are fine.
   452  	user := s.infoForNewUser(c, info)
   453  	conn2, err := api.Open(user, fastDialOpts)
   454  	c.Assert(err, jc.ErrorIsNil)
   455  	defer conn2.Close()
   456  
   457  	user2 := s.infoForNewUser(c, info)
   458  	conn3, err := api.Open(user2, fastDialOpts)
   459  	c.Assert(err, jc.ErrorIsNil)
   460  	defer conn3.Close()
   461  }
   462  
   463  func (s *loginSuite) TestNonModelUserLoginFails(c *gc.C) {
   464  	info := s.newServer(c).Info
   465  	user := s.Factory.MakeUser(c, &factory.UserParams{Password: "dummy-password", NoModelUser: true})
   466  	ctag := names.NewControllerTag(s.State.ControllerUUID())
   467  	err := s.State.RemoveUserAccess(user.UserTag(), ctag)
   468  	c.Assert(err, jc.ErrorIsNil)
   469  	info.Password = "dummy-password"
   470  	info.Tag = user.UserTag()
   471  	_, err = api.Open(info, fastDialOpts)
   472  	assertInvalidEntityPassword(c, err)
   473  }
   474  
   475  func (s *loginSuite) TestLoginValidationDuringUpgrade(c *gc.C) {
   476  	s.cfg.UpgradeComplete = func() bool {
   477  		// upgrade is in progress
   478  		return false
   479  	}
   480  	s.testLoginDuringMaintenance(c, func(st api.Connection) {
   481  		var statusResult params.FullStatus
   482  		err := st.APICall("Client", clientFacadeVersion, "", "FullStatus", params.StatusParams{}, &statusResult)
   483  		c.Assert(err, jc.ErrorIsNil)
   484  
   485  		err = st.APICall("Client", clientFacadeVersion, "", "ModelSet", params.ModelSet{}, nil)
   486  		c.Assert(err, jc.Satisfies, params.IsCodeUpgradeInProgress)
   487  	})
   488  }
   489  
   490  func (s *loginSuite) testLoginDuringMaintenance(c *gc.C, check func(api.Connection)) {
   491  	info := s.newServer(c).Info
   492  
   493  	st := s.openAPIWithoutLogin(c, info)
   494  	err := st.Login(s.Owner, s.AdminPassword, "", nil)
   495  	c.Assert(err, jc.ErrorIsNil)
   496  
   497  	check(st)
   498  }
   499  
   500  func (s *loginSuite) TestMachineLoginDuringMaintenance(c *gc.C) {
   501  	s.cfg.UpgradeComplete = func() bool {
   502  		// upgrade is in progress
   503  		return false
   504  	}
   505  	info := s.newServer(c).Info
   506  	machine := s.infoForNewMachine(c, info)
   507  	_, err := api.Open(machine, fastDialOpts)
   508  	c.Assert(err, gc.ErrorMatches, `login for machine \d+ blocked because upgrade is in progress`)
   509  }
   510  
   511  func (s *loginSuite) TestControllerMachineLoginDuringMaintenance(c *gc.C) {
   512  	s.cfg.UpgradeComplete = func() bool {
   513  		// upgrade is in progress
   514  		return false
   515  	}
   516  	info := s.newServer(c).Info
   517  
   518  	machine, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
   519  		Jobs: []state.MachineJob{state.JobManageModel},
   520  	})
   521  	info.Tag = machine.Tag()
   522  	info.Password = password
   523  	info.Nonce = "nonce"
   524  
   525  	st, err := api.Open(info, fastDialOpts)
   526  	c.Assert(err, jc.ErrorIsNil)
   527  	c.Assert(st.Close(), jc.ErrorIsNil)
   528  }
   529  
   530  func (s *loginSuite) TestControllerAgentLoginDuringMaintenance(c *gc.C) {
   531  	s.cfg.UpgradeComplete = func() bool {
   532  		// upgrade is in progress
   533  		return false
   534  	}
   535  	info := s.newServer(c).Info
   536  
   537  	node, password := s.addController(c)
   538  	info.Tag = node.Tag()
   539  	info.Password = password
   540  
   541  	st, err := api.Open(info, fastDialOpts)
   542  	c.Assert(err, jc.ErrorIsNil)
   543  	c.Assert(st.Close(), jc.ErrorIsNil)
   544  }
   545  
   546  func (s *loginSuite) TestMigratedModelLogin(c *gc.C) {
   547  	modelOwner := s.Factory.MakeUser(c, &factory.UserParams{
   548  		Password: "secret",
   549  	})
   550  	modelState := s.Factory.MakeModel(c, &factory.ModelParams{
   551  		Owner: modelOwner.UserTag(),
   552  	})
   553  	defer modelState.Close()
   554  	model, err := modelState.Model()
   555  	c.Assert(err, jc.ErrorIsNil)
   556  
   557  	controllerTag := names.NewControllerTag(utils.MustNewUUID().String())
   558  
   559  	// Migrate the model and delete it from the state
   560  	mig, err := modelState.CreateMigration(state.MigrationSpec{
   561  		InitiatedBy: names.NewUserTag("admin"),
   562  		TargetInfo: migration.TargetInfo{
   563  			ControllerTag:   controllerTag,
   564  			ControllerAlias: "target",
   565  			Addrs:           []string{"1.2.3.4:5555"},
   566  			CACert:          coretesting.CACert,
   567  			AuthTag:         names.NewUserTag("user2"),
   568  			Password:        "secret",
   569  		},
   570  	})
   571  	c.Assert(err, jc.ErrorIsNil)
   572  	for _, phase := range migration.SuccessfulMigrationPhases() {
   573  		c.Assert(mig.SetPhase(phase), jc.ErrorIsNil)
   574  	}
   575  	c.Assert(model.Destroy(state.DestroyModelParams{}), jc.ErrorIsNil)
   576  	c.Assert(modelState.RemoveDyingModel(), jc.ErrorIsNil)
   577  
   578  	info := s.newServer(c).Info
   579  	info.ModelTag = model.ModelTag()
   580  
   581  	// Attempt to open an API connection to the migrated model as a user
   582  	// that had access to the model before it got migrated. We should still
   583  	// be able to connect to the API but we should get back a Redirect
   584  	// error when we actually try to login.
   585  	info.Tag = modelOwner.Tag()
   586  	info.Password = "secret"
   587  	_, err = api.Open(info, fastDialOpts)
   588  	redirErr, ok := errors.Cause(err).(*api.RedirectError)
   589  	c.Assert(ok, gc.Equals, true)
   590  
   591  	nhp := network.NewMachineHostPorts(5555, "1.2.3.4")
   592  	c.Assert(redirErr.Servers, jc.DeepEquals, []network.MachineHostPorts{nhp})
   593  	c.Assert(redirErr.CACert, gc.Equals, coretesting.CACert)
   594  	c.Assert(redirErr.FollowRedirect, gc.Equals, false)
   595  	c.Assert(redirErr.ControllerTag, gc.Equals, controllerTag)
   596  	c.Assert(redirErr.ControllerAlias, gc.Equals, "target")
   597  
   598  	// Attempt to open an API connection to the migrated model as a user
   599  	// that had NO access to the model before it got migrated. The server
   600  	// should return a not-authorized error when attempting to log in.
   601  	info.Tag = names.NewUserTag("some-other-user")
   602  	_, err = api.Open(info, fastDialOpts)
   603  	c.Assert(params.ErrCode(errors.Cause(err)), gc.Equals, params.CodeUnauthorized)
   604  
   605  	// Attempt to open an API connection to the migrated model as the
   606  	// anonymous user; this should also be allowed on account of CMRs.
   607  	info.Tag = names.NewUserTag(api.AnonymousUsername)
   608  	_, err = api.Open(info, fastDialOpts)
   609  	_, ok = errors.Cause(err).(*api.RedirectError)
   610  	c.Assert(ok, gc.Equals, true)
   611  }
   612  
   613  func (s *loginSuite) TestAnonymousModelLogin(c *gc.C) {
   614  	info := s.newServer(c).Info
   615  	conn := s.openAPIWithoutLogin(c, info)
   616  
   617  	var result params.LoginResult
   618  	request := &params.LoginRequest{
   619  		AuthTag: names.NewUserTag(api.AnonymousUsername).String(),
   620  	}
   621  	err := conn.APICall("Admin", 3, "", "Login", request, &result)
   622  	c.Assert(err, jc.ErrorIsNil)
   623  	c.Assert(result.UserInfo, gc.IsNil)
   624  	c.Assert(result.ControllerTag, gc.Equals, s.State.ControllerTag().String())
   625  	c.Assert(result.ModelTag, gc.Equals, s.Model.ModelTag().String())
   626  	c.Assert(result.Facades, jc.DeepEquals, []params.FacadeVersions{
   627  		{Name: "CrossModelRelations", Versions: []int{2, 3}},
   628  		{Name: "CrossModelSecrets", Versions: []int{1}},
   629  		{Name: "NotifyWatcher", Versions: []int{1}},
   630  		{Name: "OfferStatusWatcher", Versions: []int{1}},
   631  		{Name: "RelationStatusWatcher", Versions: []int{1}},
   632  		{Name: "RelationUnitsWatcher", Versions: []int{1}},
   633  		{Name: "RemoteRelationWatcher", Versions: []int{1}},
   634  		{Name: "SecretsRevisionWatcher", Versions: []int{1}},
   635  		{Name: "StringsWatcher", Versions: []int{1}},
   636  	})
   637  }
   638  
   639  func (s *loginSuite) TestAnonymousControllerLogin(c *gc.C) {
   640  	info := s.newServer(c).Info
   641  	// Zero the model tag so that we log into the controller
   642  	// not the model.
   643  	info.ModelTag = names.ModelTag{}
   644  	conn := s.openAPIWithoutLogin(c, info)
   645  
   646  	var result params.LoginResult
   647  	request := &params.LoginRequest{
   648  		AuthTag:       names.NewUserTag(api.AnonymousUsername).String(),
   649  		ClientVersion: jujuversion.Current.String(),
   650  	}
   651  	err := conn.APICall("Admin", 3, "", "Login", request, &result)
   652  	c.Assert(err, jc.ErrorIsNil)
   653  	c.Assert(result.UserInfo, gc.IsNil)
   654  	c.Assert(result.ControllerTag, gc.Equals, s.State.ControllerTag().String())
   655  	c.Assert(result.Facades, jc.DeepEquals, []params.FacadeVersions{
   656  		{Name: "CrossController", Versions: []int{1}},
   657  		{Name: "NotifyWatcher", Versions: []int{1}},
   658  	})
   659  }
   660  
   661  func (s *loginSuite) TestControllerModel(c *gc.C) {
   662  	info := s.newServer(c).Info
   663  	st := s.openAPIWithoutLogin(c, info)
   664  
   665  	adminUser := s.Owner
   666  	err := st.Login(adminUser, s.AdminPassword, "", nil)
   667  	c.Assert(err, jc.ErrorIsNil)
   668  
   669  	s.assertRemoteModel(c, st, s.Model.ModelTag())
   670  }
   671  
   672  func (s *loginSuite) TestControllerModelBadCreds(c *gc.C) {
   673  	info := s.newServer(c).Info
   674  	st := s.openAPIWithoutLogin(c, info)
   675  
   676  	adminUser := s.Owner
   677  	err := st.Login(adminUser, "bad-password", "", nil)
   678  	assertInvalidEntityPassword(c, err)
   679  }
   680  
   681  func (s *loginSuite) TestNonExistentModel(c *gc.C) {
   682  	info := s.newServer(c).Info
   683  
   684  	uuid, err := utils.NewUUID()
   685  	c.Assert(err, jc.ErrorIsNil)
   686  	info.ModelTag = names.NewModelTag(uuid.String())
   687  	st := s.openAPIWithoutLogin(c, info)
   688  
   689  	adminUser := s.Owner
   690  	err = st.Login(adminUser, s.AdminPassword, "", nil)
   691  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
   692  		Message: fmt.Sprintf("unknown model: %q", uuid),
   693  		Code:    "model not found",
   694  	})
   695  }
   696  
   697  func (s *loginSuite) TestInvalidModel(c *gc.C) {
   698  	info := s.newServer(c).Info
   699  	info.ModelTag = names.NewModelTag("rubbish")
   700  	st, err := api.Open(info, fastDialOpts)
   701  	c.Assert(err, gc.ErrorMatches, `unable to connect to API: invalid model UUID "rubbish" \(Bad Request\)`)
   702  	c.Assert(st, gc.IsNil)
   703  }
   704  
   705  func (s *loginSuite) ensureCachedModel(c *gc.C, uuid string) {
   706  	timeout := time.After(testing.LongWait)
   707  	retry := time.After(0)
   708  	for {
   709  		s.WaitForModelWatchersIdle(c, uuid)
   710  		select {
   711  		case <-retry:
   712  			_, err := s.controller.Model(uuid)
   713  			if err == nil {
   714  				return
   715  			}
   716  			if !errors.IsNotFound(err) {
   717  				c.Fatalf("problem getting model from cache: %v", err)
   718  			}
   719  			retry = time.After(testing.ShortWait)
   720  		case <-timeout:
   721  			c.Fatalf("model %v not seen in cache after %v", uuid, testing.LongWait)
   722  		}
   723  	}
   724  }
   725  
   726  func (s *loginSuite) TestOtherModel(c *gc.C) {
   727  	info := s.newServer(c).Info
   728  
   729  	modelOwner := s.Factory.MakeUser(c, nil)
   730  	modelState := s.Factory.MakeModel(c, &factory.ModelParams{
   731  		Owner: modelOwner.UserTag(),
   732  	})
   733  	defer modelState.Close()
   734  	model, err := modelState.Model()
   735  	c.Assert(err, jc.ErrorIsNil)
   736  	info.ModelTag = model.ModelTag()
   737  
   738  	// Ensure that the model has been added to the cache before
   739  	// we try to log in. Otherwise we get stuck waiting for the model
   740  	// to exist, and time isn't moving as we have a test clock.
   741  	s.ensureCachedModel(c, model.UUID())
   742  
   743  	st := s.openAPIWithoutLogin(c, info)
   744  
   745  	err = st.Login(modelOwner.UserTag(), "password", "", nil)
   746  	c.Assert(err, jc.ErrorIsNil)
   747  	s.assertRemoteModel(c, st, model.ModelTag())
   748  }
   749  
   750  func (s *loginSuite) TestMachineLoginOtherModel(c *gc.C) {
   751  	// User credentials are checked against a global user list.
   752  	// Machine credentials are checked against model specific
   753  	// machines, so this makes sure that the credential checking is
   754  	// using the correct state connection.
   755  	info := s.newServer(c).Info
   756  
   757  	modelOwner := s.Factory.MakeUser(c, nil)
   758  	modelState := s.Factory.MakeModel(c, &factory.ModelParams{
   759  		Owner: modelOwner.UserTag(),
   760  		ConfigAttrs: map[string]interface{}{
   761  			"controller": false,
   762  		},
   763  	})
   764  	defer modelState.Close()
   765  
   766  	f2 := factory.NewFactory(modelState, s.StatePool)
   767  	machine, password := f2.MakeMachineReturningPassword(c, &factory.MachineParams{
   768  		Nonce: "test-nonce",
   769  	})
   770  
   771  	model, err := modelState.Model()
   772  	c.Assert(err, jc.ErrorIsNil)
   773  	// Ensure that the model has been added to the cache before
   774  	// we try to log in. Otherwise we get stuck waiting for the model
   775  	// to exist, and time isn't moving as we have a test clock.
   776  	s.ensureCachedModel(c, model.UUID())
   777  
   778  	info.ModelTag = model.ModelTag()
   779  	st := s.openAPIWithoutLogin(c, info)
   780  
   781  	err = st.Login(machine.Tag(), password, "test-nonce", nil)
   782  	c.Assert(err, jc.ErrorIsNil)
   783  }
   784  
   785  func (s *loginSuite) TestMachineLoginOtherModelNotProvisioned(c *gc.C) {
   786  	info := s.newServer(c).Info
   787  
   788  	modelOwner := s.Factory.MakeUser(c, nil)
   789  	modelState := s.Factory.MakeModel(c, &factory.ModelParams{
   790  		Owner: modelOwner.UserTag(),
   791  		ConfigAttrs: map[string]interface{}{
   792  			"controller": false,
   793  		},
   794  	})
   795  	defer modelState.Close()
   796  
   797  	f2 := factory.NewFactory(modelState, s.StatePool)
   798  	machine, password := f2.MakeUnprovisionedMachineReturningPassword(c, &factory.MachineParams{})
   799  
   800  	model, err := modelState.Model()
   801  	c.Assert(err, jc.ErrorIsNil)
   802  	// Ensure that the model has been added to the cache before
   803  	// we try to log in. Otherwise we get stuck waiting for the model
   804  	// to exist, and time isn't moving as we have a test clock.
   805  	s.ensureCachedModel(c, model.UUID())
   806  
   807  	info.ModelTag = model.ModelTag()
   808  	st := s.openAPIWithoutLogin(c, info)
   809  
   810  	// If the agent attempts Login before the provisioner has recorded
   811  	// the machine's nonce in state, then the agent should get back an
   812  	// error with code "not provisioned".
   813  	err = st.Login(machine.Tag(), password, "nonce", nil)
   814  	c.Assert(err, gc.ErrorMatches, `machine 0 not provisioned \(not provisioned\)`)
   815  	c.Assert(err, jc.Satisfies, params.IsCodeNotProvisioned)
   816  }
   817  
   818  func (s *loginSuite) TestOtherModelFromController(c *gc.C) {
   819  	info := s.newServer(c).Info
   820  
   821  	machine, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
   822  		Jobs: []state.MachineJob{state.JobManageModel},
   823  	})
   824  
   825  	modelState := s.Factory.MakeModel(c, nil)
   826  	defer modelState.Close()
   827  	model, err := modelState.Model()
   828  	c.Assert(err, jc.ErrorIsNil)
   829  
   830  	// Ensure that the model has been added to the cache before
   831  	// we try to log in. Otherwise we get stuck waiting for the model
   832  	// to exist, and time isn't moving as we have a test clock.
   833  	s.ensureCachedModel(c, model.UUID())
   834  
   835  	info.ModelTag = model.ModelTag()
   836  	st := s.openAPIWithoutLogin(c, info)
   837  
   838  	err = st.Login(machine.Tag(), password, "nonce", nil)
   839  	c.Assert(err, jc.ErrorIsNil)
   840  }
   841  
   842  func (s *loginSuite) TestOtherModelFromControllerOtherNotProvisioned(c *gc.C) {
   843  	info := s.newServer(c).Info
   844  
   845  	managerMachine, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
   846  		Jobs: []state.MachineJob{state.JobManageModel},
   847  	})
   848  
   849  	// Create a hosted model with an unprovisioned machine that has the
   850  	// same tag as the manager machine.
   851  	hostedModelState := s.Factory.MakeModel(c, nil)
   852  	defer hostedModelState.Close()
   853  	f2 := factory.NewFactory(hostedModelState, s.StatePool)
   854  	workloadMachine, _ := f2.MakeUnprovisionedMachineReturningPassword(c, &factory.MachineParams{})
   855  	c.Assert(managerMachine.Tag(), gc.Equals, workloadMachine.Tag())
   856  
   857  	hostedModel, err := hostedModelState.Model()
   858  	c.Assert(err, jc.ErrorIsNil)
   859  	// Ensure that the model has been added to the cache before
   860  	// we try to log in. Otherwise we get stuck waiting for the model
   861  	// to exist, and time isn't moving as we have a test clock.
   862  	s.ensureCachedModel(c, hostedModel.UUID())
   863  
   864  	info.ModelTag = hostedModel.ModelTag()
   865  	st := s.openAPIWithoutLogin(c, info)
   866  
   867  	// The fact that the machine with the same tag in the hosted
   868  	// model is unprovisioned should not cause the login to fail
   869  	// with "not provisioned", because the passwords don't match.
   870  	err = st.Login(managerMachine.Tag(), password, "nonce", nil)
   871  	c.Assert(err, jc.ErrorIsNil)
   872  }
   873  
   874  func (s *loginSuite) TestOtherModelWhenNotController(c *gc.C) {
   875  	info := s.newServer(c).Info
   876  
   877  	machine, password := s.Factory.MakeMachineReturningPassword(c, nil)
   878  
   879  	modelState := s.Factory.MakeModel(c, nil)
   880  	defer modelState.Close()
   881  
   882  	model, err := modelState.Model()
   883  	c.Assert(err, jc.ErrorIsNil)
   884  	// Ensure that the model has been added to the cache before
   885  	// we try to log in. Otherwise we get stuck waiting for the model
   886  	// to exist, and time isn't moving as we have a test clock.
   887  	s.ensureCachedModel(c, model.UUID())
   888  
   889  	info.ModelTag = model.ModelTag()
   890  	st := s.openAPIWithoutLogin(c, info)
   891  
   892  	err = st.Login(machine.Tag(), password, "nonce", nil)
   893  	assertInvalidEntityPassword(c, err)
   894  }
   895  
   896  func (s *loginSuite) loginLocalUser(c *gc.C, info *api.Info) (*state.User, params.LoginResult) {
   897  	password := "shhh..."
   898  	user := s.Factory.MakeUser(c, &factory.UserParams{
   899  		Password: password,
   900  	})
   901  	conn := s.openAPIWithoutLogin(c, info)
   902  
   903  	var result params.LoginResult
   904  	request := &params.LoginRequest{
   905  		AuthTag:       user.Tag().String(),
   906  		Credentials:   password,
   907  		ClientVersion: jujuversion.Current.String(),
   908  	}
   909  	err := conn.APICall("Admin", 3, "", "Login", request, &result)
   910  	c.Assert(err, jc.ErrorIsNil)
   911  	c.Assert(result.UserInfo, gc.NotNil)
   912  	return user, result
   913  }
   914  
   915  func (s *loginSuite) TestLoginResultLocalUser(c *gc.C) {
   916  	info := s.newServer(c).Info
   917  
   918  	user, result := s.loginLocalUser(c, info)
   919  	c.Check(result.UserInfo.Identity, gc.Equals, user.Tag().String())
   920  	c.Check(result.UserInfo.ControllerAccess, gc.Equals, "login")
   921  	c.Check(result.UserInfo.ModelAccess, gc.Equals, "admin")
   922  }
   923  
   924  func (s *loginSuite) TestLoginResultLocalUserEveryoneCreateOnlyNonLocal(c *gc.C) {
   925  	info := s.newServer(c).Info
   926  
   927  	setEveryoneAccess(c, s.State, s.Owner, permission.SuperuserAccess)
   928  
   929  	user, result := s.loginLocalUser(c, info)
   930  	c.Check(result.UserInfo.Identity, gc.Equals, user.Tag().String())
   931  	c.Check(result.UserInfo.ControllerAccess, gc.Equals, "login")
   932  	c.Check(result.UserInfo.ModelAccess, gc.Equals, "admin")
   933  }
   934  
   935  func (s *loginSuite) assertRemoteModel(c *gc.C, conn api.Connection, expected names.ModelTag) {
   936  	// Look at what the api thinks it has.
   937  	tag, ok := conn.ModelTag()
   938  	c.Assert(ok, jc.IsTrue)
   939  	c.Assert(tag, gc.Equals, expected)
   940  	// Look at what the api Client thinks it has.
   941  	client := modelconfig.NewClient(conn)
   942  
   943  	// The code below is to verify that the API connection is operating on
   944  	// the expected model. We make a change in state on that model, and
   945  	// then check that it is picked up by a call to the API.
   946  
   947  	st, err := s.StatePool.Get(tag.Id())
   948  	c.Assert(err, jc.ErrorIsNil)
   949  	defer st.Release()
   950  
   951  	expectedCons := constraints.MustParse("mem=8G")
   952  	err = st.SetModelConstraints(expectedCons)
   953  	c.Assert(err, jc.ErrorIsNil)
   954  
   955  	cons, err := client.GetModelConstraints()
   956  	c.Assert(err, jc.ErrorIsNil)
   957  	c.Assert(cons, jc.DeepEquals, expectedCons)
   958  }
   959  
   960  func (s *loginSuite) TestLoginUpdatesLastLoginAndConnection(c *gc.C) {
   961  	info := s.newServer(c).Info
   962  
   963  	now := s.Clock.Now().UTC()
   964  
   965  	password := "shhh..."
   966  	user := s.Factory.MakeUser(c, &factory.UserParams{
   967  		Password: password,
   968  	})
   969  
   970  	info.Tag = user.Tag()
   971  	info.Password = password
   972  	apiState, err := api.Open(info, api.DialOpts{})
   973  	c.Assert(err, jc.ErrorIsNil)
   974  	defer apiState.Close()
   975  
   976  	// The user now has last login updated.
   977  	err = user.Refresh()
   978  	c.Assert(err, jc.ErrorIsNil)
   979  	lastLogin, err := user.LastLogin()
   980  	c.Assert(err, jc.ErrorIsNil)
   981  	c.Assert(lastLogin, jc.Almost, now)
   982  
   983  	// The model user is also updated.
   984  	modelUser, err := s.State.UserAccess(user.UserTag(), s.Model.ModelTag())
   985  	c.Assert(err, jc.ErrorIsNil)
   986  	when, err := s.Model.LastModelConnection(modelUser.UserTag)
   987  	c.Assert(err, jc.ErrorIsNil)
   988  	c.Assert(when, jc.Almost, now)
   989  }
   990  
   991  func (s *loginSuite) TestLoginAddsAuditConversationEventually(c *gc.C) {
   992  	log := &servertesting.FakeAuditLog{}
   993  	s.cfg.GetAuditConfig = func() auditlog.Config {
   994  		return auditlog.Config{
   995  			Enabled: true,
   996  			Target:  log,
   997  		}
   998  	}
   999  	info := s.newServer(c).Info
  1000  
  1001  	password := "shhh..."
  1002  	user := s.Factory.MakeUser(c, &factory.UserParams{
  1003  		Password: password,
  1004  	})
  1005  	conn := s.openAPIWithoutLogin(c, info)
  1006  
  1007  	var result params.LoginResult
  1008  	request := &params.LoginRequest{
  1009  		AuthTag:       user.Tag().String(),
  1010  		Credentials:   password,
  1011  		CLIArgs:       "hey you guys",
  1012  		ClientVersion: jujuversion.Current.String(),
  1013  	}
  1014  	loginTime := s.Clock.Now()
  1015  	err := conn.APICall("Admin", 3, "", "Login", request, &result)
  1016  	c.Assert(err, jc.ErrorIsNil)
  1017  	c.Assert(result.UserInfo, gc.NotNil)
  1018  	// Nothing's logged at this point because there haven't been any
  1019  	// interesting requests.
  1020  	log.CheckCallNames(c)
  1021  
  1022  	var addResults params.AddMachinesResults
  1023  	addReq := &params.AddMachines{
  1024  		MachineParams: []params.AddMachineParams{{
  1025  			Jobs: []model.MachineJob{"JobHostUnits"},
  1026  		}},
  1027  	}
  1028  	addMachinesTime := s.Clock.Now()
  1029  	err = conn.APICall("MachineManager", machineManagerFacadeVersion, "", "AddMachines", addReq, &addResults)
  1030  	c.Assert(err, jc.ErrorIsNil)
  1031  
  1032  	log.CheckCallNames(c, "AddConversation", "AddRequest", "AddResponse")
  1033  
  1034  	convo := log.Calls()[0].Args[0].(auditlog.Conversation)
  1035  	mc := jc.NewMultiChecker()
  1036  	mc.AddExpr("_.ConversationID", gc.HasLen, 16)
  1037  	mc.AddExpr("_.ConnectionID", jc.Ignore)
  1038  	mc.AddExpr("_.When", jc.Satisfies, func(s string) bool {
  1039  		t, err := time.Parse(time.RFC3339, s)
  1040  		if err != nil {
  1041  			return false
  1042  		}
  1043  		return math.Abs(t.Sub(loginTime).Seconds()) < 1.0
  1044  	})
  1045  	c.Assert(convo, mc, auditlog.Conversation{
  1046  		Who:       user.Tag().Id(),
  1047  		What:      "hey you guys",
  1048  		ModelName: s.Model.Name(),
  1049  		ModelUUID: s.Model.UUID(),
  1050  	})
  1051  
  1052  	auditReq := log.Calls()[1].Args[0].(auditlog.Request)
  1053  	mc = jc.NewMultiChecker()
  1054  	mc.AddExpr("_.ConversationID", jc.Ignore)
  1055  	mc.AddExpr("_.ConnectionID", jc.Ignore)
  1056  	mc.AddExpr("_.RequestID", jc.Ignore)
  1057  	mc.AddExpr("_.When", jc.Satisfies, func(s string) bool {
  1058  		t, err := time.Parse(time.RFC3339, s)
  1059  		if err != nil {
  1060  			return false
  1061  		}
  1062  		return math.Abs(t.Sub(addMachinesTime).Seconds()) < 1.0
  1063  	})
  1064  	c.Assert(auditReq, mc, auditlog.Request{
  1065  		Facade:  "MachineManager",
  1066  		Method:  "AddMachines",
  1067  		Version: machineManagerFacadeVersion,
  1068  	})
  1069  }
  1070  
  1071  func (s *loginSuite) TestAuditLoggingFailureOnInterestingRequest(c *gc.C) {
  1072  	log := &servertesting.FakeAuditLog{}
  1073  	log.SetErrors(errors.Errorf("bad news bears"))
  1074  	s.cfg.GetAuditConfig = func() auditlog.Config {
  1075  		return auditlog.Config{
  1076  			Enabled: true,
  1077  			Target:  log,
  1078  		}
  1079  	}
  1080  	info := s.newServer(c).Info
  1081  
  1082  	password := "shhh..."
  1083  	user := s.Factory.MakeUser(c, &factory.UserParams{
  1084  		Password: password,
  1085  	})
  1086  	conn := s.openAPIWithoutLogin(c, info)
  1087  
  1088  	var result params.LoginResult
  1089  	request := &params.LoginRequest{
  1090  		AuthTag:       user.Tag().String(),
  1091  		Credentials:   password,
  1092  		CLIArgs:       "hey you guys",
  1093  		ClientVersion: jujuversion.Current.String(),
  1094  	}
  1095  	err := conn.APICall("Admin", 3, "", "Login", request, &result)
  1096  	// No error yet since logging the conversation is deferred until
  1097  	// something happens.
  1098  	c.Assert(err, jc.ErrorIsNil)
  1099  
  1100  	var addResults params.AddMachinesResults
  1101  	addReq := &params.AddMachines{
  1102  		MachineParams: []params.AddMachineParams{{
  1103  			Jobs: []model.MachineJob{"JobHostUnits"},
  1104  		}},
  1105  	}
  1106  	err = conn.APICall("MachineManager", machineManagerFacadeVersion, "", "AddMachines", addReq, &addResults)
  1107  	c.Assert(err, gc.ErrorMatches, "bad news bears")
  1108  }
  1109  
  1110  func (s *loginSuite) TestAuditLoggingUsesExcludeMethods(c *gc.C) {
  1111  	log := &servertesting.FakeAuditLog{}
  1112  	s.cfg.GetAuditConfig = func() auditlog.Config {
  1113  		return auditlog.Config{
  1114  			Enabled:        true,
  1115  			ExcludeMethods: set.NewStrings("MachineManager.AddMachines"),
  1116  			Target:         log,
  1117  		}
  1118  	}
  1119  	info := s.newServer(c).Info
  1120  
  1121  	password := "shhh..."
  1122  	user := s.Factory.MakeUser(c, &factory.UserParams{
  1123  		Password: password,
  1124  	})
  1125  	conn := s.openAPIWithoutLogin(c, info)
  1126  
  1127  	var result params.LoginResult
  1128  	request := &params.LoginRequest{
  1129  		AuthTag:       user.Tag().String(),
  1130  		Credentials:   password,
  1131  		CLIArgs:       "hey you guys",
  1132  		ClientVersion: jujuversion.Current.String(),
  1133  	}
  1134  	err := conn.APICall("Admin", 3, "", "Login", request, &result)
  1135  	c.Assert(err, jc.ErrorIsNil)
  1136  	c.Assert(result.UserInfo, gc.NotNil)
  1137  	// Nothing's logged at this point because there haven't been any
  1138  	// interesting requests.
  1139  	log.CheckCallNames(c)
  1140  
  1141  	var addResults params.AddMachinesResults
  1142  	addReq := &params.AddMachines{
  1143  		MachineParams: []params.AddMachineParams{{
  1144  			Jobs: []model.MachineJob{"JobHostUnits"},
  1145  		}},
  1146  	}
  1147  	err = conn.APICall("MachineManager", machineManagerFacadeVersion, "", "AddMachines", addReq, &addResults)
  1148  	c.Assert(err, jc.ErrorIsNil)
  1149  
  1150  	// Still nothing logged - the AddMachines call has been filtered out.
  1151  	log.CheckCallNames(c)
  1152  
  1153  	// Call something else.
  1154  	destroyReq := &params.DestroyMachinesParams{
  1155  		MachineTags: []string{addResults.Machines[0].Machine},
  1156  	}
  1157  	err = conn.APICall("MachineManager", machineManagerFacadeVersion, "", "DestroyMachineWithParams", destroyReq, nil)
  1158  	c.Assert(err, jc.ErrorIsNil)
  1159  
  1160  	// Now the conversation and both requests are logged.
  1161  	log.CheckCallNames(c, "AddConversation", "AddRequest", "AddResponse", "AddRequest", "AddResponse")
  1162  
  1163  	req1 := log.Calls()[1].Args[0].(auditlog.Request)
  1164  	c.Assert(req1.Facade, gc.Equals, "MachineManager")
  1165  	c.Assert(req1.Method, gc.Equals, "AddMachines")
  1166  
  1167  	req2 := log.Calls()[3].Args[0].(auditlog.Request)
  1168  	c.Assert(req2.Facade, gc.Equals, "MachineManager")
  1169  	c.Assert(req2.Method, gc.Equals, "DestroyMachineWithParams")
  1170  }
  1171  
  1172  var _ = gc.Suite(&macaroonLoginSuite{})
  1173  
  1174  type macaroonLoginSuite struct {
  1175  	apitesting.MacaroonSuite
  1176  }
  1177  
  1178  func (s *macaroonLoginSuite) TestPublicKeyLocatorErrorIsNotPersistent(c *gc.C) {
  1179  	const remoteUser = "test@somewhere"
  1180  	s.AddModelUser(c, remoteUser)
  1181  	s.AddControllerUser(c, remoteUser, permission.LoginAccess)
  1182  	s.DischargerLogin = func() string {
  1183  		return "test@somewhere"
  1184  	}
  1185  	srv := testserver.NewServer(c, s.StatePool, s.Controller)
  1186  	defer assertStop(c, srv)
  1187  	workingTransport := http.DefaultTransport
  1188  	failingTransport := errorTransport{
  1189  		fallback: workingTransport,
  1190  		location: s.DischargerLocation(),
  1191  		err:      errors.New("some error"),
  1192  	}
  1193  	s.PatchValue(&http.DefaultTransport, failingTransport)
  1194  	_, err := s.login(c, srv.Info)
  1195  	c.Assert(err, gc.ErrorMatches, `.*: some error .*`)
  1196  
  1197  	http.DefaultTransport = workingTransport
  1198  
  1199  	// The error doesn't stick around.
  1200  	_, err = s.login(c, srv.Info)
  1201  	c.Assert(err, jc.ErrorIsNil)
  1202  
  1203  	// Once we've succeeded, we shouldn't try again.
  1204  	http.DefaultTransport = failingTransport
  1205  
  1206  	_, err = s.login(c, srv.Info)
  1207  	c.Assert(err, jc.ErrorIsNil)
  1208  }
  1209  
  1210  func (s *macaroonLoginSuite) login(c *gc.C, info *api.Info) (params.LoginResult, error) {
  1211  	cookieJar := apitesting.NewClearableCookieJar()
  1212  
  1213  	infoSkipLogin := *info
  1214  	infoSkipLogin.SkipLogin = true
  1215  	infoSkipLogin.Macaroons = nil
  1216  	client := s.OpenAPI(c, &infoSkipLogin, cookieJar)
  1217  	defer client.Close()
  1218  
  1219  	var (
  1220  		request params.LoginRequest
  1221  		result  params.LoginResult
  1222  	)
  1223  	err := client.APICall("Admin", 3, "", "Login", &request, &result)
  1224  	if err != nil {
  1225  		return params.LoginResult{}, errors.Annotatef(err, "cannot log in")
  1226  	}
  1227  
  1228  	cookieURL := &url.URL{
  1229  		Scheme: "https",
  1230  		Host:   "localhost",
  1231  		Path:   "/",
  1232  	}
  1233  
  1234  	bakeryClient := httpbakery.NewClient()
  1235  
  1236  	mac := result.BakeryDischargeRequired
  1237  	if mac == nil {
  1238  		var err error
  1239  		mac, err = bakery.NewLegacyMacaroon(result.DischargeRequired)
  1240  		c.Assert(err, jc.ErrorIsNil)
  1241  	}
  1242  	err = bakeryClient.HandleError(context.Background(), cookieURL, &httpbakery.Error{
  1243  		Message: result.DischargeRequiredReason,
  1244  		Code:    httpbakery.ErrDischargeRequired,
  1245  		Info: &httpbakery.ErrorInfo{
  1246  			Macaroon:     mac,
  1247  			MacaroonPath: "/",
  1248  		},
  1249  	})
  1250  	c.Assert(err, jc.ErrorIsNil)
  1251  	// Add the macaroons that have been saved by HandleError to our login request.
  1252  	request.Macaroons = httpbakery.MacaroonsForURL(bakeryClient.Client.Jar, cookieURL)
  1253  
  1254  	err = client.APICall("Admin", 3, "", "Login", &request, &result)
  1255  	return result, err
  1256  }
  1257  
  1258  func (s *macaroonLoginSuite) TestRemoteUserLoginToControllerNoAccess(c *gc.C) {
  1259  	s.DischargerLogin = func() string {
  1260  		return "test@somewhere"
  1261  	}
  1262  	info := s.APIInfo(c)
  1263  	// Log in to the controller, not the model.
  1264  	info.ModelTag = names.ModelTag{}
  1265  
  1266  	_, err := s.login(c, info)
  1267  	assertPermissionDenied(c, err)
  1268  }
  1269  
  1270  func (s *macaroonLoginSuite) TestRemoteUserLoginToControllerLoginAccess(c *gc.C) {
  1271  	setEveryoneAccess(c, s.State, s.AdminUserTag(c), permission.LoginAccess)
  1272  	const remoteUser = "test@somewhere"
  1273  	var remoteUserTag = names.NewUserTag(remoteUser)
  1274  
  1275  	s.DischargerLogin = func() string {
  1276  		return remoteUser
  1277  	}
  1278  	info := s.APIInfo(c)
  1279  	// Log in to the controller, not the model.
  1280  	info.ModelTag = names.ModelTag{}
  1281  
  1282  	result, err := s.login(c, info)
  1283  	c.Check(err, jc.ErrorIsNil)
  1284  	c.Assert(result.UserInfo, gc.NotNil)
  1285  	c.Check(result.UserInfo.Identity, gc.Equals, remoteUserTag.String())
  1286  	c.Check(result.UserInfo.ControllerAccess, gc.Equals, "login")
  1287  	c.Check(result.UserInfo.ModelAccess, gc.Equals, "")
  1288  	c.Check(result.Servers, gc.DeepEquals, params.FromProviderHostsPorts(parseHostPortsFromAddress(c, info.Addrs...)))
  1289  }
  1290  
  1291  func parseHostPortsFromAddress(c *gc.C, addresses ...string) []network.ProviderHostPorts {
  1292  	hps := make([]network.ProviderHostPorts, len(addresses))
  1293  	for i, add := range addresses {
  1294  		hp, err := network.ParseProviderHostPorts(add)
  1295  		c.Assert(err, jc.ErrorIsNil)
  1296  		hps[i] = hp
  1297  	}
  1298  	return hps
  1299  }
  1300  
  1301  func (s *macaroonLoginSuite) TestRemoteUserLoginToControllerSuperuserAccess(c *gc.C) {
  1302  	setEveryoneAccess(c, s.State, s.AdminUserTag(c), permission.SuperuserAccess)
  1303  	const remoteUser = "test@somewhere"
  1304  	var remoteUserTag = names.NewUserTag(remoteUser)
  1305  
  1306  	s.DischargerLogin = func() string {
  1307  		return remoteUser
  1308  	}
  1309  	info := s.APIInfo(c)
  1310  	// Log in to the controller, not the model.
  1311  	info.ModelTag = names.ModelTag{}
  1312  
  1313  	result, err := s.login(c, info)
  1314  	c.Check(err, jc.ErrorIsNil)
  1315  	c.Assert(result.UserInfo, gc.NotNil)
  1316  	c.Check(result.UserInfo.Identity, gc.Equals, remoteUserTag.String())
  1317  	c.Check(result.UserInfo.ControllerAccess, gc.Equals, "superuser")
  1318  	c.Check(result.UserInfo.ModelAccess, gc.Equals, "")
  1319  }
  1320  
  1321  func (s *macaroonLoginSuite) TestRemoteUserLoginToModelNoExplicitAccess(c *gc.C) {
  1322  	// If we have a remote user which the controller knows nothing about,
  1323  	// and the macaroon is discharged successfully, and the user is attempting
  1324  	// to log into a model, that is permission denied.
  1325  	setEveryoneAccess(c, s.State, s.AdminUserTag(c), permission.LoginAccess)
  1326  	s.DischargerLogin = func() string {
  1327  		return "test@somewhere"
  1328  	}
  1329  	info := s.APIInfo(c)
  1330  
  1331  	_, err := s.login(c, info)
  1332  	assertPermissionDenied(c, err)
  1333  }
  1334  
  1335  func (s *macaroonLoginSuite) TestRemoteUserLoginToModelWithExplicitAccess(c *gc.C) {
  1336  	s.testRemoteUserLoginToModelWithExplicitAccess(c, false)
  1337  }
  1338  
  1339  func (s *macaroonLoginSuite) TestRemoteUserLoginToModelWithExplicitAccessAndAllowModelAccess(c *gc.C) {
  1340  	s.testRemoteUserLoginToModelWithExplicitAccess(c, true)
  1341  }
  1342  
  1343  func (s *macaroonLoginSuite) testRemoteUserLoginToModelWithExplicitAccess(c *gc.C, allowModelAccess bool) {
  1344  	cfg := testserver.DefaultServerConfig(c, nil)
  1345  	cfg.AllowModelAccess = allowModelAccess
  1346  	cfg.Controller = s.Controller
  1347  	srv := testserver.NewServerWithConfig(c, s.StatePool, cfg)
  1348  	defer assertStop(c, srv)
  1349  	srv.Info.ModelTag = s.Model.ModelTag()
  1350  
  1351  	// If we have a remote user which has explicit model access, but neither
  1352  	// controller access nor 'everyone' access, the user will have access
  1353  	// only if the AllowModelAccess configuration flag is true.
  1354  	const remoteUser = "test@somewhere"
  1355  	s.Factory.MakeModelUser(c, &factory.ModelUserParams{
  1356  		User: remoteUser,
  1357  
  1358  		Access: permission.WriteAccess,
  1359  	})
  1360  	s.DischargerLogin = func() string {
  1361  		return remoteUser
  1362  	}
  1363  
  1364  	_, err := s.login(c, srv.Info)
  1365  	if allowModelAccess {
  1366  		c.Assert(err, jc.ErrorIsNil)
  1367  	} else {
  1368  		assertPermissionDenied(c, err)
  1369  	}
  1370  }
  1371  
  1372  func (s *macaroonLoginSuite) TestRemoteUserLoginToModelWithControllerAccess(c *gc.C) {
  1373  	const remoteUser = "test@somewhere"
  1374  	var remoteUserTag = names.NewUserTag(remoteUser)
  1375  	s.Factory.MakeModelUser(c, &factory.ModelUserParams{
  1376  		User:   remoteUser,
  1377  		Access: permission.WriteAccess,
  1378  	})
  1379  	s.AddControllerUser(c, remoteUser, permission.SuperuserAccess)
  1380  
  1381  	s.DischargerLogin = func() string {
  1382  		return remoteUser
  1383  	}
  1384  	info := s.APIInfo(c)
  1385  
  1386  	result, err := s.login(c, info)
  1387  	c.Check(err, jc.ErrorIsNil)
  1388  	c.Assert(result.UserInfo, gc.NotNil)
  1389  	c.Check(result.UserInfo.Identity, gc.Equals, remoteUserTag.String())
  1390  	c.Check(result.UserInfo.ControllerAccess, gc.Equals, "superuser")
  1391  	c.Check(result.UserInfo.ModelAccess, gc.Equals, "write")
  1392  }
  1393  
  1394  func (s *macaroonLoginSuite) TestLoginToModelSuccess(c *gc.C) {
  1395  	const remoteUser = "test@somewhere"
  1396  	s.AddModelUser(c, remoteUser)
  1397  	s.AddControllerUser(c, remoteUser, permission.LoginAccess)
  1398  	s.DischargerLogin = func() string {
  1399  		return "test@somewhere"
  1400  	}
  1401  	loggo.GetLogger("juju.apiserver").SetLogLevel(loggo.TRACE)
  1402  	client, err := api.Open(s.APIInfo(c), api.DialOpts{})
  1403  	c.Assert(err, jc.ErrorIsNil)
  1404  	defer client.Close()
  1405  
  1406  	// The auth tag has been correctly returned by the server.
  1407  	c.Assert(client.AuthTag(), gc.Equals, names.NewUserTag(remoteUser))
  1408  }
  1409  
  1410  func (s *macaroonLoginSuite) TestFailedToObtainDischargeLogin(c *gc.C) {
  1411  	s.DischargerLogin = func() string {
  1412  		return ""
  1413  	}
  1414  	client, err := api.Open(s.APIInfo(c), api.DialOpts{})
  1415  	c.Assert(err, gc.ErrorMatches, `cannot get discharge from "https://.*": third party refused discharge: cannot discharge: login denied by discharger`)
  1416  	c.Assert(client, gc.Equals, nil)
  1417  }
  1418  
  1419  func assertInvalidEntityPassword(c *gc.C, err error) {
  1420  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
  1421  		Message: "invalid entity name or password",
  1422  		Code:    "unauthorized access",
  1423  	})
  1424  }
  1425  
  1426  func assertPermissionDenied(c *gc.C, err error) {
  1427  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
  1428  		Message: "permission denied",
  1429  		Code:    "unauthorized access",
  1430  	})
  1431  }
  1432  
  1433  func setEveryoneAccess(c *gc.C, st *state.State, adminUser names.UserTag, access permission.Access) {
  1434  	err := controller.ChangeControllerAccess(
  1435  		st, adminUser, names.NewUserTag(common.EveryoneTagName),
  1436  		params.GrantControllerAccess, access)
  1437  	c.Assert(err, jc.ErrorIsNil)
  1438  }
  1439  
  1440  var _ = gc.Suite(&migrationSuite{})
  1441  
  1442  type migrationSuite struct {
  1443  	baseLoginSuite
  1444  }
  1445  
  1446  func (s *migrationSuite) TestImportingModel(c *gc.C) {
  1447  	m, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
  1448  		Nonce: "nonce",
  1449  	})
  1450  	model, err := s.State.Model()
  1451  	c.Assert(err, jc.ErrorIsNil)
  1452  	err = model.SetMigrationMode(state.MigrationModeImporting)
  1453  	c.Assert(err, jc.ErrorIsNil)
  1454  
  1455  	// Users should be able to log in but RPC requests should fail.
  1456  	info := s.APIInfo(c)
  1457  	userConn := s.OpenAPIAs(c, info.Tag, info.Password)
  1458  	defer userConn.Close()
  1459  	_, err = apiclient.NewClient(userConn, coretesting.NoopLogger{}).Status(nil)
  1460  	c.Check(err, gc.ErrorMatches, "migration in progress, model is importing")
  1461  
  1462  	// Machines should be able to use the API.
  1463  	machineConn := s.OpenAPIAsMachine(c, m.Tag(), password, "nonce")
  1464  	defer machineConn.Close()
  1465  	_, err = apimachiner.NewState(machineConn).Machine(m.MachineTag())
  1466  	c.Check(err, jc.ErrorIsNil)
  1467  }
  1468  
  1469  func (s *migrationSuite) TestExportingModel(c *gc.C) {
  1470  	model, err := s.State.Model()
  1471  	c.Assert(err, jc.ErrorIsNil)
  1472  	err = model.SetMigrationMode(state.MigrationModeExporting)
  1473  	c.Assert(err, jc.ErrorIsNil)
  1474  
  1475  	// Users should be able to log in but RPC requests should fail.
  1476  	info := s.APIInfo(c)
  1477  	userConn := s.OpenAPIAs(c, info.Tag, info.Password)
  1478  	defer userConn.Close()
  1479  
  1480  	// Status is fine.
  1481  	_, err = apiclient.NewClient(userConn, coretesting.NoopLogger{}).Status(nil)
  1482  	c.Check(err, jc.ErrorIsNil)
  1483  
  1484  	// Modifying commands like destroy machines are not.
  1485  	_, err = machineclient.NewClient(userConn).DestroyMachinesWithParams(false, false, false, nil, "42")
  1486  	c.Check(err, gc.ErrorMatches, "model migration in progress")
  1487  }
  1488  
  1489  type loginV3Suite struct {
  1490  	baseLoginSuite
  1491  }
  1492  
  1493  var _ = gc.Suite(&loginV3Suite{})
  1494  
  1495  func (s *loginV3Suite) TestClientLoginToModel(c *gc.C) {
  1496  	info := s.APIInfo(c)
  1497  	apiState, err := api.Open(info, api.DialOpts{})
  1498  	c.Assert(err, jc.ErrorIsNil)
  1499  	defer apiState.Close()
  1500  
  1501  	client := modelconfig.NewClient(apiState)
  1502  	_, err = client.GetModelConstraints()
  1503  	c.Assert(err, jc.ErrorIsNil)
  1504  }
  1505  
  1506  func (s *loginV3Suite) TestClientLoginToController(c *gc.C) {
  1507  	info := s.APIInfo(c)
  1508  	info.ModelTag = names.ModelTag{}
  1509  	apiState, err := api.Open(info, api.DialOpts{})
  1510  	c.Assert(err, jc.ErrorIsNil)
  1511  	defer apiState.Close()
  1512  
  1513  	client := machineclient.NewClient(apiState)
  1514  	_, err = client.RetryProvisioning(false, names.NewMachineTag("machine-0"))
  1515  	c.Assert(errors.Cause(err), gc.DeepEquals, &rpc.RequestError{
  1516  		Message: `facade "MachineManager" not supported for controller API connection`,
  1517  		Code:    "not supported",
  1518  	})
  1519  }
  1520  
  1521  func (s *loginV3Suite) TestClientLoginToControllerNoAccessToControllerModel(c *gc.C) {
  1522  	password := "shhh..."
  1523  	user := s.Factory.MakeUser(c, &factory.UserParams{
  1524  		NoModelUser: true,
  1525  		Password:    password,
  1526  	})
  1527  
  1528  	info := s.APIInfo(c)
  1529  	info.Tag = user.Tag()
  1530  	info.Password = password
  1531  	info.ModelTag = names.ModelTag{}
  1532  	apiState, err := api.Open(info, api.DialOpts{})
  1533  	c.Assert(err, jc.ErrorIsNil)
  1534  	defer apiState.Close()
  1535  	// The user now has last login updated.
  1536  	err = user.Refresh()
  1537  	c.Assert(err, jc.ErrorIsNil)
  1538  	lastLogin, err := user.LastLogin()
  1539  	c.Assert(err, jc.ErrorIsNil)
  1540  	c.Assert(lastLogin, gc.NotNil)
  1541  }
  1542  
  1543  func (s *loginV3Suite) TestClientLoginToRootOldClient(c *gc.C) {
  1544  	info := s.APIInfo(c)
  1545  	info.Tag = nil
  1546  	info.Password = ""
  1547  	info.ModelTag = names.ModelTag{}
  1548  	info.SkipLogin = true
  1549  	apiState, err := api.Open(info, api.DialOpts{})
  1550  	c.Assert(err, jc.ErrorIsNil)
  1551  
  1552  	err = apiState.APICall("Admin", 2, "", "Login", struct{}{}, nil)
  1553  	c.Assert(err, gc.ErrorMatches, ".*this version of Juju does not support login from old clients.*")
  1554  }
  1555  
  1556  // errorTransport implements http.RoundTripper by always
  1557  // returning the given error from RoundTrip when it visits
  1558  // the given URL (otherwise it uses the fallback transport.
  1559  type errorTransport struct {
  1560  	err      error
  1561  	location string
  1562  	fallback http.RoundTripper
  1563  }
  1564  
  1565  func (t errorTransport) RoundTrip(req *http.Request) (*http.Response, error) {
  1566  	if req.URL.String() == t.location+"/publickey" {
  1567  		if req.Body != nil {
  1568  			req.Body.Close()
  1569  		}
  1570  		return nil, t.err
  1571  	}
  1572  	if req.URL.String() == t.location+"/discharge/info" {
  1573  		if req.Body != nil {
  1574  			req.Body.Close()
  1575  		}
  1576  		return &http.Response{
  1577  			Request:    req,
  1578  			StatusCode: http.StatusNotFound,
  1579  			Header:     http.Header{"Content-Type": {"application/text"}},
  1580  			Body:       io.NopCloser(bytes.NewReader([]byte(""))),
  1581  		}, nil
  1582  	}
  1583  	return t.fallback.RoundTrip(req)
  1584  }