github.com/niedbalski/juju@v0.0.0-20190215020005-8ff100488e47/apiserver/facades/controller/crossmodelrelations/crossmodelrelations.go (about)

     1  // Copyright 2017 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package crossmodelrelations
     5  
     6  import (
     7  	"strings"
     8  	"sync"
     9  
    10  	"github.com/juju/errors"
    11  	"github.com/juju/loggo"
    12  	"gopkg.in/juju/charm.v6"
    13  	"gopkg.in/juju/names.v2"
    14  	"gopkg.in/macaroon.v2-unstable"
    15  
    16  	"github.com/juju/juju/apiserver/common"
    17  	commoncrossmodel "github.com/juju/juju/apiserver/common/crossmodel"
    18  	"github.com/juju/juju/apiserver/common/firewall"
    19  	"github.com/juju/juju/apiserver/facade"
    20  	"github.com/juju/juju/apiserver/params"
    21  	"github.com/juju/juju/state"
    22  	"github.com/juju/juju/state/watcher"
    23  )
    24  
    25  var logger = loggo.GetLogger("juju.apiserver.crossmodelrelations")
    26  
    27  type egressAddressWatcherFunc func(facade.Resources, firewall.State, params.Entities) (params.StringsWatchResults, error)
    28  type relationStatusWatcherFunc func(CrossModelRelationsState, names.RelationTag) (state.StringsWatcher, error)
    29  type offerStatusWatcherFunc func(CrossModelRelationsState, string) (OfferWatcher, error)
    30  
    31  // CrossModelRelationsAPI provides access to the CrossModelRelations API facade.
    32  type CrossModelRelationsAPI struct {
    33  	st         CrossModelRelationsState
    34  	fw         firewall.State
    35  	resources  facade.Resources
    36  	authorizer facade.Authorizer
    37  
    38  	mu              sync.Mutex
    39  	authCtxt        *commoncrossmodel.AuthContext
    40  	relationToOffer map[string]string
    41  
    42  	egressAddressWatcher  egressAddressWatcherFunc
    43  	relationStatusWatcher relationStatusWatcherFunc
    44  	offerStatusWatcher    offerStatusWatcherFunc
    45  }
    46  
    47  // NewStateCrossModelRelationsAPI creates a new server-side CrossModelRelations API facade
    48  // backed by global state.
    49  func NewStateCrossModelRelationsAPI(ctx facade.Context) (*CrossModelRelationsAPI, error) {
    50  	authCtxt := ctx.Resources().Get("offerAccessAuthContext").(common.ValueResource).Value
    51  	st := ctx.State()
    52  	model, err := st.Model()
    53  	if err != nil {
    54  		return nil, err
    55  	}
    56  
    57  	return NewCrossModelRelationsAPI(
    58  		stateShim{
    59  			st:      st,
    60  			Backend: commoncrossmodel.GetBackend(st),
    61  		},
    62  		firewall.StateShim(st, model),
    63  		ctx.Resources(), ctx.Auth(), authCtxt.(*commoncrossmodel.AuthContext),
    64  		firewall.WatchEgressAddressesForRelations,
    65  		watchRelationLifeSuspendedStatus,
    66  		watchOfferStatus,
    67  	)
    68  }
    69  
    70  // NewCrossModelRelationsAPI returns a new server-side CrossModelRelationsAPI facade.
    71  func NewCrossModelRelationsAPI(
    72  	st CrossModelRelationsState,
    73  	fw firewall.State,
    74  	resources facade.Resources,
    75  	authorizer facade.Authorizer,
    76  	authCtxt *commoncrossmodel.AuthContext,
    77  	egressAddressWatcher egressAddressWatcherFunc,
    78  	relationStatusWatcher relationStatusWatcherFunc,
    79  	offerStatusWatcher offerStatusWatcherFunc,
    80  ) (*CrossModelRelationsAPI, error) {
    81  	return &CrossModelRelationsAPI{
    82  		st:                    st,
    83  		fw:                    fw,
    84  		resources:             resources,
    85  		authorizer:            authorizer,
    86  		authCtxt:              authCtxt,
    87  		egressAddressWatcher:  egressAddressWatcher,
    88  		relationStatusWatcher: relationStatusWatcher,
    89  		offerStatusWatcher:    offerStatusWatcher,
    90  		relationToOffer:       make(map[string]string),
    91  	}, nil
    92  }
    93  
    94  func (api *CrossModelRelationsAPI) checkMacaroonsForRelation(relationTag names.Tag, mac macaroon.Slice) error {
    95  	api.mu.Lock()
    96  	defer api.mu.Unlock()
    97  
    98  	offerUUID, ok := api.relationToOffer[relationTag.Id()]
    99  	if !ok {
   100  		oc, err := api.st.OfferConnectionForRelation(relationTag.Id())
   101  		if err != nil {
   102  			return errors.Trace(err)
   103  		}
   104  		offerUUID = oc.OfferUUID()
   105  	}
   106  	auth := api.authCtxt.Authenticator(api.st.ModelUUID(), offerUUID)
   107  	return auth.CheckRelationMacaroons(relationTag, mac)
   108  }
   109  
   110  // PublishRelationChanges publishes relation changes to the
   111  // model hosting the remote application involved in the relation.
   112  func (api *CrossModelRelationsAPI) PublishRelationChanges(
   113  	changes params.RemoteRelationsChanges,
   114  ) (params.ErrorResults, error) {
   115  	results := params.ErrorResults{
   116  		Results: make([]params.ErrorResult, len(changes.Changes)),
   117  	}
   118  	for i, change := range changes.Changes {
   119  		relationTag, err := api.st.GetRemoteEntity(change.RelationToken)
   120  		if err != nil {
   121  			if errors.IsNotFound(err) {
   122  				logger.Debugf("no relation tag %+v in model %v, exit early", change.RelationToken, api.st.ModelUUID())
   123  				continue
   124  			}
   125  			results.Results[i].Error = common.ServerError(err)
   126  			continue
   127  		}
   128  		logger.Debugf("relation tag for token %+v is %v", change.RelationToken, relationTag)
   129  		if err := api.checkMacaroonsForRelation(relationTag, change.Macaroons); err != nil {
   130  			results.Results[i].Error = common.ServerError(err)
   131  			continue
   132  		}
   133  		if err := commoncrossmodel.PublishRelationChange(api.st, relationTag, change); err != nil {
   134  			results.Results[i].Error = common.ServerError(err)
   135  			continue
   136  		}
   137  		if change.Life != params.Alive {
   138  			delete(api.relationToOffer, relationTag.Id())
   139  		}
   140  	}
   141  	return results, nil
   142  }
   143  
   144  // RegisterRemoteRelationArgs sets up the model to participate
   145  // in the specified relations. This operation is idempotent.
   146  func (api *CrossModelRelationsAPI) RegisterRemoteRelations(
   147  	relations params.RegisterRemoteRelationArgs,
   148  ) (params.RegisterRemoteRelationResults, error) {
   149  	results := params.RegisterRemoteRelationResults{
   150  		Results: make([]params.RegisterRemoteRelationResult, len(relations.Relations)),
   151  	}
   152  	for i, relation := range relations.Relations {
   153  		id, err := api.registerRemoteRelation(relation)
   154  		results.Results[i].Result = id
   155  		results.Results[i].Error = common.ServerError(err)
   156  	}
   157  	return results, nil
   158  }
   159  
   160  func (api *CrossModelRelationsAPI) registerRemoteRelation(relation params.RegisterRemoteRelationArg) (*params.RemoteRelationDetails, error) {
   161  	logger.Debugf("register remote relation %+v", relation)
   162  	// TODO(wallyworld) - do this as a transaction so the result is atomic
   163  	// Perform some initial validation - is the local application alive?
   164  
   165  	// Look up the offer record so get the local application to which we need to relate.
   166  	appOffer, err := api.st.ApplicationOfferForUUID(relation.OfferUUID)
   167  	if err != nil {
   168  		return nil, errors.Trace(err)
   169  	}
   170  
   171  	// Check that the supplied macaroon allows access.
   172  	auth := api.authCtxt.Authenticator(api.st.ModelUUID(), appOffer.OfferUUID)
   173  	attr, err := auth.CheckOfferMacaroons(appOffer.OfferUUID, relation.Macaroons)
   174  	if err != nil {
   175  		return nil, err
   176  	}
   177  	// The macaroon needs to be attenuated to a user.
   178  	username, ok := attr["username"]
   179  	if username == "" || !ok {
   180  		return nil, common.ErrPerm
   181  	}
   182  	localApplicationName := appOffer.ApplicationName
   183  	localApp, err := api.st.Application(localApplicationName)
   184  	if err != nil {
   185  		return nil, errors.Annotatef(err, "cannot get application for offer %q", relation.OfferUUID)
   186  	}
   187  	if localApp.Life() != state.Alive {
   188  		// We don't want to leak the application name so just log it.
   189  		logger.Warningf("local application for offer %v not found", localApplicationName)
   190  		return nil, errors.NotFoundf("local application for offer %v", relation.OfferUUID)
   191  	}
   192  	eps, err := localApp.Endpoints()
   193  	if err != nil {
   194  		return nil, errors.Trace(err)
   195  	}
   196  
   197  	// Does the requested local endpoint exist?
   198  	var localEndpoint *state.Endpoint
   199  	for _, ep := range eps {
   200  		if ep.Name == relation.LocalEndpointName {
   201  			localEndpoint = &ep
   202  			break
   203  		}
   204  	}
   205  	if localEndpoint == nil {
   206  		return nil, errors.NotFoundf("relation endpoint %v", relation.LocalEndpointName)
   207  	}
   208  
   209  	// Add the remote application reference. We construct a unique, opaque application name based on the
   210  	// token passed in from the consuming model. This model, which is offering the application being
   211  	// related to, does not need to know the name of the consuming application.
   212  	uniqueRemoteApplicationName := "remote-" + strings.Replace(relation.ApplicationToken, "-", "", -1)
   213  	remoteEndpoint := state.Endpoint{
   214  		ApplicationName: uniqueRemoteApplicationName,
   215  		Relation: charm.Relation{
   216  			Name:      relation.RemoteEndpoint.Name,
   217  			Interface: relation.RemoteEndpoint.Interface,
   218  			Role:      relation.RemoteEndpoint.Role,
   219  		},
   220  	}
   221  
   222  	sourceModelTag, err := names.ParseModelTag(relation.SourceModelTag)
   223  	if err != nil {
   224  		return nil, errors.Trace(err)
   225  	}
   226  	_, err = api.st.AddRemoteApplication(state.AddRemoteApplicationParams{
   227  		Name:            uniqueRemoteApplicationName,
   228  		OfferUUID:       relation.OfferUUID,
   229  		SourceModel:     sourceModelTag,
   230  		Token:           relation.ApplicationToken,
   231  		Endpoints:       []charm.Relation{remoteEndpoint.Relation},
   232  		IsConsumerProxy: true,
   233  	})
   234  	// If it already exists, that's fine.
   235  	if err != nil && !errors.IsAlreadyExists(err) {
   236  		return nil, errors.Annotatef(err, "adding remote application %v", uniqueRemoteApplicationName)
   237  	}
   238  	logger.Debugf("added remote application %v to local model with token %v from model %v", uniqueRemoteApplicationName, relation.ApplicationToken, sourceModelTag.Id())
   239  
   240  	// Now add the relation if it doesn't already exist.
   241  	localRel, err := api.st.EndpointsRelation(*localEndpoint, remoteEndpoint)
   242  	if err != nil && !errors.IsNotFound(err) {
   243  		return nil, errors.Trace(err)
   244  	}
   245  	if err != nil { // not found
   246  		localRel, err = api.st.AddRelation(*localEndpoint, remoteEndpoint)
   247  		// Again, if it already exists, that's fine.
   248  		if err != nil && !errors.IsAlreadyExists(err) {
   249  			return nil, errors.Annotate(err, "adding remote relation")
   250  		}
   251  		logger.Debugf("added relation %v to model %v", localRel.Tag().Id(), api.st.ModelUUID())
   252  	}
   253  	_, err = api.st.AddOfferConnection(state.AddOfferConnectionParams{
   254  		SourceModelUUID: sourceModelTag.Id(), Username: username,
   255  		OfferUUID:   appOffer.OfferUUID,
   256  		RelationId:  localRel.Id(),
   257  		RelationKey: localRel.Tag().Id(),
   258  	})
   259  	if err != nil && !errors.IsAlreadyExists(err) {
   260  		return nil, errors.Annotate(err, "adding offer connection details")
   261  	}
   262  	api.relationToOffer[localRel.Tag().Id()] = relation.OfferUUID
   263  
   264  	// Ensure we have references recorded.
   265  	logger.Debugf("importing remote relation into model %v", api.st.ModelUUID())
   266  	logger.Debugf("remote model is %v", sourceModelTag.Id())
   267  
   268  	err = api.st.ImportRemoteEntity(localRel.Tag(), relation.RelationToken)
   269  	if err != nil && !errors.IsAlreadyExists(err) {
   270  		return nil, errors.Annotatef(err, "importing remote relation %v to local model", localRel.Tag().Id())
   271  	}
   272  	logger.Debugf("relation token %v exported for %v ", relation.RelationToken, localRel.Tag().Id())
   273  
   274  	// Export the local offer from this model so we can tell the caller what the remote id is.
   275  	// The offer is exported as an application name since it models the behaviour of an application
   276  	// as far as the consuming side is concerned, and also needs to be unique.
   277  	// This allows > 1 offers off the one application to be made.
   278  	// NB we need to export the application last so that everything else is in place when the worker is
   279  	// woken up by the watcher.
   280  
   281  	// Juju 2.5.1 and earlier exported the local application name so for backwards compatibility
   282  	// use that if it's there.
   283  	token, err := api.st.GetToken(names.NewApplicationTag(localApplicationName))
   284  	if err != nil && !errors.IsNotFound(err) {
   285  		return nil, errors.Annotatef(err, "checking local application token for %v", localApplicationName)
   286  	}
   287  	if err != nil {
   288  		// No token yet so export using the offer name which we prefer.
   289  		token, err = api.st.ExportLocalEntity(names.NewApplicationTag(appOffer.OfferName))
   290  		if err != nil && !errors.IsAlreadyExists(err) {
   291  			return nil, errors.Annotatef(err, "exporting local application offer %v", appOffer.OfferName)
   292  		}
   293  	}
   294  	logger.Debugf("local application offer %v from model %v exported with token %v ", appOffer.OfferName, api.st.ModelUUID(), token)
   295  
   296  	// Mint a new macaroon attenuated to the actual relation.
   297  	relationMacaroon, err := api.authCtxt.CreateRemoteRelationMacaroon(
   298  		api.st.ModelUUID(), relation.OfferUUID, username, localRel.Tag())
   299  	if err != nil {
   300  		return nil, errors.Annotate(err, "creating relation macaroon")
   301  	}
   302  	return &params.RemoteRelationDetails{
   303  		Token:    token,
   304  		Macaroon: relationMacaroon,
   305  	}, nil
   306  }
   307  
   308  // WatchRelationUnits starts a RelationUnitsWatcher for watching the
   309  // relation units involved in each specified relation, and returns the
   310  // watcher IDs and initial values, or an error if the relation units could not be watched.
   311  func (api *CrossModelRelationsAPI) WatchRelationUnits(remoteRelationArgs params.RemoteEntityArgs) (params.RelationUnitsWatchResults, error) {
   312  	results := params.RelationUnitsWatchResults{
   313  		Results: make([]params.RelationUnitsWatchResult, len(remoteRelationArgs.Args)),
   314  	}
   315  	for i, arg := range remoteRelationArgs.Args {
   316  		relationTag, err := api.st.GetRemoteEntity(arg.Token)
   317  		if err != nil {
   318  			results.Results[i].Error = common.ServerError(err)
   319  			continue
   320  		}
   321  		if err := api.checkMacaroonsForRelation(relationTag, arg.Macaroons); err != nil {
   322  			results.Results[i].Error = common.ServerError(err)
   323  			continue
   324  		}
   325  		w, err := commoncrossmodel.WatchRelationUnits(api.st, relationTag.(names.RelationTag))
   326  		if err != nil {
   327  			results.Results[i].Error = common.ServerError(err)
   328  			continue
   329  		}
   330  		changes, ok := <-w.Changes()
   331  		if !ok {
   332  			results.Results[i].Error = common.ServerError(watcher.EnsureErr(w))
   333  			continue
   334  		}
   335  		results.Results[i].RelationUnitsWatcherId = api.resources.Register(w)
   336  		results.Results[i].Changes = changes
   337  	}
   338  	return results, nil
   339  }
   340  
   341  // RelationUnitSettings returns the relation unit settings for the given relation units.
   342  func (api *CrossModelRelationsAPI) RelationUnitSettings(relationUnits params.RemoteRelationUnits) (params.SettingsResults, error) {
   343  	results := params.SettingsResults{
   344  		Results: make([]params.SettingsResult, len(relationUnits.RelationUnits)),
   345  	}
   346  	for i, arg := range relationUnits.RelationUnits {
   347  		relationTag, err := api.st.GetRemoteEntity(arg.RelationToken)
   348  		if err != nil {
   349  			results.Results[i].Error = common.ServerError(err)
   350  			continue
   351  		}
   352  		if err := api.checkMacaroonsForRelation(relationTag, arg.Macaroons); err != nil {
   353  			results.Results[i].Error = common.ServerError(err)
   354  			continue
   355  		}
   356  
   357  		ru := params.RelationUnit{
   358  			Relation: relationTag.String(),
   359  			Unit:     arg.Unit,
   360  		}
   361  		settings, err := commoncrossmodel.RelationUnitSettings(api.st, ru)
   362  		if err != nil {
   363  			results.Results[i].Error = common.ServerError(err)
   364  			continue
   365  		}
   366  		results.Results[i].Settings = settings
   367  	}
   368  	return results, nil
   369  }
   370  
   371  func watchRelationLifeSuspendedStatus(st CrossModelRelationsState, tag names.RelationTag) (state.StringsWatcher, error) {
   372  	relation, err := st.KeyRelation(tag.Id())
   373  	if err != nil {
   374  		return nil, errors.Trace(err)
   375  	}
   376  	return relation.WatchLifeSuspendedStatus(), nil
   377  }
   378  
   379  // WatchRelationsSuspendedStatus starts a RelationStatusWatcher for
   380  // watching the life and suspended status of a relation.
   381  func (api *CrossModelRelationsAPI) WatchRelationsSuspendedStatus(
   382  	remoteRelationArgs params.RemoteEntityArgs,
   383  ) (params.RelationStatusWatchResults, error) {
   384  	results := params.RelationStatusWatchResults{
   385  		Results: make([]params.RelationLifeSuspendedStatusWatchResult, len(remoteRelationArgs.Args)),
   386  	}
   387  
   388  	for i, arg := range remoteRelationArgs.Args {
   389  		relationTag, err := api.st.GetRemoteEntity(arg.Token)
   390  		if err != nil {
   391  			results.Results[i].Error = common.ServerError(err)
   392  			continue
   393  		}
   394  		if err := api.checkMacaroonsForRelation(relationTag, arg.Macaroons); err != nil {
   395  			results.Results[i].Error = common.ServerError(err)
   396  			continue
   397  		}
   398  		w, err := api.relationStatusWatcher(api.st, relationTag.(names.RelationTag))
   399  		if err != nil {
   400  			results.Results[i].Error = common.ServerError(err)
   401  			continue
   402  		}
   403  		changes, ok := <-w.Changes()
   404  		if !ok {
   405  			results.Results[i].Error = common.ServerError(watcher.EnsureErr(w))
   406  			continue
   407  		}
   408  		changesParams := make([]params.RelationLifeSuspendedStatusChange, len(changes))
   409  		for j, key := range changes {
   410  			change, err := commoncrossmodel.GetRelationLifeSuspendedStatusChange(api.st, key)
   411  			if err != nil {
   412  				results.Results[i].Error = common.ServerError(err)
   413  				changesParams = nil
   414  				w.Stop()
   415  				break
   416  			}
   417  			changesParams[j] = *change
   418  		}
   419  		results.Results[i].Changes = changesParams
   420  		results.Results[i].RelationStatusWatcherId = api.resources.Register(w)
   421  	}
   422  	return results, nil
   423  }
   424  
   425  // OfferWatcher instances track changes to a specified offer.
   426  type OfferWatcher interface {
   427  	state.NotifyWatcher
   428  	OfferUUID() string
   429  }
   430  
   431  type offerWatcher struct {
   432  	state.NotifyWatcher
   433  	offerUUID string
   434  }
   435  
   436  func (w *offerWatcher) OfferUUID() string {
   437  	return w.offerUUID
   438  }
   439  
   440  func watchOfferStatus(st CrossModelRelationsState, offerUUID string) (OfferWatcher, error) {
   441  	w, err := st.WatchOfferStatus(offerUUID)
   442  	if err != nil {
   443  		return nil, errors.Trace(err)
   444  	}
   445  	return &offerWatcher{w, offerUUID}, nil
   446  }
   447  
   448  // WatchOfferStatus starts an OfferStatusWatcher for
   449  // watching the status of an offer.
   450  func (api *CrossModelRelationsAPI) WatchOfferStatus(
   451  	offerArgs params.OfferArgs,
   452  ) (params.OfferStatusWatchResults, error) {
   453  	results := params.OfferStatusWatchResults{
   454  		Results: make([]params.OfferStatusWatchResult, len(offerArgs.Args)),
   455  	}
   456  
   457  	for i, arg := range offerArgs.Args {
   458  		// Ensure the supplied macaroon allows access.
   459  		auth := api.authCtxt.Authenticator(api.st.ModelUUID(), arg.OfferUUID)
   460  		_, err := auth.CheckOfferMacaroons(arg.OfferUUID, arg.Macaroons)
   461  		if err != nil {
   462  			results.Results[i].Error = common.ServerError(err)
   463  			continue
   464  		}
   465  
   466  		w, err := api.offerStatusWatcher(api.st, arg.OfferUUID)
   467  		if err != nil {
   468  			results.Results[i].Error = common.ServerError(err)
   469  			continue
   470  		}
   471  		_, ok := <-w.Changes()
   472  		if !ok {
   473  			results.Results[i].Error = common.ServerError(watcher.EnsureErr(w))
   474  			continue
   475  		}
   476  		change, err := commoncrossmodel.GetOfferStatusChange(api.st, arg.OfferUUID)
   477  		if err != nil {
   478  			results.Results[i].Error = common.ServerError(err)
   479  			w.Stop()
   480  			break
   481  		}
   482  		results.Results[i].Changes = []params.OfferStatusChange{*change}
   483  		results.Results[i].OfferStatusWatcherId = api.resources.Register(w)
   484  	}
   485  	return results, nil
   486  }
   487  
   488  // PublishIngressNetworkChanges publishes changes to the required
   489  // ingress addresses to the model hosting the offer in the relation.
   490  func (api *CrossModelRelationsAPI) PublishIngressNetworkChanges(
   491  	changes params.IngressNetworksChanges,
   492  ) (params.ErrorResults, error) {
   493  	results := params.ErrorResults{
   494  		Results: make([]params.ErrorResult, len(changes.Changes)),
   495  	}
   496  	for i, change := range changes.Changes {
   497  		relationTag, err := api.st.GetRemoteEntity(change.RelationToken)
   498  		if err != nil {
   499  			results.Results[i].Error = common.ServerError(err)
   500  			continue
   501  		}
   502  		logger.Debugf("relation tag for token %+v is %v", change.RelationToken, relationTag)
   503  
   504  		if err := api.checkMacaroonsForRelation(relationTag, change.Macaroons); err != nil {
   505  			results.Results[i].Error = common.ServerError(err)
   506  			continue
   507  		}
   508  		if err := commoncrossmodel.PublishIngressNetworkChange(api.st, relationTag, change); err != nil {
   509  			results.Results[i].Error = common.ServerError(err)
   510  			continue
   511  		}
   512  	}
   513  	return results, nil
   514  }
   515  
   516  // WatchEgressAddressesForRelations creates a watcher that notifies when addresses, from which
   517  // connections will originate for the relation, change.
   518  // Each event contains the entire set of addresses which are required for ingress for the relation.
   519  func (api *CrossModelRelationsAPI) WatchEgressAddressesForRelations(remoteRelationArgs params.RemoteEntityArgs) (params.StringsWatchResults, error) {
   520  	results := params.StringsWatchResults{
   521  		Results: make([]params.StringsWatchResult, len(remoteRelationArgs.Args)),
   522  	}
   523  	var relations params.Entities
   524  	for i, arg := range remoteRelationArgs.Args {
   525  		relationTag, err := api.st.GetRemoteEntity(arg.Token)
   526  		if err != nil {
   527  			results.Results[i].Error = common.ServerError(err)
   528  			continue
   529  		}
   530  		if err := api.checkMacaroonsForRelation(relationTag, arg.Macaroons); err != nil {
   531  			results.Results[i].Error = common.ServerError(err)
   532  			continue
   533  		}
   534  		relations.Entities = append(relations.Entities, params.Entity{Tag: relationTag.String()})
   535  	}
   536  	watchResults, err := api.egressAddressWatcher(api.resources, api.fw, relations)
   537  	if err != nil {
   538  		return results, err
   539  	}
   540  	index := 0
   541  	for i, r := range results.Results {
   542  		if r.Error != nil {
   543  			continue
   544  		}
   545  		results.Results[i] = watchResults.Results[index]
   546  		index++
   547  	}
   548  	return results, nil
   549  }