get.pme.sh/pnats@v0.0.0-20240304004023-26bb5a137ed0/server/accounts.go (about)

     1  // Copyright 2018-2024 The NATS Authors
     2  // Licensed under the Apache License, Version 2.0 (the "License");
     3  // you may not use this file except in compliance with the License.
     4  // You may obtain a copy of the License at
     5  //
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package server
    15  
    16  import (
    17  	"bytes"
    18  	"encoding/hex"
    19  	"errors"
    20  	"fmt"
    21  	"io"
    22  	"io/fs"
    23  	"math"
    24  	"math/rand"
    25  	"net/http"
    26  	"net/textproto"
    27  	"reflect"
    28  	"sort"
    29  	"strconv"
    30  	"strings"
    31  	"sync"
    32  	"sync/atomic"
    33  	"time"
    34  
    35  	"get.pme.sh/pnats/internal/fastrand"
    36  	"github.com/nats-io/jwt/v2"
    37  	"github.com/nats-io/nkeys"
    38  	"github.com/nats-io/nuid"
    39  )
    40  
    41  // For backwards compatibility with NATS < 2.0, users who are not explicitly defined into an
    42  // account will be grouped in the default global account.
    43  const globalAccountName = DEFAULT_GLOBAL_ACCOUNT
    44  
    45  const defaultMaxSubLimitReportThreshold = int64(2 * time.Second)
    46  
    47  var maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold
    48  
    49  // Account are subject namespace definitions. By default no messages are shared between accounts.
    50  // You can share via Exports and Imports of Streams and Services.
    51  type Account struct {
    52  	stats
    53  	gwReplyMapping
    54  	Name         string
    55  	Nkey         string
    56  	Issuer       string
    57  	claimJWT     string
    58  	updated      time.Time
    59  	mu           sync.RWMutex
    60  	sqmu         sync.Mutex
    61  	sl           *Sublist
    62  	ic           *client
    63  	isid         uint64
    64  	etmr         *time.Timer
    65  	ctmr         *time.Timer
    66  	strack       map[string]sconns
    67  	nrclients    int32
    68  	sysclients   int32
    69  	nleafs       int32
    70  	nrleafs      int32
    71  	clients      map[*client]struct{}
    72  	rm           map[string]int32
    73  	lqws         map[string]int32
    74  	usersRevoked map[string]int64
    75  	mappings     []*mapping
    76  	hasMapped    atomic.Bool
    77  	lmu          sync.RWMutex
    78  	lleafs       []*client
    79  	leafClusters map[string]uint64
    80  	imports      importMap
    81  	exports      exportMap
    82  	js           *jsAccount
    83  	jsLimits     map[string]JetStreamAccountLimits
    84  	limits
    85  	expired      bool
    86  	incomplete   bool
    87  	signingKeys  map[string]jwt.Scope
    88  	extAuth      *jwt.ExternalAuthorization
    89  	srv          *Server // server this account is registered with (possibly nil)
    90  	lds          string  // loop detection subject for leaf nodes
    91  	siReply      []byte  // service reply prefix, will form wildcard subscription.
    92  	eventIds     *nuid.NUID
    93  	eventIdsMu   sync.Mutex
    94  	defaultPerms *Permissions
    95  	tags         jwt.TagList
    96  	nameTag      string
    97  	lastLimErr   int64
    98  	routePoolIdx int
    99  	// If the trace destination is specified and a message with a traceParentHdr
   100  	// is received, and has the least significant bit of the last token set to 1,
   101  	// then if traceDestSampling is > 0 and < 100, a random value will be selected
   102  	// and if it falls between 0 and that value, message tracing will be triggered.
   103  	traceDest         string
   104  	traceDestSampling int
   105  }
   106  
   107  const (
   108  	accDedicatedRoute                = -1
   109  	accTransitioningToDedicatedRoute = -2
   110  )
   111  
   112  // Account based limits.
   113  type limits struct {
   114  	mpay           int32
   115  	msubs          int32
   116  	mconns         int32
   117  	mleafs         int32
   118  	disallowBearer bool
   119  }
   120  
   121  // Used to track remote clients and leafnodes per remote server.
   122  type sconns struct {
   123  	conns int32
   124  	leafs int32
   125  }
   126  
   127  // Import stream mapping struct
   128  type streamImport struct {
   129  	acc     *Account
   130  	from    string
   131  	to      string
   132  	tr      *subjectTransform
   133  	rtr     *subjectTransform
   134  	claim   *jwt.Import
   135  	usePub  bool
   136  	invalid bool
   137  	// This is `allow_trace` and when true and message tracing is happening,
   138  	// we will trace egresses past the account boundary, if `false`, we stop
   139  	// at the account boundary.
   140  	atrc bool
   141  }
   142  
   143  const ClientInfoHdr = "Nats-Request-Info"
   144  
   145  // Import service mapping struct
   146  type serviceImport struct {
   147  	acc         *Account
   148  	claim       *jwt.Import
   149  	se          *serviceExport
   150  	sid         []byte
   151  	from        string
   152  	to          string
   153  	tr          *subjectTransform
   154  	ts          int64
   155  	rt          ServiceRespType
   156  	latency     *serviceLatency
   157  	m1          *ServiceLatency
   158  	rc          *client
   159  	usePub      bool
   160  	response    bool
   161  	invalid     bool
   162  	share       bool
   163  	tracking    bool
   164  	didDeliver  bool
   165  	atrc        bool        // allow trace (got from service export)
   166  	trackingHdr http.Header // header from request
   167  }
   168  
   169  // This is used to record when we create a mapping for implicit service
   170  // imports. We use this to clean up entries that are not singletons when
   171  // we detect that interest is no longer present. The key to the map will
   172  // be the actual interest. We record the mapped subject and the account.
   173  type serviceRespEntry struct {
   174  	acc  *Account
   175  	msub string
   176  }
   177  
   178  // ServiceRespType represents the types of service request response types.
   179  type ServiceRespType uint8
   180  
   181  // Service response types. Defaults to a singleton.
   182  const (
   183  	Singleton ServiceRespType = iota
   184  	Streamed
   185  	Chunked
   186  )
   187  
   188  // String helper.
   189  func (rt ServiceRespType) String() string {
   190  	switch rt {
   191  	case Singleton:
   192  		return "Singleton"
   193  	case Streamed:
   194  		return "Streamed"
   195  	case Chunked:
   196  		return "Chunked"
   197  	}
   198  	return "Unknown ServiceResType"
   199  }
   200  
   201  // exportAuth holds configured approvals or boolean indicating an
   202  // auth token is required for import.
   203  type exportAuth struct {
   204  	tokenReq    bool
   205  	accountPos  uint
   206  	approved    map[string]*Account
   207  	actsRevoked map[string]int64
   208  }
   209  
   210  // streamExport
   211  type streamExport struct {
   212  	exportAuth
   213  }
   214  
   215  // serviceExport holds additional information for exported services.
   216  type serviceExport struct {
   217  	exportAuth
   218  	acc        *Account
   219  	respType   ServiceRespType
   220  	latency    *serviceLatency
   221  	rtmr       *time.Timer
   222  	respThresh time.Duration
   223  	// This is `allow_trace` and when true and message tracing is happening,
   224  	// when processing a service import we will go through account boundary
   225  	// and trace egresses on that other account. If `false`, we stop at the
   226  	// account boundary.
   227  	atrc bool
   228  }
   229  
   230  // Used to track service latency.
   231  type serviceLatency struct {
   232  	sampling int8 // percentage from 1-100 or 0 to indicate triggered by header
   233  	subject  string
   234  }
   235  
   236  // exportMap tracks the exported streams and services.
   237  type exportMap struct {
   238  	streams   map[string]*streamExport
   239  	services  map[string]*serviceExport
   240  	responses map[string]*serviceImport
   241  }
   242  
   243  // importMap tracks the imported streams and services.
   244  // For services we will also track the response mappings as well.
   245  type importMap struct {
   246  	streams  []*streamImport
   247  	services map[string]*serviceImport
   248  	rrMap    map[string][]*serviceRespEntry
   249  }
   250  
   251  // NewAccount creates a new unlimited account with the given name.
   252  func NewAccount(name string) *Account {
   253  	a := &Account{
   254  		Name:     name,
   255  		limits:   limits{-1, -1, -1, -1, false},
   256  		eventIds: nuid.New(),
   257  	}
   258  	return a
   259  }
   260  
   261  func (a *Account) String() string {
   262  	return a.Name
   263  }
   264  
   265  func (a *Account) setTraceDest(dest string) {
   266  	a.mu.Lock()
   267  	a.traceDest = dest
   268  	a.mu.Unlock()
   269  }
   270  
   271  func (a *Account) getTraceDestAndSampling() (string, int) {
   272  	a.mu.RLock()
   273  	dest := a.traceDest
   274  	sampling := a.traceDestSampling
   275  	a.mu.RUnlock()
   276  	return dest, sampling
   277  }
   278  
   279  // Used to create shallow copies of accounts for transfer
   280  // from opts to real accounts in server struct.
   281  // Account `na` write lock is expected to be held on entry
   282  // while account `a` is the one from the Options struct
   283  // being loaded/reloaded and do not need locking.
   284  func (a *Account) shallowCopy(na *Account) {
   285  	na.Nkey = a.Nkey
   286  	na.Issuer = a.Issuer
   287  	na.traceDest, na.traceDestSampling = a.traceDest, a.traceDestSampling
   288  
   289  	if a.imports.streams != nil {
   290  		na.imports.streams = make([]*streamImport, 0, len(a.imports.streams))
   291  		for _, v := range a.imports.streams {
   292  			si := *v
   293  			na.imports.streams = append(na.imports.streams, &si)
   294  		}
   295  	}
   296  	if a.imports.services != nil {
   297  		na.imports.services = make(map[string]*serviceImport)
   298  		for k, v := range a.imports.services {
   299  			si := *v
   300  			na.imports.services[k] = &si
   301  		}
   302  	}
   303  	if a.exports.streams != nil {
   304  		na.exports.streams = make(map[string]*streamExport)
   305  		for k, v := range a.exports.streams {
   306  			if v != nil {
   307  				se := *v
   308  				na.exports.streams[k] = &se
   309  			} else {
   310  				na.exports.streams[k] = nil
   311  			}
   312  		}
   313  	}
   314  	if a.exports.services != nil {
   315  		na.exports.services = make(map[string]*serviceExport)
   316  		for k, v := range a.exports.services {
   317  			if v != nil {
   318  				se := *v
   319  				na.exports.services[k] = &se
   320  			} else {
   321  				na.exports.services[k] = nil
   322  			}
   323  		}
   324  	}
   325  	na.mappings = a.mappings
   326  	na.hasMapped.Store(len(na.mappings) > 0)
   327  
   328  	// JetStream
   329  	na.jsLimits = a.jsLimits
   330  	// Server config account limits.
   331  	na.limits = a.limits
   332  }
   333  
   334  // nextEventID uses its own lock for better concurrency.
   335  func (a *Account) nextEventID() string {
   336  	a.eventIdsMu.Lock()
   337  	id := a.eventIds.Next()
   338  	a.eventIdsMu.Unlock()
   339  	return id
   340  }
   341  
   342  // Returns a slice of clients stored in the account, or nil if none is present.
   343  // Lock is held on entry.
   344  func (a *Account) getClientsLocked() []*client {
   345  	if len(a.clients) == 0 {
   346  		return nil
   347  	}
   348  	clients := make([]*client, 0, len(a.clients))
   349  	for c := range a.clients {
   350  		clients = append(clients, c)
   351  	}
   352  	return clients
   353  }
   354  
   355  // Returns a slice of clients stored in the account, or nil if none is present.
   356  func (a *Account) getClients() []*client {
   357  	a.mu.RLock()
   358  	clients := a.getClientsLocked()
   359  	a.mu.RUnlock()
   360  	return clients
   361  }
   362  
   363  // Called to track a remote server and connections and leafnodes it
   364  // has for this account.
   365  func (a *Account) updateRemoteServer(m *AccountNumConns) []*client {
   366  	a.mu.Lock()
   367  	if a.strack == nil {
   368  		a.strack = make(map[string]sconns)
   369  	}
   370  	// This does not depend on receiving all updates since each one is idempotent.
   371  	// FIXME(dlc) - We should cleanup when these both go to zero.
   372  	prev := a.strack[m.Server.ID]
   373  	a.strack[m.Server.ID] = sconns{conns: int32(m.Conns), leafs: int32(m.LeafNodes)}
   374  	a.nrclients += int32(m.Conns) - prev.conns
   375  	a.nrleafs += int32(m.LeafNodes) - prev.leafs
   376  
   377  	mtce := a.mconns != jwt.NoLimit && (len(a.clients)-int(a.sysclients)+int(a.nrclients) > int(a.mconns))
   378  	// If we are over here some have snuck in and we need to rebalance.
   379  	// All others will probably be doing the same thing but better to be
   380  	// conservative and bit harsh here. Clients will reconnect if we over compensate.
   381  	var clients []*client
   382  	if mtce {
   383  		clients := a.getClientsLocked()
   384  		sort.Slice(clients, func(i, j int) bool {
   385  			return clients[i].start.After(clients[j].start)
   386  		})
   387  		over := (len(a.clients) - int(a.sysclients) + int(a.nrclients)) - int(a.mconns)
   388  		if over < len(clients) {
   389  			clients = clients[:over]
   390  		}
   391  	}
   392  	// Now check leafnodes.
   393  	mtlce := a.mleafs != jwt.NoLimit && (a.nleafs+a.nrleafs > a.mleafs)
   394  	if mtlce {
   395  		// Take ones from the end.
   396  		a.lmu.RLock()
   397  		leafs := a.lleafs
   398  		over := int(a.nleafs + a.nrleafs - a.mleafs)
   399  		if over < len(leafs) {
   400  			leafs = leafs[len(leafs)-over:]
   401  		}
   402  		clients = append(clients, leafs...)
   403  		a.lmu.RUnlock()
   404  	}
   405  	a.mu.Unlock()
   406  
   407  	// If we have exceeded our max clients this will be populated.
   408  	return clients
   409  }
   410  
   411  // Removes tracking for a remote server that has shutdown.
   412  func (a *Account) removeRemoteServer(sid string) {
   413  	a.mu.Lock()
   414  	if a.strack != nil {
   415  		prev := a.strack[sid]
   416  		delete(a.strack, sid)
   417  		a.nrclients -= prev.conns
   418  		a.nrleafs -= prev.leafs
   419  	}
   420  	a.mu.Unlock()
   421  }
   422  
   423  // When querying for subject interest this is the number of
   424  // expected responses. We need to actually check that the entry
   425  // has active connections.
   426  func (a *Account) expectedRemoteResponses() (expected int32) {
   427  	a.mu.RLock()
   428  	for _, sc := range a.strack {
   429  		if sc.conns > 0 || sc.leafs > 0 {
   430  			expected++
   431  		}
   432  	}
   433  	a.mu.RUnlock()
   434  	return
   435  }
   436  
   437  // Clears eventing and tracking for this account.
   438  func (a *Account) clearEventing() {
   439  	a.mu.Lock()
   440  	a.nrclients = 0
   441  	// Now clear state
   442  	clearTimer(&a.etmr)
   443  	clearTimer(&a.ctmr)
   444  	a.clients = nil
   445  	a.strack = nil
   446  	a.mu.Unlock()
   447  }
   448  
   449  // GetName will return the accounts name.
   450  func (a *Account) GetName() string {
   451  	if a == nil {
   452  		return "n/a"
   453  	}
   454  	a.mu.RLock()
   455  	name := a.Name
   456  	a.mu.RUnlock()
   457  	return name
   458  }
   459  
   460  // NumConnections returns active number of clients for this account for
   461  // all known servers.
   462  func (a *Account) NumConnections() int {
   463  	a.mu.RLock()
   464  	nc := len(a.clients) - int(a.sysclients) + int(a.nrclients)
   465  	a.mu.RUnlock()
   466  	return nc
   467  }
   468  
   469  // NumRemoteConnections returns the number of client or leaf connections that
   470  // are not on this server.
   471  func (a *Account) NumRemoteConnections() int {
   472  	a.mu.RLock()
   473  	nc := int(a.nrclients + a.nrleafs)
   474  	a.mu.RUnlock()
   475  	return nc
   476  }
   477  
   478  // NumLocalConnections returns active number of clients for this account
   479  // on this server.
   480  func (a *Account) NumLocalConnections() int {
   481  	a.mu.RLock()
   482  	nlc := a.numLocalConnections()
   483  	a.mu.RUnlock()
   484  	return nlc
   485  }
   486  
   487  // Do not account for the system accounts.
   488  func (a *Account) numLocalConnections() int {
   489  	return len(a.clients) - int(a.sysclients) - int(a.nleafs)
   490  }
   491  
   492  // This is for extended local interest.
   493  // Lock should not be held.
   494  func (a *Account) numLocalAndLeafConnections() int {
   495  	a.mu.RLock()
   496  	nlc := len(a.clients) - int(a.sysclients)
   497  	a.mu.RUnlock()
   498  	return nlc
   499  }
   500  
   501  func (a *Account) numLocalLeafNodes() int {
   502  	return int(a.nleafs)
   503  }
   504  
   505  // MaxTotalConnectionsReached returns if we have reached our limit for number of connections.
   506  func (a *Account) MaxTotalConnectionsReached() bool {
   507  	var mtce bool
   508  	a.mu.RLock()
   509  	if a.mconns != jwt.NoLimit {
   510  		mtce = len(a.clients)-int(a.sysclients)+int(a.nrclients) >= int(a.mconns)
   511  	}
   512  	a.mu.RUnlock()
   513  	return mtce
   514  }
   515  
   516  // MaxActiveConnections return the set limit for the account system
   517  // wide for total number of active connections.
   518  func (a *Account) MaxActiveConnections() int {
   519  	a.mu.RLock()
   520  	mconns := int(a.mconns)
   521  	a.mu.RUnlock()
   522  	return mconns
   523  }
   524  
   525  // MaxTotalLeafNodesReached returns if we have reached our limit for number of leafnodes.
   526  func (a *Account) MaxTotalLeafNodesReached() bool {
   527  	a.mu.RLock()
   528  	mtc := a.maxTotalLeafNodesReached()
   529  	a.mu.RUnlock()
   530  	return mtc
   531  }
   532  
   533  func (a *Account) maxTotalLeafNodesReached() bool {
   534  	if a.mleafs != jwt.NoLimit {
   535  		return a.nleafs+a.nrleafs >= a.mleafs
   536  	}
   537  	return false
   538  }
   539  
   540  // NumLeafNodes returns the active number of local and remote
   541  // leaf node connections.
   542  func (a *Account) NumLeafNodes() int {
   543  	a.mu.RLock()
   544  	nln := int(a.nleafs + a.nrleafs)
   545  	a.mu.RUnlock()
   546  	return nln
   547  }
   548  
   549  // NumRemoteLeafNodes returns the active number of remote
   550  // leaf node connections.
   551  func (a *Account) NumRemoteLeafNodes() int {
   552  	a.mu.RLock()
   553  	nrn := int(a.nrleafs)
   554  	a.mu.RUnlock()
   555  	return nrn
   556  }
   557  
   558  // MaxActiveLeafNodes return the set limit for the account system
   559  // wide for total number of leavenode connections.
   560  // NOTE: these are tracked separately.
   561  func (a *Account) MaxActiveLeafNodes() int {
   562  	a.mu.RLock()
   563  	mleafs := int(a.mleafs)
   564  	a.mu.RUnlock()
   565  	return mleafs
   566  }
   567  
   568  // RoutedSubs returns how many subjects we would send across a route when first
   569  // connected or expressing interest. Local client subs.
   570  func (a *Account) RoutedSubs() int {
   571  	a.mu.RLock()
   572  	defer a.mu.RUnlock()
   573  	return len(a.rm)
   574  }
   575  
   576  // TotalSubs returns total number of Subscriptions for this account.
   577  func (a *Account) TotalSubs() int {
   578  	a.mu.RLock()
   579  	defer a.mu.RUnlock()
   580  	if a.sl == nil {
   581  		return 0
   582  	}
   583  	return int(a.sl.Count())
   584  }
   585  
   586  func (a *Account) shouldLogMaxSubErr() bool {
   587  	if a == nil {
   588  		return true
   589  	}
   590  	a.mu.RLock()
   591  	last := a.lastLimErr
   592  	a.mu.RUnlock()
   593  	if now := time.Now().UnixNano(); now-last >= maxSubLimitReportThreshold {
   594  		a.mu.Lock()
   595  		a.lastLimErr = now
   596  		a.mu.Unlock()
   597  		return true
   598  	}
   599  	return false
   600  }
   601  
   602  // MapDest is for mapping published subjects for clients.
   603  type MapDest struct {
   604  	Subject string `json:"subject"`
   605  	Weight  uint8  `json:"weight"`
   606  	Cluster string `json:"cluster,omitempty"`
   607  }
   608  
   609  func NewMapDest(subject string, weight uint8) *MapDest {
   610  	return &MapDest{subject, weight, _EMPTY_}
   611  }
   612  
   613  // destination is for internal representation for a weighted mapped destination.
   614  type destination struct {
   615  	tr     *subjectTransform
   616  	weight uint8
   617  }
   618  
   619  // mapping is an internal entry for mapping subjects.
   620  type mapping struct {
   621  	src    string
   622  	wc     bool
   623  	dests  []*destination
   624  	cdests map[string][]*destination
   625  }
   626  
   627  // AddMapping adds in a simple route mapping from src subject to dest subject
   628  // for inbound client messages.
   629  func (a *Account) AddMapping(src, dest string) error {
   630  	return a.AddWeightedMappings(src, NewMapDest(dest, 100))
   631  }
   632  
   633  // AddWeightedMappings will add in a weighted mappings for the destinations.
   634  func (a *Account) AddWeightedMappings(src string, dests ...*MapDest) error {
   635  	a.mu.Lock()
   636  	defer a.mu.Unlock()
   637  
   638  	if !IsValidSubject(src) {
   639  		return ErrBadSubject
   640  	}
   641  
   642  	m := &mapping{src: src, wc: subjectHasWildcard(src), dests: make([]*destination, 0, len(dests)+1)}
   643  	seen := make(map[string]struct{})
   644  
   645  	var tw = make(map[string]uint8)
   646  	for _, d := range dests {
   647  		if _, ok := seen[d.Subject]; ok {
   648  			return fmt.Errorf("duplicate entry for %q", d.Subject)
   649  		}
   650  		seen[d.Subject] = struct{}{}
   651  		if d.Weight > 100 {
   652  			return fmt.Errorf("individual weights need to be <= 100")
   653  		}
   654  		tw[d.Cluster] += d.Weight
   655  		if tw[d.Cluster] > 100 {
   656  			return fmt.Errorf("total weight needs to be <= 100")
   657  		}
   658  		err := ValidateMappingDestination(d.Subject)
   659  		if err != nil {
   660  			return err
   661  		}
   662  		tr, err := NewSubjectTransform(src, d.Subject)
   663  		if err != nil {
   664  			return err
   665  		}
   666  		if d.Cluster == _EMPTY_ {
   667  			m.dests = append(m.dests, &destination{tr, d.Weight})
   668  		} else {
   669  			// We have a cluster scoped filter.
   670  			if m.cdests == nil {
   671  				m.cdests = make(map[string][]*destination)
   672  			}
   673  			ad := m.cdests[d.Cluster]
   674  			ad = append(ad, &destination{tr, d.Weight})
   675  			m.cdests[d.Cluster] = ad
   676  		}
   677  	}
   678  
   679  	processDestinations := func(dests []*destination) ([]*destination, error) {
   680  		var ltw uint8
   681  		for _, d := range dests {
   682  			ltw += d.weight
   683  		}
   684  		// Auto add in original at weight difference if all entries weight does not total to 100.
   685  		// Iff the src was not already added in explicitly, meaning they want loss.
   686  		_, haveSrc := seen[src]
   687  		if ltw != 100 && !haveSrc {
   688  			dest := src
   689  			if m.wc {
   690  				// We need to make the appropriate markers for the wildcards etc.
   691  				dest = transformTokenize(dest)
   692  			}
   693  			tr, err := NewSubjectTransform(src, dest)
   694  			if err != nil {
   695  				return nil, err
   696  			}
   697  			aw := 100 - ltw
   698  			if len(dests) == 0 {
   699  				aw = 100
   700  			}
   701  			dests = append(dests, &destination{tr, aw})
   702  		}
   703  		sort.Slice(dests, func(i, j int) bool { return dests[i].weight < dests[j].weight })
   704  
   705  		var lw uint8
   706  		for _, d := range dests {
   707  			d.weight += lw
   708  			lw = d.weight
   709  		}
   710  		return dests, nil
   711  	}
   712  
   713  	var err error
   714  	if m.dests, err = processDestinations(m.dests); err != nil {
   715  		return err
   716  	}
   717  
   718  	// Option cluster scoped destinations
   719  	for cluster, dests := range m.cdests {
   720  		if dests, err = processDestinations(dests); err != nil {
   721  			return err
   722  		}
   723  		m.cdests[cluster] = dests
   724  	}
   725  
   726  	// Replace an old one if it exists.
   727  	for i, em := range a.mappings {
   728  		if em.src == src {
   729  			a.mappings[i] = m
   730  			return nil
   731  		}
   732  	}
   733  	// If we did not replace add to the end.
   734  	a.mappings = append(a.mappings, m)
   735  	a.hasMapped.Store(len(a.mappings) > 0)
   736  
   737  	// If we have connected leafnodes make sure to update.
   738  	if a.nleafs > 0 {
   739  		// Need to release because lock ordering is client -> account
   740  		a.mu.Unlock()
   741  		// Now grab the leaf list lock. We can hold client lock under this one.
   742  		a.lmu.RLock()
   743  		for _, lc := range a.lleafs {
   744  			lc.forceAddToSmap(src)
   745  		}
   746  		a.lmu.RUnlock()
   747  		a.mu.Lock()
   748  	}
   749  	return nil
   750  }
   751  
   752  // RemoveMapping will remove an existing mapping.
   753  func (a *Account) RemoveMapping(src string) bool {
   754  	a.mu.Lock()
   755  	defer a.mu.Unlock()
   756  	for i, m := range a.mappings {
   757  		if m.src == src {
   758  			// Swap last one into this spot. Its ok to change order.
   759  			a.mappings[i] = a.mappings[len(a.mappings)-1]
   760  			a.mappings[len(a.mappings)-1] = nil // gc
   761  			a.mappings = a.mappings[:len(a.mappings)-1]
   762  			a.hasMapped.Store(len(a.mappings) > 0)
   763  			// If we have connected leafnodes make sure to update.
   764  			if a.nleafs > 0 {
   765  				// Need to release because lock ordering is client -> account
   766  				a.mu.Unlock()
   767  				// Now grab the leaf list lock. We can hold client lock under this one.
   768  				a.lmu.RLock()
   769  				for _, lc := range a.lleafs {
   770  					lc.forceRemoveFromSmap(src)
   771  				}
   772  				a.lmu.RUnlock()
   773  				a.mu.Lock()
   774  			}
   775  			return true
   776  		}
   777  	}
   778  	return false
   779  }
   780  
   781  // Indicates we have mapping entries.
   782  func (a *Account) hasMappings() bool {
   783  	if a == nil {
   784  		return false
   785  	}
   786  	return a.hasMapped.Load()
   787  }
   788  
   789  // This performs the logic to map to a new dest subject based on mappings.
   790  // Should only be called from processInboundClientMsg or service import processing.
   791  func (a *Account) selectMappedSubject(dest string) (string, bool) {
   792  	if !a.hasMappings() {
   793  		return dest, false
   794  	}
   795  
   796  	a.mu.Lock()
   797  	// In case we have to tokenize for subset matching.
   798  	tsa := [32]string{}
   799  	tts := tsa[:0]
   800  
   801  	var m *mapping
   802  	for _, rm := range a.mappings {
   803  		if !rm.wc && rm.src == dest {
   804  			m = rm
   805  			break
   806  		} else {
   807  			// tokenize and reuse for subset matching.
   808  			if len(tts) == 0 {
   809  				start := 0
   810  				subject := dest
   811  				for i := 0; i < len(subject); i++ {
   812  					if subject[i] == btsep {
   813  						tts = append(tts, subject[start:i])
   814  						start = i + 1
   815  					}
   816  				}
   817  				tts = append(tts, subject[start:])
   818  			}
   819  			if isSubsetMatch(tts, rm.src) {
   820  				m = rm
   821  				break
   822  			}
   823  		}
   824  	}
   825  
   826  	if m == nil {
   827  		a.mu.Unlock()
   828  		return dest, false
   829  	}
   830  
   831  	// The selected destination for the mapping.
   832  	var d *destination
   833  	var ndest string
   834  
   835  	dests := m.dests
   836  	if len(m.cdests) > 0 {
   837  		cn := a.srv.cachedClusterName()
   838  		dests = m.cdests[cn]
   839  		if dests == nil {
   840  			// Fallback to main if we do not match the cluster.
   841  			dests = m.dests
   842  		}
   843  	}
   844  
   845  	// Optimize for single entry case.
   846  	if len(dests) == 1 && dests[0].weight == 100 {
   847  		d = dests[0]
   848  	} else {
   849  		w := uint8(fastrand.Uint32n(100))
   850  		for _, rm := range dests {
   851  			if w < rm.weight {
   852  				d = rm
   853  				break
   854  			}
   855  		}
   856  	}
   857  
   858  	if d != nil {
   859  		if len(d.tr.dtokmftokindexesargs) == 0 {
   860  			ndest = d.tr.dest
   861  		} else {
   862  			ndest = d.tr.TransformTokenizedSubject(tts)
   863  		}
   864  	}
   865  
   866  	a.mu.Unlock()
   867  	return ndest, true
   868  }
   869  
   870  // SubscriptionInterest returns true if this account has a matching subscription
   871  // for the given `subject`.
   872  func (a *Account) SubscriptionInterest(subject string) bool {
   873  	return a.Interest(subject) > 0
   874  }
   875  
   876  // Interest returns the number of subscriptions for a given subject that match.
   877  func (a *Account) Interest(subject string) int {
   878  	var nms int
   879  	a.mu.RLock()
   880  	if a.sl != nil {
   881  		res := a.sl.Match(subject)
   882  		nms = len(res.psubs) + len(res.qsubs)
   883  	}
   884  	a.mu.RUnlock()
   885  	return nms
   886  }
   887  
   888  // addClient keeps our accounting of local active clients or leafnodes updated.
   889  // Returns previous total.
   890  func (a *Account) addClient(c *client) int {
   891  	a.mu.Lock()
   892  	n := len(a.clients)
   893  	if a.clients != nil {
   894  		a.clients[c] = struct{}{}
   895  	}
   896  	added := n != len(a.clients)
   897  	if added {
   898  		if c.kind != CLIENT && c.kind != LEAF {
   899  			a.sysclients++
   900  		} else if c.kind == LEAF {
   901  			a.nleafs++
   902  		}
   903  	}
   904  	a.mu.Unlock()
   905  
   906  	// If we added a new leaf use the list lock and add it to the list.
   907  	if added && c.kind == LEAF {
   908  		a.lmu.Lock()
   909  		a.lleafs = append(a.lleafs, c)
   910  		a.lmu.Unlock()
   911  	}
   912  
   913  	if c != nil && c.srv != nil && added {
   914  		c.srv.accConnsUpdate(a)
   915  	}
   916  
   917  	return n
   918  }
   919  
   920  // For registering clusters for remote leafnodes.
   921  // We only register as the hub.
   922  func (a *Account) registerLeafNodeCluster(cluster string) {
   923  	a.mu.Lock()
   924  	defer a.mu.Unlock()
   925  	if a.leafClusters == nil {
   926  		a.leafClusters = make(map[string]uint64)
   927  	}
   928  	a.leafClusters[cluster]++
   929  }
   930  
   931  // Check to see if we already have this cluster registered.
   932  func (a *Account) hasLeafNodeCluster(cluster string) bool {
   933  	a.mu.RLock()
   934  	defer a.mu.RUnlock()
   935  	return a.leafClusters[cluster] > 0
   936  }
   937  
   938  // Check to see if this cluster is isolated, meaning the only one.
   939  // Read Lock should be held.
   940  func (a *Account) isLeafNodeClusterIsolated(cluster string) bool {
   941  	if cluster == _EMPTY_ {
   942  		return false
   943  	}
   944  	if len(a.leafClusters) > 1 {
   945  		return false
   946  	}
   947  	return a.leafClusters[cluster] == uint64(a.nleafs)
   948  }
   949  
   950  // Helper function to remove leaf nodes. If number of leafnodes gets large
   951  // this may need to be optimized out of linear search but believe number
   952  // of active leafnodes per account scope to be small and therefore cache friendly.
   953  // Lock should not be held on general account lock.
   954  func (a *Account) removeLeafNode(c *client) {
   955  	// Make sure we hold the list lock as well.
   956  	a.lmu.Lock()
   957  	defer a.lmu.Unlock()
   958  
   959  	ll := len(a.lleafs)
   960  	for i, l := range a.lleafs {
   961  		if l == c {
   962  			a.lleafs[i] = a.lleafs[ll-1]
   963  			if ll == 1 {
   964  				a.lleafs = nil
   965  			} else {
   966  				a.lleafs = a.lleafs[:ll-1]
   967  			}
   968  			return
   969  		}
   970  	}
   971  }
   972  
   973  // removeClient keeps our accounting of local active clients updated.
   974  func (a *Account) removeClient(c *client) int {
   975  	a.mu.Lock()
   976  	n := len(a.clients)
   977  	delete(a.clients, c)
   978  	removed := n != len(a.clients)
   979  	if removed {
   980  		if c.kind != CLIENT && c.kind != LEAF {
   981  			a.sysclients--
   982  		} else if c.kind == LEAF {
   983  			a.nleafs--
   984  			// Need to do cluster accounting here.
   985  			// Do cluster accounting if we are a hub.
   986  			if c.isHubLeafNode() {
   987  				cluster := c.remoteCluster()
   988  				if count := a.leafClusters[cluster]; count > 1 {
   989  					a.leafClusters[cluster]--
   990  				} else if count == 1 {
   991  					delete(a.leafClusters, cluster)
   992  				}
   993  			}
   994  		}
   995  	}
   996  	a.mu.Unlock()
   997  
   998  	if removed && c.kind == LEAF {
   999  		a.removeLeafNode(c)
  1000  	}
  1001  
  1002  	if c != nil && c.srv != nil && removed {
  1003  		c.srv.accConnsUpdate(a)
  1004  	}
  1005  
  1006  	return n
  1007  }
  1008  
  1009  func setExportAuth(ea *exportAuth, subject string, accounts []*Account, accountPos uint) error {
  1010  	if accountPos > 0 {
  1011  		token := strings.Split(subject, tsep)
  1012  		if len(token) < int(accountPos) || token[accountPos-1] != "*" {
  1013  			return ErrInvalidSubject
  1014  		}
  1015  	}
  1016  	ea.accountPos = accountPos
  1017  	// empty means auth required but will be import token.
  1018  	if accounts == nil {
  1019  		return nil
  1020  	}
  1021  	if len(accounts) == 0 {
  1022  		ea.tokenReq = true
  1023  		return nil
  1024  	}
  1025  	if ea.approved == nil {
  1026  		ea.approved = make(map[string]*Account, len(accounts))
  1027  	}
  1028  	for _, acc := range accounts {
  1029  		ea.approved[acc.Name] = acc
  1030  	}
  1031  	return nil
  1032  }
  1033  
  1034  // AddServiceExport will configure the account with the defined export.
  1035  func (a *Account) AddServiceExport(subject string, accounts []*Account) error {
  1036  	return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, 0)
  1037  }
  1038  
  1039  // AddServiceExport will configure the account with the defined export.
  1040  func (a *Account) addServiceExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error {
  1041  	return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, accountPos)
  1042  }
  1043  
  1044  // AddServiceExportWithResponse will configure the account with the defined export and response type.
  1045  func (a *Account) AddServiceExportWithResponse(subject string, respType ServiceRespType, accounts []*Account) error {
  1046  	return a.addServiceExportWithResponseAndAccountPos(subject, respType, accounts, 0)
  1047  }
  1048  
  1049  // AddServiceExportWithresponse will configure the account with the defined export and response type.
  1050  func (a *Account) addServiceExportWithResponseAndAccountPos(
  1051  	subject string, respType ServiceRespType, accounts []*Account, accountPos uint) error {
  1052  	if a == nil {
  1053  		return ErrMissingAccount
  1054  	}
  1055  
  1056  	a.mu.Lock()
  1057  	if a.exports.services == nil {
  1058  		a.exports.services = make(map[string]*serviceExport)
  1059  	}
  1060  
  1061  	se := a.exports.services[subject]
  1062  	// Always  create a service export
  1063  	if se == nil {
  1064  		se = &serviceExport{}
  1065  	}
  1066  
  1067  	if respType != Singleton {
  1068  		se.respType = respType
  1069  	}
  1070  
  1071  	if accounts != nil || accountPos > 0 {
  1072  		if err := setExportAuth(&se.exportAuth, subject, accounts, accountPos); err != nil {
  1073  			a.mu.Unlock()
  1074  			return err
  1075  		}
  1076  	}
  1077  	lrt := a.lowestServiceExportResponseTime()
  1078  	se.acc = a
  1079  	se.respThresh = DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD
  1080  	a.exports.services[subject] = se
  1081  
  1082  	var clients []*client
  1083  	nlrt := a.lowestServiceExportResponseTime()
  1084  	if nlrt != lrt && len(a.clients) > 0 {
  1085  		clients = a.getClientsLocked()
  1086  	}
  1087  	// Need to release because lock ordering is client -> Account
  1088  	a.mu.Unlock()
  1089  	if len(clients) > 0 {
  1090  		updateAllClientsServiceExportResponseTime(clients, nlrt)
  1091  	}
  1092  	return nil
  1093  }
  1094  
  1095  // TrackServiceExport will enable latency tracking of the named service.
  1096  // Results will be published in this account to the given results subject.
  1097  func (a *Account) TrackServiceExport(service, results string) error {
  1098  	return a.TrackServiceExportWithSampling(service, results, DEFAULT_SERVICE_LATENCY_SAMPLING)
  1099  }
  1100  
  1101  // TrackServiceExportWithSampling will enable latency tracking of the named service for the given
  1102  // sampling rate (1-100). Results will be published in this account to the given results subject.
  1103  func (a *Account) TrackServiceExportWithSampling(service, results string, sampling int) error {
  1104  	if a == nil {
  1105  		return ErrMissingAccount
  1106  	}
  1107  
  1108  	if sampling != 0 { // 0 means triggered by header
  1109  		if sampling < 1 || sampling > 100 {
  1110  			return ErrBadSampling
  1111  		}
  1112  	}
  1113  	if !IsValidPublishSubject(results) {
  1114  		return ErrBadPublishSubject
  1115  	}
  1116  	// Don't loop back on outselves.
  1117  	if a.IsExportService(results) {
  1118  		return ErrBadPublishSubject
  1119  	}
  1120  
  1121  	if a.srv != nil && !a.srv.EventsEnabled() {
  1122  		return ErrNoSysAccount
  1123  	}
  1124  
  1125  	a.mu.Lock()
  1126  	if a.exports.services == nil {
  1127  		a.mu.Unlock()
  1128  		return ErrMissingService
  1129  	}
  1130  	ea, ok := a.exports.services[service]
  1131  	if !ok {
  1132  		a.mu.Unlock()
  1133  		return ErrMissingService
  1134  	}
  1135  	if ea == nil {
  1136  		ea = &serviceExport{}
  1137  		a.exports.services[service] = ea
  1138  	} else if ea.respType != Singleton {
  1139  		a.mu.Unlock()
  1140  		return ErrBadServiceType
  1141  	}
  1142  	ea.latency = &serviceLatency{
  1143  		sampling: int8(sampling),
  1144  		subject:  results,
  1145  	}
  1146  	s := a.srv
  1147  	a.mu.Unlock()
  1148  
  1149  	if s == nil {
  1150  		return nil
  1151  	}
  1152  
  1153  	// Now track down the imports and add in latency as needed to enable.
  1154  	s.accounts.Range(func(k, v interface{}) bool {
  1155  		acc := v.(*Account)
  1156  		acc.mu.Lock()
  1157  		for _, im := range acc.imports.services {
  1158  			if im != nil && im.acc.Name == a.Name && subjectIsSubsetMatch(im.to, service) {
  1159  				im.latency = ea.latency
  1160  			}
  1161  		}
  1162  		acc.mu.Unlock()
  1163  		return true
  1164  	})
  1165  
  1166  	return nil
  1167  }
  1168  
  1169  // UnTrackServiceExport will disable latency tracking of the named service.
  1170  func (a *Account) UnTrackServiceExport(service string) {
  1171  	if a == nil || (a.srv != nil && !a.srv.EventsEnabled()) {
  1172  		return
  1173  	}
  1174  
  1175  	a.mu.Lock()
  1176  	if a.exports.services == nil {
  1177  		a.mu.Unlock()
  1178  		return
  1179  	}
  1180  	ea, ok := a.exports.services[service]
  1181  	if !ok || ea == nil || ea.latency == nil {
  1182  		a.mu.Unlock()
  1183  		return
  1184  	}
  1185  	// We have latency here.
  1186  	ea.latency = nil
  1187  	s := a.srv
  1188  	a.mu.Unlock()
  1189  
  1190  	if s == nil {
  1191  		return
  1192  	}
  1193  
  1194  	// Now track down the imports and clean them up.
  1195  	s.accounts.Range(func(k, v interface{}) bool {
  1196  		acc := v.(*Account)
  1197  		acc.mu.Lock()
  1198  		for _, im := range acc.imports.services {
  1199  			if im != nil && im.acc.Name == a.Name {
  1200  				if subjectIsSubsetMatch(im.to, service) {
  1201  					im.latency, im.m1 = nil, nil
  1202  				}
  1203  			}
  1204  		}
  1205  		acc.mu.Unlock()
  1206  		return true
  1207  	})
  1208  }
  1209  
  1210  // IsExportService will indicate if this service exists. Will check wildcard scenarios.
  1211  func (a *Account) IsExportService(service string) bool {
  1212  	a.mu.RLock()
  1213  	defer a.mu.RUnlock()
  1214  	_, ok := a.exports.services[service]
  1215  	if ok {
  1216  		return true
  1217  	}
  1218  	tokens := strings.Split(service, tsep)
  1219  	for subj := range a.exports.services {
  1220  		if isSubsetMatch(tokens, subj) {
  1221  			return true
  1222  		}
  1223  	}
  1224  	return false
  1225  }
  1226  
  1227  // IsExportServiceTracking will indicate if given publish subject is an export service with tracking enabled.
  1228  func (a *Account) IsExportServiceTracking(service string) bool {
  1229  	a.mu.RLock()
  1230  	ea, ok := a.exports.services[service]
  1231  	if ok && ea == nil {
  1232  		a.mu.RUnlock()
  1233  		return false
  1234  	}
  1235  	if ok && ea != nil && ea.latency != nil {
  1236  		a.mu.RUnlock()
  1237  		return true
  1238  	}
  1239  	// FIXME(dlc) - Might want to cache this is in the hot path checking for latency tracking.
  1240  	tokens := strings.Split(service, tsep)
  1241  	for subj, ea := range a.exports.services {
  1242  		if isSubsetMatch(tokens, subj) && ea != nil && ea.latency != nil {
  1243  			a.mu.RUnlock()
  1244  			return true
  1245  		}
  1246  	}
  1247  	a.mu.RUnlock()
  1248  	return false
  1249  }
  1250  
  1251  // ServiceLatency is the JSON message sent out in response to latency tracking for
  1252  // an accounts exported services. Additional client info is available in requestor
  1253  // and responder. Note that for a requestor, the only information shared by default
  1254  // is the RTT used to calculate the total latency. The requestor's account can
  1255  // designate to share the additional information in the service import.
  1256  type ServiceLatency struct {
  1257  	TypedEvent
  1258  	Status         int           `json:"status"`
  1259  	Error          string        `json:"description,omitempty"`
  1260  	Requestor      *ClientInfo   `json:"requestor,omitempty"`
  1261  	Responder      *ClientInfo   `json:"responder,omitempty"`
  1262  	RequestHeader  http.Header   `json:"header,omitempty"` // only contains header(s) triggering the measurement
  1263  	RequestStart   time.Time     `json:"start"`
  1264  	ServiceLatency time.Duration `json:"service"`
  1265  	SystemLatency  time.Duration `json:"system"`
  1266  	TotalLatency   time.Duration `json:"total"`
  1267  }
  1268  
  1269  // ServiceLatencyType is the NATS Event Type for ServiceLatency
  1270  const ServiceLatencyType = "io.nats.server.metric.v1.service_latency"
  1271  
  1272  // NATSTotalTime is a helper function that totals the NATS latencies.
  1273  func (m1 *ServiceLatency) NATSTotalTime() time.Duration {
  1274  	return m1.Requestor.RTT + m1.Responder.RTT + m1.SystemLatency
  1275  }
  1276  
  1277  // Merge function to merge m1 and m2 (requestor and responder) measurements
  1278  // when there are two samples. This happens when the requestor and responder
  1279  // are on different servers.
  1280  //
  1281  // m2 ServiceLatency is correct, so use that.
  1282  // m1 TotalLatency is correct, so use that.
  1283  // Will use those to back into NATS latency.
  1284  func (m1 *ServiceLatency) merge(m2 *ServiceLatency) {
  1285  	rtt := time.Duration(0)
  1286  	if m2.Responder != nil {
  1287  		rtt = m2.Responder.RTT
  1288  	}
  1289  	m1.SystemLatency = m1.ServiceLatency - (m2.ServiceLatency + rtt)
  1290  	m1.ServiceLatency = m2.ServiceLatency
  1291  	m1.Responder = m2.Responder
  1292  	sanitizeLatencyMetric(m1)
  1293  }
  1294  
  1295  // sanitizeLatencyMetric adjusts latency metric values that could go
  1296  // negative in some edge conditions since we estimate client RTT
  1297  // for both requestor and responder.
  1298  // These numbers are never meant to be negative, it just could be
  1299  // how we back into the values based on estimated RTT.
  1300  func sanitizeLatencyMetric(sl *ServiceLatency) {
  1301  	if sl.ServiceLatency < 0 {
  1302  		sl.ServiceLatency = 0
  1303  	}
  1304  	if sl.SystemLatency < 0 {
  1305  		sl.SystemLatency = 0
  1306  	}
  1307  }
  1308  
  1309  // Used for transporting remote latency measurements.
  1310  type remoteLatency struct {
  1311  	Account    string         `json:"account"`
  1312  	ReqId      string         `json:"req_id"`
  1313  	M2         ServiceLatency `json:"m2"`
  1314  	respThresh time.Duration
  1315  }
  1316  
  1317  // sendLatencyResult will send a latency result and clear the si of the requestor(rc).
  1318  func (a *Account) sendLatencyResult(si *serviceImport, sl *ServiceLatency) {
  1319  	sl.Type = ServiceLatencyType
  1320  	sl.ID = a.nextEventID()
  1321  	sl.Time = time.Now().UTC()
  1322  	a.mu.Lock()
  1323  	lsubj := si.latency.subject
  1324  	si.rc = nil
  1325  	a.mu.Unlock()
  1326  
  1327  	a.srv.sendInternalAccountMsg(a, lsubj, sl)
  1328  }
  1329  
  1330  // Used to send a bad request metric when we do not have a reply subject
  1331  func (a *Account) sendBadRequestTrackingLatency(si *serviceImport, requestor *client, header http.Header) {
  1332  	sl := &ServiceLatency{
  1333  		Status:    400,
  1334  		Error:     "Bad Request",
  1335  		Requestor: requestor.getClientInfo(si.share),
  1336  	}
  1337  	sl.RequestHeader = header
  1338  	sl.RequestStart = time.Now().Add(-sl.Requestor.RTT).UTC()
  1339  	a.sendLatencyResult(si, sl)
  1340  }
  1341  
  1342  // Used to send a latency result when the requestor interest was lost before the
  1343  // response could be delivered.
  1344  func (a *Account) sendReplyInterestLostTrackLatency(si *serviceImport) {
  1345  	sl := &ServiceLatency{
  1346  		Status: 408,
  1347  		Error:  "Request Timeout",
  1348  	}
  1349  	a.mu.RLock()
  1350  	rc, share, ts := si.rc, si.share, si.ts
  1351  	sl.RequestHeader = si.trackingHdr
  1352  	a.mu.RUnlock()
  1353  	if rc != nil {
  1354  		sl.Requestor = rc.getClientInfo(share)
  1355  	}
  1356  	sl.RequestStart = time.Unix(0, ts-int64(sl.Requestor.RTT)).UTC()
  1357  	a.sendLatencyResult(si, sl)
  1358  }
  1359  
  1360  func (a *Account) sendBackendErrorTrackingLatency(si *serviceImport, reason rsiReason) {
  1361  	sl := &ServiceLatency{}
  1362  	a.mu.RLock()
  1363  	rc, share, ts := si.rc, si.share, si.ts
  1364  	sl.RequestHeader = si.trackingHdr
  1365  	a.mu.RUnlock()
  1366  	if rc != nil {
  1367  		sl.Requestor = rc.getClientInfo(share)
  1368  	}
  1369  	var reqRTT time.Duration
  1370  	if sl.Requestor != nil {
  1371  		reqRTT = sl.Requestor.RTT
  1372  	}
  1373  	sl.RequestStart = time.Unix(0, ts-int64(reqRTT)).UTC()
  1374  	if reason == rsiNoDelivery {
  1375  		sl.Status = 503
  1376  		sl.Error = "Service Unavailable"
  1377  	} else if reason == rsiTimeout {
  1378  		sl.Status = 504
  1379  		sl.Error = "Service Timeout"
  1380  	}
  1381  	a.sendLatencyResult(si, sl)
  1382  }
  1383  
  1384  // sendTrackingLatency will send out the appropriate tracking information for the
  1385  // service request/response latency. This is called when the requestor's server has
  1386  // received the response.
  1387  // TODO(dlc) - holding locks for RTTs may be too much long term. Should revisit.
  1388  func (a *Account) sendTrackingLatency(si *serviceImport, responder *client) bool {
  1389  	a.mu.RLock()
  1390  	rc := si.rc
  1391  	a.mu.RUnlock()
  1392  	if rc == nil {
  1393  		return true
  1394  	}
  1395  
  1396  	ts := time.Now()
  1397  	serviceRTT := time.Duration(ts.UnixNano() - si.ts)
  1398  	requestor := si.rc
  1399  
  1400  	sl := &ServiceLatency{
  1401  		Status:    200,
  1402  		Requestor: requestor.getClientInfo(si.share),
  1403  		Responder: responder.getClientInfo(true),
  1404  	}
  1405  	var respRTT, reqRTT time.Duration
  1406  	if sl.Responder != nil {
  1407  		respRTT = sl.Responder.RTT
  1408  	}
  1409  	if sl.Requestor != nil {
  1410  		reqRTT = sl.Requestor.RTT
  1411  	}
  1412  	sl.RequestStart = time.Unix(0, si.ts-int64(reqRTT)).UTC()
  1413  	sl.ServiceLatency = serviceRTT - respRTT
  1414  	sl.TotalLatency = reqRTT + serviceRTT
  1415  	if respRTT > 0 {
  1416  		sl.SystemLatency = time.Since(ts)
  1417  		sl.TotalLatency += sl.SystemLatency
  1418  	}
  1419  	sl.RequestHeader = si.trackingHdr
  1420  	sanitizeLatencyMetric(sl)
  1421  
  1422  	sl.Type = ServiceLatencyType
  1423  	sl.ID = a.nextEventID()
  1424  	sl.Time = time.Now().UTC()
  1425  
  1426  	// If we are expecting a remote measurement, store our sl here.
  1427  	// We need to account for the race between this and us receiving the
  1428  	// remote measurement.
  1429  	// FIXME(dlc) - We need to clean these up but this should happen
  1430  	// already with the auto-expire logic.
  1431  	if responder != nil && responder.kind != CLIENT {
  1432  		si.acc.mu.Lock()
  1433  		if si.m1 != nil {
  1434  			m1, m2 := sl, si.m1
  1435  			m1.merge(m2)
  1436  			si.acc.mu.Unlock()
  1437  			a.srv.sendInternalAccountMsg(a, si.latency.subject, m1)
  1438  			a.mu.Lock()
  1439  			si.rc = nil
  1440  			a.mu.Unlock()
  1441  			return true
  1442  		}
  1443  		si.m1 = sl
  1444  		si.acc.mu.Unlock()
  1445  		return false
  1446  	} else {
  1447  		a.srv.sendInternalAccountMsg(a, si.latency.subject, sl)
  1448  		a.mu.Lock()
  1449  		si.rc = nil
  1450  		a.mu.Unlock()
  1451  	}
  1452  	return true
  1453  }
  1454  
  1455  // This will check to make sure our response lower threshold is set
  1456  // properly in any clients doing rrTracking.
  1457  func updateAllClientsServiceExportResponseTime(clients []*client, lrt time.Duration) {
  1458  	for _, c := range clients {
  1459  		c.mu.Lock()
  1460  		if c.rrTracking != nil && lrt != c.rrTracking.lrt {
  1461  			c.rrTracking.lrt = lrt
  1462  			if c.rrTracking.ptmr.Stop() {
  1463  				c.rrTracking.ptmr.Reset(lrt)
  1464  			}
  1465  		}
  1466  		c.mu.Unlock()
  1467  	}
  1468  }
  1469  
  1470  // Will select the lowest respThresh from all service exports.
  1471  // Read lock should be held.
  1472  func (a *Account) lowestServiceExportResponseTime() time.Duration {
  1473  	// Lowest we will allow is 5 minutes. Its an upper bound for this function.
  1474  	lrt := 5 * time.Minute
  1475  	for _, se := range a.exports.services {
  1476  		if se.respThresh < lrt {
  1477  			lrt = se.respThresh
  1478  		}
  1479  	}
  1480  	return lrt
  1481  }
  1482  
  1483  // AddServiceImportWithClaim will add in the service import via the jwt claim.
  1484  func (a *Account) AddServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import) error {
  1485  	return a.addServiceImportWithClaim(destination, from, to, imClaim, false)
  1486  }
  1487  
  1488  // addServiceImportWithClaim will add in the service import via the jwt claim.
  1489  // It will also skip the authorization check in cases where internal is true
  1490  func (a *Account) addServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import, internal bool) error {
  1491  	if destination == nil {
  1492  		return ErrMissingAccount
  1493  	}
  1494  	// Empty means use from.
  1495  	if to == _EMPTY_ {
  1496  		to = from
  1497  	}
  1498  	if !IsValidSubject(from) || !IsValidSubject(to) {
  1499  		return ErrInvalidSubject
  1500  	}
  1501  
  1502  	// First check to see if the account has authorized us to route to the "to" subject.
  1503  	if !internal && !destination.checkServiceImportAuthorized(a, to, imClaim) {
  1504  		return ErrServiceImportAuthorization
  1505  	}
  1506  
  1507  	// Check if this introduces a cycle before proceeding.
  1508  	if err := a.serviceImportFormsCycle(destination, from); err != nil {
  1509  		return err
  1510  	}
  1511  
  1512  	_, err := a.addServiceImport(destination, from, to, imClaim)
  1513  
  1514  	return err
  1515  }
  1516  
  1517  const MaxAccountCycleSearchDepth = 1024
  1518  
  1519  func (a *Account) serviceImportFormsCycle(dest *Account, from string) error {
  1520  	return dest.checkServiceImportsForCycles(from, map[string]bool{a.Name: true})
  1521  }
  1522  
  1523  func (a *Account) checkServiceImportsForCycles(from string, visited map[string]bool) error {
  1524  	if len(visited) >= MaxAccountCycleSearchDepth {
  1525  		return ErrCycleSearchDepth
  1526  	}
  1527  	a.mu.RLock()
  1528  	for _, si := range a.imports.services {
  1529  		if SubjectsCollide(from, si.to) {
  1530  			a.mu.RUnlock()
  1531  			if visited[si.acc.Name] {
  1532  				return ErrImportFormsCycle
  1533  			}
  1534  			// Push ourselves and check si.acc
  1535  			visited[a.Name] = true
  1536  			if subjectIsSubsetMatch(si.from, from) {
  1537  				from = si.from
  1538  			}
  1539  			if err := si.acc.checkServiceImportsForCycles(from, visited); err != nil {
  1540  				return err
  1541  			}
  1542  			a.mu.RLock()
  1543  		}
  1544  	}
  1545  	a.mu.RUnlock()
  1546  	return nil
  1547  }
  1548  
  1549  func (a *Account) streamImportFormsCycle(dest *Account, to string) error {
  1550  	return dest.checkStreamImportsForCycles(to, map[string]bool{a.Name: true})
  1551  }
  1552  
  1553  // Lock should be held.
  1554  func (a *Account) hasServiceExportMatching(to string) bool {
  1555  	for subj := range a.exports.services {
  1556  		if subjectIsSubsetMatch(to, subj) {
  1557  			return true
  1558  		}
  1559  	}
  1560  	return false
  1561  }
  1562  
  1563  // Lock should be held.
  1564  func (a *Account) hasStreamExportMatching(to string) bool {
  1565  	for subj := range a.exports.streams {
  1566  		if subjectIsSubsetMatch(to, subj) {
  1567  			return true
  1568  		}
  1569  	}
  1570  	return false
  1571  }
  1572  
  1573  func (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool) error {
  1574  	if len(visited) >= MaxAccountCycleSearchDepth {
  1575  		return ErrCycleSearchDepth
  1576  	}
  1577  
  1578  	a.mu.RLock()
  1579  
  1580  	if !a.hasStreamExportMatching(to) {
  1581  		a.mu.RUnlock()
  1582  		return nil
  1583  	}
  1584  
  1585  	for _, si := range a.imports.streams {
  1586  		if SubjectsCollide(to, si.to) {
  1587  			a.mu.RUnlock()
  1588  			if visited[si.acc.Name] {
  1589  				return ErrImportFormsCycle
  1590  			}
  1591  			// Push ourselves and check si.acc
  1592  			visited[a.Name] = true
  1593  			if subjectIsSubsetMatch(si.to, to) {
  1594  				to = si.to
  1595  			}
  1596  			if err := si.acc.checkStreamImportsForCycles(to, visited); err != nil {
  1597  				return err
  1598  			}
  1599  			a.mu.RLock()
  1600  		}
  1601  	}
  1602  	a.mu.RUnlock()
  1603  	return nil
  1604  }
  1605  
  1606  // SetServiceImportSharing will allow sharing of information about requests with the export account.
  1607  // Used for service latency tracking at the moment.
  1608  func (a *Account) SetServiceImportSharing(destination *Account, to string, allow bool) error {
  1609  	return a.setServiceImportSharing(destination, to, true, allow)
  1610  }
  1611  
  1612  // setServiceImportSharing will allow sharing of information about requests with the export account.
  1613  func (a *Account) setServiceImportSharing(destination *Account, to string, check, allow bool) error {
  1614  	a.mu.Lock()
  1615  	defer a.mu.Unlock()
  1616  	if check && a.isClaimAccount() {
  1617  		return fmt.Errorf("claim based accounts can not be updated directly")
  1618  	}
  1619  	for _, si := range a.imports.services {
  1620  		if si.acc == destination && si.to == to {
  1621  			si.share = allow
  1622  			return nil
  1623  		}
  1624  	}
  1625  	return fmt.Errorf("service import not found")
  1626  }
  1627  
  1628  // AddServiceImport will add a route to an account to send published messages / requests
  1629  // to the destination account. From is the local subject to map, To is the
  1630  // subject that will appear on the destination account. Destination will need
  1631  // to have an import rule to allow access via addService.
  1632  func (a *Account) AddServiceImport(destination *Account, from, to string) error {
  1633  	return a.AddServiceImportWithClaim(destination, from, to, nil)
  1634  }
  1635  
  1636  // NumPendingReverseResponses returns the number of response mappings we have for all outstanding
  1637  // requests for service imports.
  1638  func (a *Account) NumPendingReverseResponses() int {
  1639  	a.mu.RLock()
  1640  	defer a.mu.RUnlock()
  1641  	return len(a.imports.rrMap)
  1642  }
  1643  
  1644  // NumPendingAllResponses return the number of all responses outstanding for service exports.
  1645  func (a *Account) NumPendingAllResponses() int {
  1646  	return a.NumPendingResponses(_EMPTY_)
  1647  }
  1648  
  1649  // NumPendingResponses returns the number of responses outstanding for service exports
  1650  // on this account. An empty filter string returns all responses regardless of which export.
  1651  // If you specify the filter we will only return ones that are for that export.
  1652  // NOTE this is only for what this server is tracking.
  1653  func (a *Account) NumPendingResponses(filter string) int {
  1654  	a.mu.RLock()
  1655  	defer a.mu.RUnlock()
  1656  	if filter == _EMPTY_ {
  1657  		return len(a.exports.responses)
  1658  	}
  1659  	se := a.getServiceExport(filter)
  1660  	if se == nil {
  1661  		return 0
  1662  	}
  1663  	var nre int
  1664  	for _, si := range a.exports.responses {
  1665  		if si.se == se {
  1666  			nre++
  1667  		}
  1668  	}
  1669  	return nre
  1670  }
  1671  
  1672  // NumServiceImports returns the number of service imports we have configured.
  1673  func (a *Account) NumServiceImports() int {
  1674  	a.mu.RLock()
  1675  	defer a.mu.RUnlock()
  1676  	return len(a.imports.services)
  1677  }
  1678  
  1679  // Reason why we are removing this response serviceImport.
  1680  type rsiReason int
  1681  
  1682  const (
  1683  	rsiOk = rsiReason(iota)
  1684  	rsiNoDelivery
  1685  	rsiTimeout
  1686  )
  1687  
  1688  // removeRespServiceImport removes a response si mapping and the reverse entries for interest detection.
  1689  func (a *Account) removeRespServiceImport(si *serviceImport, reason rsiReason) {
  1690  	if si == nil {
  1691  		return
  1692  	}
  1693  
  1694  	a.mu.Lock()
  1695  	c := a.ic
  1696  	delete(a.exports.responses, si.from)
  1697  	dest, to, tracking, rc, didDeliver := si.acc, si.to, si.tracking, si.rc, si.didDeliver
  1698  	a.mu.Unlock()
  1699  
  1700  	// If we have a sid make sure to unsub.
  1701  	if len(si.sid) > 0 && c != nil {
  1702  		c.processUnsub(si.sid)
  1703  	}
  1704  
  1705  	if tracking && rc != nil && !didDeliver {
  1706  		a.sendBackendErrorTrackingLatency(si, reason)
  1707  	}
  1708  
  1709  	dest.checkForReverseEntry(to, si, false)
  1710  }
  1711  
  1712  // removeServiceImport will remove the route by subject.
  1713  func (a *Account) removeServiceImport(subject string) {
  1714  	a.mu.Lock()
  1715  	si, ok := a.imports.services[subject]
  1716  	delete(a.imports.services, subject)
  1717  
  1718  	var sid []byte
  1719  	c := a.ic
  1720  
  1721  	if ok && si != nil {
  1722  		if a.ic != nil && si.sid != nil {
  1723  			sid = si.sid
  1724  		}
  1725  	}
  1726  	a.mu.Unlock()
  1727  
  1728  	if sid != nil {
  1729  		c.processUnsub(sid)
  1730  	}
  1731  }
  1732  
  1733  // This tracks responses to service requests mappings. This is used for cleanup.
  1734  func (a *Account) addReverseRespMapEntry(acc *Account, reply, from string) {
  1735  	a.mu.Lock()
  1736  	if a.imports.rrMap == nil {
  1737  		a.imports.rrMap = make(map[string][]*serviceRespEntry)
  1738  	}
  1739  	sre := &serviceRespEntry{acc, from}
  1740  	sra := a.imports.rrMap[reply]
  1741  	a.imports.rrMap[reply] = append(sra, sre)
  1742  	a.mu.Unlock()
  1743  }
  1744  
  1745  // checkForReverseEntries is for when we are trying to match reverse entries to a wildcard.
  1746  // This will be called from checkForReverseEntry when the reply arg is a wildcard subject.
  1747  // This will usually be called in a go routine since we need to walk all the entries.
  1748  func (a *Account) checkForReverseEntries(reply string, checkInterest, recursed bool) {
  1749  	if subjectIsLiteral(reply) {
  1750  		a._checkForReverseEntry(reply, nil, checkInterest, recursed)
  1751  		return
  1752  	}
  1753  
  1754  	a.mu.RLock()
  1755  	if len(a.imports.rrMap) == 0 {
  1756  		a.mu.RUnlock()
  1757  		return
  1758  	}
  1759  
  1760  	var _rs [64]string
  1761  	rs := _rs[:0]
  1762  	if n := len(a.imports.rrMap); n > cap(rs) {
  1763  		rs = make([]string, 0, n)
  1764  	}
  1765  
  1766  	for k := range a.imports.rrMap {
  1767  		rs = append(rs, k)
  1768  	}
  1769  	a.mu.RUnlock()
  1770  
  1771  	tsa := [32]string{}
  1772  	tts := tokenizeSubjectIntoSlice(tsa[:0], reply)
  1773  
  1774  	rsa := [32]string{}
  1775  	for _, r := range rs {
  1776  		rts := tokenizeSubjectIntoSlice(rsa[:0], r)
  1777  		//  isSubsetMatchTokenized is heavy so make sure we do this without the lock.
  1778  		if isSubsetMatchTokenized(rts, tts) {
  1779  			a._checkForReverseEntry(r, nil, checkInterest, recursed)
  1780  		}
  1781  	}
  1782  }
  1783  
  1784  // This checks for any response map entries. If you specify an si we will only match and
  1785  // clean up for that one, otherwise we remove them all.
  1786  func (a *Account) checkForReverseEntry(reply string, si *serviceImport, checkInterest bool) {
  1787  	a._checkForReverseEntry(reply, si, checkInterest, false)
  1788  }
  1789  
  1790  // Callers should use checkForReverseEntry instead. This function exists to help prevent
  1791  // infinite recursion.
  1792  func (a *Account) _checkForReverseEntry(reply string, si *serviceImport, checkInterest, recursed bool) {
  1793  	a.mu.RLock()
  1794  	if len(a.imports.rrMap) == 0 {
  1795  		a.mu.RUnlock()
  1796  		return
  1797  	}
  1798  
  1799  	if subjectHasWildcard(reply) {
  1800  		if recursed {
  1801  			// If we have reached this condition then it is because the reverse entries also
  1802  			// contain wildcards (that shouldn't happen but a client *could* provide an inbox
  1803  			// prefix that is illegal because it ends in a wildcard character), at which point
  1804  			// we will end up with infinite recursion between this func and checkForReverseEntries.
  1805  			// To avoid a stack overflow panic, we'll give up instead.
  1806  			a.mu.RUnlock()
  1807  			return
  1808  		}
  1809  
  1810  		doInline := len(a.imports.rrMap) <= 64
  1811  		a.mu.RUnlock()
  1812  
  1813  		if doInline {
  1814  			a.checkForReverseEntries(reply, checkInterest, true)
  1815  		} else {
  1816  			go a.checkForReverseEntries(reply, checkInterest, true)
  1817  		}
  1818  		return
  1819  	}
  1820  
  1821  	if sres := a.imports.rrMap[reply]; sres == nil {
  1822  		a.mu.RUnlock()
  1823  		return
  1824  	}
  1825  
  1826  	// If we are here we have an entry we should check.
  1827  	// If requested we will first check if there is any
  1828  	// interest for this subject for the entire account.
  1829  	// If there is we can not delete any entries yet.
  1830  	// Note that if we are here reply has to be a literal subject.
  1831  	if checkInterest {
  1832  		// If interest still exists we can not clean these up yet.
  1833  		if rr := a.sl.Match(reply); len(rr.psubs)+len(rr.qsubs) > 0 {
  1834  			a.mu.RUnlock()
  1835  			return
  1836  		}
  1837  	}
  1838  	a.mu.RUnlock()
  1839  
  1840  	// Delete the appropriate entries here based on optional si.
  1841  	a.mu.Lock()
  1842  	// We need a new lookup here because we have released the lock.
  1843  	sres := a.imports.rrMap[reply]
  1844  	if si == nil {
  1845  		delete(a.imports.rrMap, reply)
  1846  	} else if sres != nil {
  1847  		// Find the one we are looking for..
  1848  		for i, sre := range sres {
  1849  			if sre.msub == si.from {
  1850  				sres = append(sres[:i], sres[i+1:]...)
  1851  				break
  1852  			}
  1853  		}
  1854  		if len(sres) > 0 {
  1855  			a.imports.rrMap[si.to] = sres
  1856  		} else {
  1857  			delete(a.imports.rrMap, si.to)
  1858  		}
  1859  	}
  1860  	a.mu.Unlock()
  1861  
  1862  	// If we are here we no longer have interest and we have
  1863  	// response entries that we should clean up.
  1864  	if si == nil {
  1865  		// sres is now known to have been removed from a.imports.rrMap, so we
  1866  		// can safely (data race wise) iterate through.
  1867  		for _, sre := range sres {
  1868  			acc := sre.acc
  1869  			var trackingCleanup bool
  1870  			var rsi *serviceImport
  1871  			acc.mu.Lock()
  1872  			c := acc.ic
  1873  			if rsi = acc.exports.responses[sre.msub]; rsi != nil && !rsi.didDeliver {
  1874  				delete(acc.exports.responses, rsi.from)
  1875  				trackingCleanup = rsi.tracking && rsi.rc != nil
  1876  			}
  1877  			acc.mu.Unlock()
  1878  			// If we are doing explicit subs for all responses (e.g. bound to leafnode)
  1879  			// we will have a non-empty sid here.
  1880  			if rsi != nil && len(rsi.sid) > 0 && c != nil {
  1881  				c.processUnsub(rsi.sid)
  1882  			}
  1883  			if trackingCleanup {
  1884  				acc.sendReplyInterestLostTrackLatency(rsi)
  1885  			}
  1886  		}
  1887  	}
  1888  }
  1889  
  1890  // Checks to see if a potential service import subject is already overshadowed.
  1891  func (a *Account) serviceImportShadowed(from string) bool {
  1892  	a.mu.RLock()
  1893  	defer a.mu.RUnlock()
  1894  	if a.imports.services[from] != nil {
  1895  		return true
  1896  	}
  1897  	// We did not find a direct match, so check individually.
  1898  	for subj := range a.imports.services {
  1899  		if subjectIsSubsetMatch(from, subj) {
  1900  			return true
  1901  		}
  1902  	}
  1903  	return false
  1904  }
  1905  
  1906  // Internal check to see if a service import exists.
  1907  func (a *Account) serviceImportExists(from string) bool {
  1908  	a.mu.RLock()
  1909  	dup := a.imports.services[from]
  1910  	a.mu.RUnlock()
  1911  	return dup != nil
  1912  }
  1913  
  1914  // Add a service import.
  1915  // This does no checks and should only be called by the msg processing code.
  1916  // Use AddServiceImport from above if responding to user input or config changes, etc.
  1917  func (a *Account) addServiceImport(dest *Account, from, to string, claim *jwt.Import) (*serviceImport, error) {
  1918  	rt := Singleton
  1919  	var lat *serviceLatency
  1920  
  1921  	if dest == nil {
  1922  		return nil, ErrMissingAccount
  1923  	}
  1924  
  1925  	var atrc bool
  1926  	dest.mu.RLock()
  1927  	se := dest.getServiceExport(to)
  1928  	if se != nil {
  1929  		rt = se.respType
  1930  		lat = se.latency
  1931  		atrc = se.atrc
  1932  	}
  1933  	dest.mu.RUnlock()
  1934  
  1935  	a.mu.Lock()
  1936  	if a.imports.services == nil {
  1937  		a.imports.services = make(map[string]*serviceImport)
  1938  	} else if dup := a.imports.services[from]; dup != nil {
  1939  		a.mu.Unlock()
  1940  		return nil, fmt.Errorf("duplicate service import subject %q, previously used in import for account %q, subject %q",
  1941  			from, dup.acc.Name, dup.to)
  1942  	}
  1943  
  1944  	if to == _EMPTY_ {
  1945  		to = from
  1946  	}
  1947  	// Check to see if we have a wildcard
  1948  	var (
  1949  		usePub bool
  1950  		tr     *subjectTransform
  1951  		err    error
  1952  	)
  1953  	if subjectHasWildcard(to) {
  1954  		// If to and from match, then we use the published subject.
  1955  		if to == from {
  1956  			usePub = true
  1957  		} else {
  1958  			to, _ = transformUntokenize(to)
  1959  			// Create a transform. Do so in reverse such that $ symbols only exist in to
  1960  			if tr, err = NewSubjectTransformStrict(to, transformTokenize(from)); err != nil {
  1961  				a.mu.Unlock()
  1962  				return nil, fmt.Errorf("failed to create mapping transform for service import subject from %q to %q: %v",
  1963  					from, to, err)
  1964  			} else {
  1965  				// un-tokenize and reverse transform so we get the transform needed
  1966  				from, _ = transformUntokenize(from)
  1967  				tr = tr.reverse()
  1968  			}
  1969  		}
  1970  	}
  1971  	var share bool
  1972  	if claim != nil {
  1973  		share = claim.Share
  1974  	}
  1975  	si := &serviceImport{dest, claim, se, nil, from, to, tr, 0, rt, lat, nil, nil, usePub, false, false, share, false, false, atrc, nil}
  1976  	a.imports.services[from] = si
  1977  	a.mu.Unlock()
  1978  
  1979  	if err := a.addServiceImportSub(si); err != nil {
  1980  		a.removeServiceImport(si.from)
  1981  		return nil, err
  1982  	}
  1983  	return si, nil
  1984  }
  1985  
  1986  // Returns the internal client, will create one if not present.
  1987  // Lock should be held.
  1988  func (a *Account) internalClient() *client {
  1989  	if a.ic == nil && a.srv != nil {
  1990  		a.ic = a.srv.createInternalAccountClient()
  1991  		a.ic.acc = a
  1992  	}
  1993  	return a.ic
  1994  }
  1995  
  1996  // Internal account scoped subscriptions.
  1997  func (a *Account) subscribeInternal(subject string, cb msgHandler) (*subscription, error) {
  1998  	return a.subscribeInternalEx(subject, cb, false)
  1999  }
  2000  
  2001  // Unsubscribe from an internal account subscription.
  2002  func (a *Account) unsubscribeInternal(sub *subscription) {
  2003  	if ic := a.internalClient(); ic != nil {
  2004  		ic.processUnsub(sub.sid)
  2005  	}
  2006  }
  2007  
  2008  // Creates internal subscription for service import responses.
  2009  func (a *Account) subscribeServiceImportResponse(subject string) (*subscription, error) {
  2010  	return a.subscribeInternalEx(subject, a.processServiceImportResponse, true)
  2011  }
  2012  
  2013  func (a *Account) subscribeInternalEx(subject string, cb msgHandler, ri bool) (*subscription, error) {
  2014  	a.mu.Lock()
  2015  	a.isid++
  2016  	c, sid := a.internalClient(), strconv.FormatUint(a.isid, 10)
  2017  	a.mu.Unlock()
  2018  
  2019  	// This will happen in parsing when the account has not been properly setup.
  2020  	if c == nil {
  2021  		return nil, fmt.Errorf("no internal account client")
  2022  	}
  2023  
  2024  	return c.processSubEx([]byte(subject), nil, []byte(sid), cb, false, false, ri)
  2025  }
  2026  
  2027  // This will add an account subscription that matches the "from" from a service import entry.
  2028  func (a *Account) addServiceImportSub(si *serviceImport) error {
  2029  	a.mu.Lock()
  2030  	c := a.internalClient()
  2031  	// This will happen in parsing when the account has not been properly setup.
  2032  	if c == nil {
  2033  		a.mu.Unlock()
  2034  		return nil
  2035  	}
  2036  	if si.sid != nil {
  2037  		a.mu.Unlock()
  2038  		return fmt.Errorf("duplicate call to create subscription for service import")
  2039  	}
  2040  	a.isid++
  2041  	sid := strconv.FormatUint(a.isid, 10)
  2042  	si.sid = []byte(sid)
  2043  	subject := si.from
  2044  	a.mu.Unlock()
  2045  
  2046  	cb := func(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) {
  2047  		c.processServiceImport(si, acc, msg)
  2048  	}
  2049  	sub, err := c.processSubEx([]byte(subject), nil, []byte(sid), cb, true, true, false)
  2050  	if err != nil {
  2051  		return err
  2052  	}
  2053  	// Leafnodes introduce a new way to introduce messages into the system. Therefore forward import subscription
  2054  	// This is similar to what initLeafNodeSmapAndSendSubs does
  2055  	// TODO we need to consider performing this update as we get client subscriptions.
  2056  	//      This behavior would result in subscription propagation only where actually used.
  2057  	a.updateLeafNodes(sub, 1)
  2058  	return nil
  2059  }
  2060  
  2061  // Remove all the subscriptions associated with service imports.
  2062  func (a *Account) removeAllServiceImportSubs() {
  2063  	a.mu.RLock()
  2064  	var sids [][]byte
  2065  	for _, si := range a.imports.services {
  2066  		if si.sid != nil {
  2067  			sids = append(sids, si.sid)
  2068  			si.sid = nil
  2069  		}
  2070  	}
  2071  	c := a.ic
  2072  	a.ic = nil
  2073  	a.mu.RUnlock()
  2074  
  2075  	if c == nil {
  2076  		return
  2077  	}
  2078  	for _, sid := range sids {
  2079  		c.processUnsub(sid)
  2080  	}
  2081  	c.closeConnection(InternalClient)
  2082  }
  2083  
  2084  // Add in subscriptions for all registered service imports.
  2085  func (a *Account) addAllServiceImportSubs() {
  2086  	var sis [32]*serviceImport
  2087  	serviceImports := sis[:0]
  2088  	a.mu.RLock()
  2089  	for _, si := range a.imports.services {
  2090  		serviceImports = append(serviceImports, si)
  2091  	}
  2092  	a.mu.RUnlock()
  2093  	for _, si := range serviceImports {
  2094  		a.addServiceImportSub(si)
  2095  	}
  2096  }
  2097  
  2098  var (
  2099  	// header where all information is encoded in one value.
  2100  	trcUber = textproto.CanonicalMIMEHeaderKey("Uber-Trace-Id")
  2101  	trcCtx  = textproto.CanonicalMIMEHeaderKey("Traceparent")
  2102  	trcB3   = textproto.CanonicalMIMEHeaderKey("B3")
  2103  	// openzipkin header to check
  2104  	trcB3Sm = textproto.CanonicalMIMEHeaderKey("X-B3-Sampled")
  2105  	trcB3Id = textproto.CanonicalMIMEHeaderKey("X-B3-TraceId")
  2106  	// additional header needed to include when present
  2107  	trcB3PSId        = textproto.CanonicalMIMEHeaderKey("X-B3-ParentSpanId")
  2108  	trcB3SId         = textproto.CanonicalMIMEHeaderKey("X-B3-SpanId")
  2109  	trcCtxSt         = textproto.CanonicalMIMEHeaderKey("Tracestate")
  2110  	trcUberCtxPrefix = textproto.CanonicalMIMEHeaderKey("Uberctx-")
  2111  )
  2112  
  2113  func newB3Header(h http.Header) http.Header {
  2114  	retHdr := http.Header{}
  2115  	if v, ok := h[trcB3Sm]; ok {
  2116  		retHdr[trcB3Sm] = v
  2117  	}
  2118  	if v, ok := h[trcB3Id]; ok {
  2119  		retHdr[trcB3Id] = v
  2120  	}
  2121  	if v, ok := h[trcB3PSId]; ok {
  2122  		retHdr[trcB3PSId] = v
  2123  	}
  2124  	if v, ok := h[trcB3SId]; ok {
  2125  		retHdr[trcB3SId] = v
  2126  	}
  2127  	return retHdr
  2128  }
  2129  
  2130  func newUberHeader(h http.Header, tId []string) http.Header {
  2131  	retHdr := http.Header{trcUber: tId}
  2132  	for k, v := range h {
  2133  		if strings.HasPrefix(k, trcUberCtxPrefix) {
  2134  			retHdr[k] = v
  2135  		}
  2136  	}
  2137  	return retHdr
  2138  }
  2139  
  2140  func newTraceCtxHeader(h http.Header, tId []string) http.Header {
  2141  	retHdr := http.Header{trcCtx: tId}
  2142  	if v, ok := h[trcCtxSt]; ok {
  2143  		retHdr[trcCtxSt] = v
  2144  	}
  2145  	return retHdr
  2146  }
  2147  
  2148  // Helper to determine when to sample. When header has a value, sampling is driven by header
  2149  func shouldSample(l *serviceLatency, c *client) (bool, http.Header) {
  2150  	if l == nil {
  2151  		return false, nil
  2152  	}
  2153  	if l.sampling < 0 {
  2154  		return false, nil
  2155  	}
  2156  	if l.sampling >= 100 {
  2157  		return true, nil
  2158  	}
  2159  	if l.sampling > 0 && rand.Int31n(100) <= int32(l.sampling) {
  2160  		return true, nil
  2161  	}
  2162  	h := c.parseState.getHeader()
  2163  	if len(h) == 0 {
  2164  		return false, nil
  2165  	}
  2166  	if tId := h[trcUber]; len(tId) != 0 {
  2167  		// sample 479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1
  2168  		tk := strings.Split(tId[0], ":")
  2169  		if len(tk) == 4 && len(tk[3]) > 0 && len(tk[3]) <= 2 {
  2170  			dst := [2]byte{}
  2171  			src := [2]byte{'0', tk[3][0]}
  2172  			if len(tk[3]) == 2 {
  2173  				src[1] = tk[3][1]
  2174  			}
  2175  			if _, err := hex.Decode(dst[:], src[:]); err == nil && dst[0]&1 == 1 {
  2176  				return true, newUberHeader(h, tId)
  2177  			}
  2178  		}
  2179  		return false, nil
  2180  	} else if sampled := h[trcB3Sm]; len(sampled) != 0 && sampled[0] == "1" {
  2181  		return true, newB3Header(h) // allowed
  2182  	} else if len(sampled) != 0 && sampled[0] == "0" {
  2183  		return false, nil // denied
  2184  	} else if _, ok := h[trcB3Id]; ok {
  2185  		// sample 80f198ee56343ba864fe8b2a57d3eff7
  2186  		// presence (with X-B3-Sampled not being 0) means sampling left to recipient
  2187  		return true, newB3Header(h)
  2188  	} else if b3 := h[trcB3]; len(b3) != 0 {
  2189  		// sample 80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90
  2190  		// sample 0
  2191  		tk := strings.Split(b3[0], "-")
  2192  		if len(tk) > 2 && tk[2] == "0" {
  2193  			return false, nil // denied
  2194  		} else if len(tk) == 1 && tk[0] == "0" {
  2195  			return false, nil // denied
  2196  		}
  2197  		return true, http.Header{trcB3: b3} // sampling allowed or left to recipient of header
  2198  	} else if tId := h[trcCtx]; len(tId) != 0 {
  2199  		var sample bool
  2200  		// sample 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
  2201  		tk := strings.Split(tId[0], "-")
  2202  		if len(tk) == 4 && len([]byte(tk[3])) == 2 {
  2203  			if hexVal, err := strconv.ParseInt(tk[3], 16, 8); err == nil {
  2204  				sample = hexVal&0x1 == 0x1
  2205  			}
  2206  		}
  2207  		if sample {
  2208  			return true, newTraceCtxHeader(h, tId)
  2209  		} else {
  2210  			return false, nil
  2211  		}
  2212  	}
  2213  	return false, nil
  2214  }
  2215  
  2216  // Used to mimic client like replies.
  2217  const (
  2218  	replyPrefix    = "_R_."
  2219  	replyPrefixLen = len(replyPrefix)
  2220  	baseServerLen  = 10
  2221  	replyLen       = 6
  2222  	minReplyLen    = 15
  2223  	digits         = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
  2224  	base           = 62
  2225  )
  2226  
  2227  // This is where all service export responses are handled.
  2228  func (a *Account) processServiceImportResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {
  2229  	a.mu.RLock()
  2230  	if a.expired || len(a.exports.responses) == 0 {
  2231  		a.mu.RUnlock()
  2232  		return
  2233  	}
  2234  	si := a.exports.responses[subject]
  2235  
  2236  	if si == nil || si.invalid {
  2237  		a.mu.RUnlock()
  2238  		return
  2239  	}
  2240  	a.mu.RUnlock()
  2241  
  2242  	// Send for normal processing.
  2243  	c.processServiceImport(si, a, msg)
  2244  }
  2245  
  2246  // Will create the response prefix for fast generation of responses.
  2247  // A wildcard subscription may be used handle interest graph propagation
  2248  // for all service replies, unless we are bound to a leafnode.
  2249  // Lock should be held.
  2250  func (a *Account) createRespWildcard() {
  2251  	var b = [baseServerLen]byte{'_', 'R', '_', '.'}
  2252  	rn := fastrand.Uint64()
  2253  	for i, l := replyPrefixLen, rn; i < len(b); i++ {
  2254  		b[i] = digits[l%base]
  2255  		l /= base
  2256  	}
  2257  	a.siReply = append(b[:], '.')
  2258  }
  2259  
  2260  // Test whether this is a tracked reply.
  2261  func isTrackedReply(reply []byte) bool {
  2262  	lreply := len(reply) - 1
  2263  	return lreply > 3 && reply[lreply-1] == '.' && reply[lreply] == 'T'
  2264  }
  2265  
  2266  // Generate a new service reply from the wildcard prefix.
  2267  // FIXME(dlc) - probably do not have to use rand here. about 25ns per.
  2268  func (a *Account) newServiceReply(tracking bool) []byte {
  2269  	a.mu.Lock()
  2270  	s := a.srv
  2271  	rn := fastrand.Uint64()
  2272  
  2273  	// Check if we need to create the reply here.
  2274  	var createdSiReply bool
  2275  	if a.siReply == nil {
  2276  		a.createRespWildcard()
  2277  		createdSiReply = true
  2278  	}
  2279  	replyPre := a.siReply
  2280  	a.mu.Unlock()
  2281  
  2282  	// If we created the siReply and we are not bound to a leafnode
  2283  	// we need to do the wildcard subscription.
  2284  	if createdSiReply {
  2285  		a.subscribeServiceImportResponse(string(append(replyPre, '>')))
  2286  	}
  2287  
  2288  	var b [replyLen]byte
  2289  	for i, l := 0, rn; i < len(b); i++ {
  2290  		b[i] = digits[l%base]
  2291  		l /= base
  2292  	}
  2293  	// Make sure to copy.
  2294  	reply := make([]byte, 0, len(replyPre)+len(b))
  2295  	reply = append(reply, replyPre...)
  2296  	reply = append(reply, b[:]...)
  2297  
  2298  	if tracking && s.sys != nil {
  2299  		// Add in our tracking identifier. This allows the metrics to get back to only
  2300  		// this server without needless SUBS/UNSUBS.
  2301  		reply = append(reply, '.')
  2302  		reply = append(reply, s.sys.shash...)
  2303  		reply = append(reply, '.', 'T')
  2304  	}
  2305  
  2306  	return reply
  2307  }
  2308  
  2309  // Checks if a serviceImport was created to map responses.
  2310  func (si *serviceImport) isRespServiceImport() bool {
  2311  	return si != nil && si.response
  2312  }
  2313  
  2314  // Sets the response threshold timer for a service export.
  2315  // Account lock should be held
  2316  func (se *serviceExport) setResponseThresholdTimer() {
  2317  	if se.rtmr != nil {
  2318  		return // Already set
  2319  	}
  2320  	se.rtmr = time.AfterFunc(se.respThresh, se.checkExpiredResponses)
  2321  }
  2322  
  2323  // Account lock should be held
  2324  func (se *serviceExport) clearResponseThresholdTimer() bool {
  2325  	if se.rtmr == nil {
  2326  		return true
  2327  	}
  2328  	stopped := se.rtmr.Stop()
  2329  	se.rtmr = nil
  2330  	return stopped
  2331  }
  2332  
  2333  // checkExpiredResponses will check for any pending responses that need to
  2334  // be cleaned up.
  2335  func (se *serviceExport) checkExpiredResponses() {
  2336  	acc := se.acc
  2337  	if acc == nil {
  2338  		se.clearResponseThresholdTimer()
  2339  		return
  2340  	}
  2341  
  2342  	var expired []*serviceImport
  2343  	mints := time.Now().UnixNano() - int64(se.respThresh)
  2344  
  2345  	// TODO(dlc) - Should we release lock while doing this? Or only do these in batches?
  2346  	// Should we break this up for responses only from this service export?
  2347  	// Responses live on acc directly for fast inbound processsing for the _R_ wildcard.
  2348  	// We could do another indirection at this level but just to get to the service export?
  2349  	var totalResponses int
  2350  	acc.mu.RLock()
  2351  	for _, si := range acc.exports.responses {
  2352  		if si.se == se {
  2353  			totalResponses++
  2354  			if si.ts <= mints {
  2355  				expired = append(expired, si)
  2356  			}
  2357  		}
  2358  	}
  2359  	acc.mu.RUnlock()
  2360  
  2361  	for _, si := range expired {
  2362  		acc.removeRespServiceImport(si, rsiTimeout)
  2363  	}
  2364  
  2365  	// Pull out expired to determine if we have any left for timer.
  2366  	totalResponses -= len(expired)
  2367  
  2368  	// Redo timer as needed.
  2369  	acc.mu.Lock()
  2370  	if totalResponses > 0 && se.rtmr != nil {
  2371  		se.rtmr.Stop()
  2372  		se.rtmr.Reset(se.respThresh)
  2373  	} else {
  2374  		se.clearResponseThresholdTimer()
  2375  	}
  2376  	acc.mu.Unlock()
  2377  }
  2378  
  2379  // ServiceExportResponseThreshold returns the current threshold.
  2380  func (a *Account) ServiceExportResponseThreshold(export string) (time.Duration, error) {
  2381  	a.mu.Lock()
  2382  	defer a.mu.Unlock()
  2383  	se := a.getServiceExport(export)
  2384  	if se == nil {
  2385  		return 0, fmt.Errorf("no export defined for %q", export)
  2386  	}
  2387  	return se.respThresh, nil
  2388  }
  2389  
  2390  // SetServiceExportResponseThreshold sets the maximum time the system will a response to be delivered
  2391  // from a service export responder.
  2392  func (a *Account) SetServiceExportResponseThreshold(export string, maxTime time.Duration) error {
  2393  	a.mu.Lock()
  2394  	if a.isClaimAccount() {
  2395  		a.mu.Unlock()
  2396  		return fmt.Errorf("claim based accounts can not be updated directly")
  2397  	}
  2398  	lrt := a.lowestServiceExportResponseTime()
  2399  	se := a.getServiceExport(export)
  2400  	if se == nil {
  2401  		a.mu.Unlock()
  2402  		return fmt.Errorf("no export defined for %q", export)
  2403  	}
  2404  	se.respThresh = maxTime
  2405  
  2406  	var clients []*client
  2407  	nlrt := a.lowestServiceExportResponseTime()
  2408  	if nlrt != lrt && len(a.clients) > 0 {
  2409  		clients = a.getClientsLocked()
  2410  	}
  2411  	// Need to release because lock ordering is client -> Account
  2412  	a.mu.Unlock()
  2413  	if len(clients) > 0 {
  2414  		updateAllClientsServiceExportResponseTime(clients, nlrt)
  2415  	}
  2416  	return nil
  2417  }
  2418  
  2419  func (a *Account) SetServiceExportAllowTrace(export string, allowTrace bool) error {
  2420  	a.mu.Lock()
  2421  	se := a.getServiceExport(export)
  2422  	if se == nil {
  2423  		a.mu.Unlock()
  2424  		return fmt.Errorf("no export defined for %q", export)
  2425  	}
  2426  	se.atrc = allowTrace
  2427  	a.mu.Unlock()
  2428  	return nil
  2429  }
  2430  
  2431  // This is for internal service import responses.
  2432  func (a *Account) addRespServiceImport(dest *Account, to string, osi *serviceImport, tracking bool, header http.Header) *serviceImport {
  2433  	nrr := string(osi.acc.newServiceReply(tracking))
  2434  
  2435  	a.mu.Lock()
  2436  	rt := osi.rt
  2437  
  2438  	// dest is the requestor's account. a is the service responder with the export.
  2439  	// Marked as internal here, that is how we distinguish.
  2440  	si := &serviceImport{dest, nil, osi.se, nil, nrr, to, nil, 0, rt, nil, nil, nil, false, true, false, osi.share, false, false, false, nil}
  2441  
  2442  	if a.exports.responses == nil {
  2443  		a.exports.responses = make(map[string]*serviceImport)
  2444  	}
  2445  	a.exports.responses[nrr] = si
  2446  
  2447  	// Always grab time and make sure response threshold timer is running.
  2448  	si.ts = time.Now().UnixNano()
  2449  	if osi.se != nil {
  2450  		osi.se.setResponseThresholdTimer()
  2451  	}
  2452  
  2453  	if rt == Singleton && tracking {
  2454  		si.latency = osi.latency
  2455  		si.tracking = true
  2456  		si.trackingHdr = header
  2457  	}
  2458  	a.mu.Unlock()
  2459  
  2460  	// We do add in the reverse map such that we can detect loss of interest and do proper
  2461  	// cleanup of this si as interest goes away.
  2462  	dest.addReverseRespMapEntry(a, to, nrr)
  2463  
  2464  	return si
  2465  }
  2466  
  2467  // AddStreamImportWithClaim will add in the stream import from a specific account with optional token.
  2468  func (a *Account) AddStreamImportWithClaim(account *Account, from, prefix string, imClaim *jwt.Import) error {
  2469  	return a.addStreamImportWithClaim(account, from, prefix, false, imClaim)
  2470  }
  2471  
  2472  func (a *Account) addStreamImportWithClaim(account *Account, from, prefix string, allowTrace bool, imClaim *jwt.Import) error {
  2473  	if account == nil {
  2474  		return ErrMissingAccount
  2475  	}
  2476  
  2477  	// First check to see if the account has authorized export of the subject.
  2478  	if !account.checkStreamImportAuthorized(a, from, imClaim) {
  2479  		return ErrStreamImportAuthorization
  2480  	}
  2481  
  2482  	// Check prefix if it exists and make sure its a literal.
  2483  	// Append token separator if not already present.
  2484  	if prefix != _EMPTY_ {
  2485  		// Make sure there are no wildcards here, this prefix needs to be a literal
  2486  		// since it will be prepended to a publish subject.
  2487  		if !subjectIsLiteral(prefix) {
  2488  			return ErrStreamImportBadPrefix
  2489  		}
  2490  		if prefix[len(prefix)-1] != btsep {
  2491  			prefix = prefix + string(btsep)
  2492  		}
  2493  	}
  2494  
  2495  	return a.addMappedStreamImportWithClaim(account, from, prefix+from, allowTrace, imClaim)
  2496  }
  2497  
  2498  // AddMappedStreamImport helper for AddMappedStreamImportWithClaim
  2499  func (a *Account) AddMappedStreamImport(account *Account, from, to string) error {
  2500  	return a.AddMappedStreamImportWithClaim(account, from, to, nil)
  2501  }
  2502  
  2503  // AddMappedStreamImportWithClaim will add in the stream import from a specific account with optional token.
  2504  func (a *Account) AddMappedStreamImportWithClaim(account *Account, from, to string, imClaim *jwt.Import) error {
  2505  	return a.addMappedStreamImportWithClaim(account, from, to, false, imClaim)
  2506  }
  2507  
  2508  func (a *Account) addMappedStreamImportWithClaim(account *Account, from, to string, allowTrace bool, imClaim *jwt.Import) error {
  2509  	if account == nil {
  2510  		return ErrMissingAccount
  2511  	}
  2512  
  2513  	// First check to see if the account has authorized export of the subject.
  2514  	if !account.checkStreamImportAuthorized(a, from, imClaim) {
  2515  		return ErrStreamImportAuthorization
  2516  	}
  2517  
  2518  	if to == _EMPTY_ {
  2519  		to = from
  2520  	}
  2521  
  2522  	// Check if this forms a cycle.
  2523  	if err := a.streamImportFormsCycle(account, to); err != nil {
  2524  		return err
  2525  	}
  2526  
  2527  	var (
  2528  		usePub bool
  2529  		tr     *subjectTransform
  2530  		err    error
  2531  	)
  2532  	if subjectHasWildcard(from) {
  2533  		if to == from {
  2534  			usePub = true
  2535  		} else {
  2536  			// Create a transform
  2537  			if tr, err = NewSubjectTransformStrict(from, transformTokenize(to)); err != nil {
  2538  				return fmt.Errorf("failed to create mapping transform for stream import subject from %q to %q: %v",
  2539  					from, to, err)
  2540  			}
  2541  			to, _ = transformUntokenize(to)
  2542  		}
  2543  	}
  2544  
  2545  	a.mu.Lock()
  2546  	if a.isStreamImportDuplicate(account, from) {
  2547  		a.mu.Unlock()
  2548  		return ErrStreamImportDuplicate
  2549  	}
  2550  	if imClaim != nil {
  2551  		allowTrace = imClaim.AllowTrace
  2552  	}
  2553  	a.imports.streams = append(a.imports.streams, &streamImport{account, from, to, tr, nil, imClaim, usePub, false, allowTrace})
  2554  	a.mu.Unlock()
  2555  	return nil
  2556  }
  2557  
  2558  // isStreamImportDuplicate checks for duplicate.
  2559  // Lock should be held.
  2560  func (a *Account) isStreamImportDuplicate(acc *Account, from string) bool {
  2561  	for _, si := range a.imports.streams {
  2562  		if si.acc == acc && si.from == from {
  2563  			return true
  2564  		}
  2565  	}
  2566  	return false
  2567  }
  2568  
  2569  // AddStreamImport will add in the stream import from a specific account.
  2570  func (a *Account) AddStreamImport(account *Account, from, prefix string) error {
  2571  	return a.addStreamImportWithClaim(account, from, prefix, false, nil)
  2572  }
  2573  
  2574  // IsPublicExport is a placeholder to denote a public export.
  2575  var IsPublicExport = []*Account(nil)
  2576  
  2577  // AddStreamExport will add an export to the account. If accounts is nil
  2578  // it will signify a public export, meaning anyone can import.
  2579  func (a *Account) AddStreamExport(subject string, accounts []*Account) error {
  2580  	return a.addStreamExportWithAccountPos(subject, accounts, 0)
  2581  }
  2582  
  2583  // AddStreamExport will add an export to the account. If accounts is nil
  2584  // it will signify a public export, meaning anyone can import.
  2585  // if accountPos is > 0, all imports will be granted where the following holds:
  2586  // strings.Split(subject, tsep)[accountPos] == account id will be granted.
  2587  func (a *Account) addStreamExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error {
  2588  	if a == nil {
  2589  		return ErrMissingAccount
  2590  	}
  2591  
  2592  	a.mu.Lock()
  2593  	defer a.mu.Unlock()
  2594  
  2595  	if a.exports.streams == nil {
  2596  		a.exports.streams = make(map[string]*streamExport)
  2597  	}
  2598  	ea := a.exports.streams[subject]
  2599  	if accounts != nil || accountPos > 0 {
  2600  		if ea == nil {
  2601  			ea = &streamExport{}
  2602  		}
  2603  		if err := setExportAuth(&ea.exportAuth, subject, accounts, accountPos); err != nil {
  2604  			return err
  2605  		}
  2606  	}
  2607  	a.exports.streams[subject] = ea
  2608  	return nil
  2609  }
  2610  
  2611  // Check if another account is authorized to import from us.
  2612  func (a *Account) checkStreamImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool {
  2613  	// Find the subject in the exports list.
  2614  	a.mu.RLock()
  2615  	auth := a.checkStreamImportAuthorizedNoLock(account, subject, imClaim)
  2616  	a.mu.RUnlock()
  2617  	return auth
  2618  }
  2619  
  2620  func (a *Account) checkStreamImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool {
  2621  	if a.exports.streams == nil || !IsValidSubject(subject) {
  2622  		return false
  2623  	}
  2624  	return a.checkStreamExportApproved(account, subject, imClaim)
  2625  }
  2626  
  2627  func (a *Account) checkAuth(ea *exportAuth, account *Account, imClaim *jwt.Import, tokens []string) bool {
  2628  	// if ea is nil or ea.approved is nil, that denotes a public export
  2629  	if ea == nil || (len(ea.approved) == 0 && !ea.tokenReq && ea.accountPos == 0) {
  2630  		return true
  2631  	}
  2632  	// Check if the export is protected and enforces presence of importing account identity
  2633  	if ea.accountPos > 0 {
  2634  		return ea.accountPos <= uint(len(tokens)) && tokens[ea.accountPos-1] == account.Name
  2635  	}
  2636  	// Check if token required
  2637  	if ea.tokenReq {
  2638  		return a.checkActivation(account, imClaim, ea, true)
  2639  	}
  2640  	if ea.approved == nil {
  2641  		return false
  2642  	}
  2643  	// If we have a matching account we are authorized
  2644  	_, ok := ea.approved[account.Name]
  2645  	return ok
  2646  }
  2647  
  2648  func (a *Account) checkStreamExportApproved(account *Account, subject string, imClaim *jwt.Import) bool {
  2649  	// Check direct match of subject first
  2650  	ea, ok := a.exports.streams[subject]
  2651  	if ok {
  2652  		// if ea is nil or eq.approved is nil, that denotes a public export
  2653  		if ea == nil {
  2654  			return true
  2655  		}
  2656  		return a.checkAuth(&ea.exportAuth, account, imClaim, nil)
  2657  	}
  2658  
  2659  	// ok if we are here we did not match directly so we need to test each one.
  2660  	// The import subject arg has to take precedence, meaning the export
  2661  	// has to be a true subset of the import claim. We already checked for
  2662  	// exact matches above.
  2663  	tokens := strings.Split(subject, tsep)
  2664  	for subj, ea := range a.exports.streams {
  2665  		if isSubsetMatch(tokens, subj) {
  2666  			if ea == nil {
  2667  				return true
  2668  			}
  2669  			return a.checkAuth(&ea.exportAuth, account, imClaim, tokens)
  2670  		}
  2671  	}
  2672  	return false
  2673  }
  2674  
  2675  func (a *Account) checkServiceExportApproved(account *Account, subject string, imClaim *jwt.Import) bool {
  2676  	// Check direct match of subject first
  2677  	se, ok := a.exports.services[subject]
  2678  	if ok {
  2679  		// if se is nil or eq.approved is nil, that denotes a public export
  2680  		if se == nil {
  2681  			return true
  2682  		}
  2683  		return a.checkAuth(&se.exportAuth, account, imClaim, nil)
  2684  	}
  2685  	// ok if we are here we did not match directly so we need to test each one.
  2686  	// The import subject arg has to take precedence, meaning the export
  2687  	// has to be a true subset of the import claim. We already checked for
  2688  	// exact matches above.
  2689  	tokens := strings.Split(subject, tsep)
  2690  	for subj, se := range a.exports.services {
  2691  		if isSubsetMatch(tokens, subj) {
  2692  			if se == nil {
  2693  				return true
  2694  			}
  2695  			return a.checkAuth(&se.exportAuth, account, imClaim, tokens)
  2696  		}
  2697  	}
  2698  	return false
  2699  }
  2700  
  2701  // Helper function to get a serviceExport.
  2702  // Lock should be held on entry.
  2703  func (a *Account) getServiceExport(subj string) *serviceExport {
  2704  	se, ok := a.exports.services[subj]
  2705  	// The export probably has a wildcard, so lookup that up.
  2706  	if !ok {
  2707  		se = a.getWildcardServiceExport(subj)
  2708  	}
  2709  	return se
  2710  }
  2711  
  2712  // This helper is used when trying to match a serviceExport record that is
  2713  // represented by a wildcard.
  2714  // Lock should be held on entry.
  2715  func (a *Account) getWildcardServiceExport(from string) *serviceExport {
  2716  	tokens := strings.Split(from, tsep)
  2717  	for subj, se := range a.exports.services {
  2718  		if isSubsetMatch(tokens, subj) {
  2719  			return se
  2720  		}
  2721  	}
  2722  	return nil
  2723  }
  2724  
  2725  // These are import stream specific versions for when an activation expires.
  2726  func (a *Account) streamActivationExpired(exportAcc *Account, subject string) {
  2727  	a.mu.RLock()
  2728  	if a.expired || a.imports.streams == nil {
  2729  		a.mu.RUnlock()
  2730  		return
  2731  	}
  2732  	var si *streamImport
  2733  	for _, si = range a.imports.streams {
  2734  		if si.acc == exportAcc && si.from == subject {
  2735  			break
  2736  		}
  2737  	}
  2738  
  2739  	if si == nil || si.invalid {
  2740  		a.mu.RUnlock()
  2741  		return
  2742  	}
  2743  	a.mu.RUnlock()
  2744  
  2745  	if si.acc.checkActivation(a, si.claim, nil, false) {
  2746  		// The token has been updated most likely and we are good to go.
  2747  		return
  2748  	}
  2749  
  2750  	a.mu.Lock()
  2751  	si.invalid = true
  2752  	clients := a.getClientsLocked()
  2753  	awcsti := map[string]struct{}{a.Name: {}}
  2754  	a.mu.Unlock()
  2755  	for _, c := range clients {
  2756  		c.processSubsOnConfigReload(awcsti)
  2757  	}
  2758  }
  2759  
  2760  // These are import service specific versions for when an activation expires.
  2761  func (a *Account) serviceActivationExpired(subject string) {
  2762  	a.mu.RLock()
  2763  	if a.expired || a.imports.services == nil {
  2764  		a.mu.RUnlock()
  2765  		return
  2766  	}
  2767  	si := a.imports.services[subject]
  2768  	if si == nil || si.invalid {
  2769  		a.mu.RUnlock()
  2770  		return
  2771  	}
  2772  	a.mu.RUnlock()
  2773  
  2774  	if si.acc.checkActivation(a, si.claim, nil, false) {
  2775  		// The token has been updated most likely and we are good to go.
  2776  		return
  2777  	}
  2778  
  2779  	a.mu.Lock()
  2780  	si.invalid = true
  2781  	a.mu.Unlock()
  2782  }
  2783  
  2784  // Fires for expired activation tokens. We could track this with timers etc.
  2785  // Instead we just re-analyze where we are and if we need to act.
  2786  func (a *Account) activationExpired(exportAcc *Account, subject string, kind jwt.ExportType) {
  2787  	switch kind {
  2788  	case jwt.Stream:
  2789  		a.streamActivationExpired(exportAcc, subject)
  2790  	case jwt.Service:
  2791  		a.serviceActivationExpired(subject)
  2792  	}
  2793  }
  2794  
  2795  func isRevoked(revocations map[string]int64, subject string, issuedAt int64) bool {
  2796  	if len(revocations) == 0 {
  2797  		return false
  2798  	}
  2799  	if t, ok := revocations[subject]; !ok || t < issuedAt {
  2800  		if t, ok := revocations[jwt.All]; !ok || t < issuedAt {
  2801  			return false
  2802  		}
  2803  	}
  2804  	return true
  2805  }
  2806  
  2807  // checkActivation will check the activation token for validity.
  2808  // ea may only be nil in cases where revocation may not be checked, say triggered by expiration timer.
  2809  func (a *Account) checkActivation(importAcc *Account, claim *jwt.Import, ea *exportAuth, expTimer bool) bool {
  2810  	if claim == nil || claim.Token == _EMPTY_ {
  2811  		return false
  2812  	}
  2813  	// Create a quick clone so we can inline Token JWT.
  2814  	clone := *claim
  2815  
  2816  	vr := jwt.CreateValidationResults()
  2817  	clone.Validate(importAcc.Name, vr)
  2818  	if vr.IsBlocking(true) {
  2819  		return false
  2820  	}
  2821  	act, err := jwt.DecodeActivationClaims(clone.Token)
  2822  	if err != nil {
  2823  		return false
  2824  	}
  2825  	if !a.isIssuerClaimTrusted(act) {
  2826  		return false
  2827  	}
  2828  	vr = jwt.CreateValidationResults()
  2829  	act.Validate(vr)
  2830  	if vr.IsBlocking(true) {
  2831  		return false
  2832  	}
  2833  	if act.Expires != 0 {
  2834  		tn := time.Now().Unix()
  2835  		if act.Expires <= tn {
  2836  			return false
  2837  		}
  2838  		if expTimer {
  2839  			expiresAt := time.Duration(act.Expires - tn)
  2840  			time.AfterFunc(expiresAt*time.Second, func() {
  2841  				importAcc.activationExpired(a, string(act.ImportSubject), claim.Type)
  2842  			})
  2843  		}
  2844  	}
  2845  	if ea == nil {
  2846  		return true
  2847  	}
  2848  	// Check for token revocation..
  2849  	return !isRevoked(ea.actsRevoked, act.Subject, act.IssuedAt)
  2850  }
  2851  
  2852  // Returns true if the activation claim is trusted. That is the issuer matches
  2853  // the account or is an entry in the signing keys.
  2854  func (a *Account) isIssuerClaimTrusted(claims *jwt.ActivationClaims) bool {
  2855  	// if no issuer account, issuer is the account
  2856  	if claims.IssuerAccount == _EMPTY_ {
  2857  		return true
  2858  	}
  2859  	// If the IssuerAccount is not us, then this is considered an error.
  2860  	if a.Name != claims.IssuerAccount {
  2861  		if a.srv != nil {
  2862  			a.srv.Errorf("Invalid issuer account %q in activation claim (subject: %q - type: %q) for account %q",
  2863  				claims.IssuerAccount, claims.Activation.ImportSubject, claims.Activation.ImportType, a.Name)
  2864  		}
  2865  		return false
  2866  	}
  2867  	_, ok := a.hasIssuerNoLock(claims.Issuer)
  2868  	return ok
  2869  }
  2870  
  2871  // Returns true if `a` and `b` stream imports are the same. Note that the
  2872  // check is done with the account's name, not the pointer. This is used
  2873  // during config reload where we are comparing current and new config
  2874  // in which pointers are different.
  2875  // No lock is acquired in this function, so it is assumed that the
  2876  // import maps are not changed while this executes.
  2877  func (a *Account) checkStreamImportsEqual(b *Account) bool {
  2878  	if len(a.imports.streams) != len(b.imports.streams) {
  2879  		return false
  2880  	}
  2881  	// Load the b imports into a map index by what we are looking for.
  2882  	bm := make(map[string]*streamImport, len(b.imports.streams))
  2883  	for _, bim := range b.imports.streams {
  2884  		bm[bim.acc.Name+bim.from+bim.to] = bim
  2885  	}
  2886  	for _, aim := range a.imports.streams {
  2887  		if bim, ok := bm[aim.acc.Name+aim.from+aim.to]; !ok {
  2888  			return false
  2889  		} else if aim.atrc != bim.atrc {
  2890  			return false
  2891  		}
  2892  	}
  2893  	return true
  2894  }
  2895  
  2896  // Returns true if `a` and `b` stream exports are the same.
  2897  // Acquires `a` read lock, but `b` is assumed to not be accessed
  2898  // by anyone but the caller (`b` is not registered anywhere).
  2899  func (a *Account) checkStreamExportsEqual(b *Account) bool {
  2900  	a.mu.RLock()
  2901  	defer a.mu.RUnlock()
  2902  	if len(a.exports.streams) != len(b.exports.streams) {
  2903  		return false
  2904  	}
  2905  	for subj, aea := range a.exports.streams {
  2906  		bea, ok := b.exports.streams[subj]
  2907  		if !ok {
  2908  			return false
  2909  		}
  2910  		if !isStreamExportEqual(aea, bea) {
  2911  			return false
  2912  		}
  2913  	}
  2914  	return true
  2915  }
  2916  
  2917  func isStreamExportEqual(a, b *streamExport) bool {
  2918  	if a == nil && b == nil {
  2919  		return true
  2920  	}
  2921  	if (a == nil && b != nil) || (a != nil && b == nil) {
  2922  		return false
  2923  	}
  2924  	return isExportAuthEqual(&a.exportAuth, &b.exportAuth)
  2925  }
  2926  
  2927  // Returns true if `a` and `b` service exports are the same.
  2928  // Acquires `a` read lock, but `b` is assumed to not be accessed
  2929  // by anyone but the caller (`b` is not registered anywhere).
  2930  func (a *Account) checkServiceExportsEqual(b *Account) bool {
  2931  	a.mu.RLock()
  2932  	defer a.mu.RUnlock()
  2933  	if len(a.exports.services) != len(b.exports.services) {
  2934  		return false
  2935  	}
  2936  	for subj, aea := range a.exports.services {
  2937  		bea, ok := b.exports.services[subj]
  2938  		if !ok {
  2939  			return false
  2940  		}
  2941  		if !isServiceExportEqual(aea, bea) {
  2942  			return false
  2943  		}
  2944  	}
  2945  	return true
  2946  }
  2947  
  2948  func isServiceExportEqual(a, b *serviceExport) bool {
  2949  	if a == nil && b == nil {
  2950  		return true
  2951  	}
  2952  	if (a == nil && b != nil) || (a != nil && b == nil) {
  2953  		return false
  2954  	}
  2955  	if !isExportAuthEqual(&a.exportAuth, &b.exportAuth) {
  2956  		return false
  2957  	}
  2958  	if a.acc.Name != b.acc.Name {
  2959  		return false
  2960  	}
  2961  	if a.respType != b.respType {
  2962  		return false
  2963  	}
  2964  	if a.latency != nil || b.latency != nil {
  2965  		if (a.latency != nil && b.latency == nil) || (a.latency == nil && b.latency != nil) {
  2966  			return false
  2967  		}
  2968  		if a.latency.sampling != b.latency.sampling {
  2969  			return false
  2970  		}
  2971  		if a.latency.subject != b.latency.subject {
  2972  			return false
  2973  		}
  2974  	}
  2975  	if a.atrc != b.atrc {
  2976  		return false
  2977  	}
  2978  	return true
  2979  }
  2980  
  2981  // Returns true if `a` and `b` exportAuth structures are equal.
  2982  // Both `a` and `b` are guaranteed to be non-nil.
  2983  // Locking is handled by the caller.
  2984  func isExportAuthEqual(a, b *exportAuth) bool {
  2985  	if a.tokenReq != b.tokenReq {
  2986  		return false
  2987  	}
  2988  	if a.accountPos != b.accountPos {
  2989  		return false
  2990  	}
  2991  	if len(a.approved) != len(b.approved) {
  2992  		return false
  2993  	}
  2994  	for ak, av := range a.approved {
  2995  		if bv, ok := b.approved[ak]; !ok || av.Name != bv.Name {
  2996  			return false
  2997  		}
  2998  	}
  2999  	if len(a.actsRevoked) != len(b.actsRevoked) {
  3000  		return false
  3001  	}
  3002  	for ak, av := range a.actsRevoked {
  3003  		if bv, ok := b.actsRevoked[ak]; !ok || av != bv {
  3004  			return false
  3005  		}
  3006  	}
  3007  	return true
  3008  }
  3009  
  3010  // Check if another account is authorized to route requests to this service.
  3011  func (a *Account) checkServiceImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool {
  3012  	a.mu.RLock()
  3013  	authorized := a.checkServiceImportAuthorizedNoLock(account, subject, imClaim)
  3014  	a.mu.RUnlock()
  3015  	return authorized
  3016  }
  3017  
  3018  // Check if another account is authorized to route requests to this service.
  3019  func (a *Account) checkServiceImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool {
  3020  	// Find the subject in the services list.
  3021  	if a.exports.services == nil {
  3022  		return false
  3023  	}
  3024  	return a.checkServiceExportApproved(account, subject, imClaim)
  3025  }
  3026  
  3027  // IsExpired returns expiration status.
  3028  func (a *Account) IsExpired() bool {
  3029  	a.mu.RLock()
  3030  	exp := a.expired
  3031  	a.mu.RUnlock()
  3032  	return exp
  3033  }
  3034  
  3035  // Called when an account has expired.
  3036  func (a *Account) expiredTimeout() {
  3037  	// Mark expired first.
  3038  	a.mu.Lock()
  3039  	a.expired = true
  3040  	a.mu.Unlock()
  3041  
  3042  	// Collect the clients and expire them.
  3043  	cs := a.getClients()
  3044  	for _, c := range cs {
  3045  		c.accountAuthExpired()
  3046  	}
  3047  }
  3048  
  3049  // Sets the expiration timer for an account JWT that has it set.
  3050  func (a *Account) setExpirationTimer(d time.Duration) {
  3051  	a.etmr = time.AfterFunc(d, a.expiredTimeout)
  3052  }
  3053  
  3054  // Lock should be held
  3055  func (a *Account) clearExpirationTimer() bool {
  3056  	if a.etmr == nil {
  3057  		return true
  3058  	}
  3059  	stopped := a.etmr.Stop()
  3060  	a.etmr = nil
  3061  	return stopped
  3062  }
  3063  
  3064  // checkUserRevoked will check if a user has been revoked.
  3065  func (a *Account) checkUserRevoked(nkey string, issuedAt int64) bool {
  3066  	a.mu.RLock()
  3067  	defer a.mu.RUnlock()
  3068  	return isRevoked(a.usersRevoked, nkey, issuedAt)
  3069  }
  3070  
  3071  // failBearer will return if bearer token are allowed (false) or disallowed (true)
  3072  func (a *Account) failBearer() bool {
  3073  	a.mu.RLock()
  3074  	defer a.mu.RUnlock()
  3075  	return a.disallowBearer
  3076  }
  3077  
  3078  // Check expiration and set the proper state as needed.
  3079  func (a *Account) checkExpiration(claims *jwt.ClaimsData) {
  3080  	a.mu.Lock()
  3081  	defer a.mu.Unlock()
  3082  
  3083  	a.clearExpirationTimer()
  3084  	if claims.Expires == 0 {
  3085  		a.expired = false
  3086  		return
  3087  	}
  3088  	tn := time.Now().Unix()
  3089  	if claims.Expires <= tn {
  3090  		a.expired = true
  3091  		return
  3092  	}
  3093  	expiresAt := time.Duration(claims.Expires - tn)
  3094  	a.setExpirationTimer(expiresAt * time.Second)
  3095  	a.expired = false
  3096  }
  3097  
  3098  // hasIssuer returns true if the issuer matches the account
  3099  // If the issuer is a scoped signing key, the scope will be returned as well
  3100  // issuer or it is a signing key for the account.
  3101  func (a *Account) hasIssuer(issuer string) (jwt.Scope, bool) {
  3102  	a.mu.RLock()
  3103  	scope, ok := a.hasIssuerNoLock(issuer)
  3104  	a.mu.RUnlock()
  3105  	return scope, ok
  3106  }
  3107  
  3108  // hasIssuerNoLock is the unlocked version of hasIssuer
  3109  func (a *Account) hasIssuerNoLock(issuer string) (jwt.Scope, bool) {
  3110  	scope, ok := a.signingKeys[issuer]
  3111  	return scope, ok
  3112  }
  3113  
  3114  // Returns the loop detection subject used for leafnodes
  3115  func (a *Account) getLDSubject() string {
  3116  	a.mu.RLock()
  3117  	lds := a.lds
  3118  	a.mu.RUnlock()
  3119  	return lds
  3120  }
  3121  
  3122  // Placeholder for signaling token auth required.
  3123  var tokenAuthReq = []*Account{}
  3124  
  3125  func authAccounts(tokenReq bool) []*Account {
  3126  	if tokenReq {
  3127  		return tokenAuthReq
  3128  	}
  3129  	return nil
  3130  }
  3131  
  3132  // SetAccountResolver will assign the account resolver.
  3133  func (s *Server) SetAccountResolver(ar AccountResolver) {
  3134  	s.mu.Lock()
  3135  	s.accResolver = ar
  3136  	s.mu.Unlock()
  3137  }
  3138  
  3139  // AccountResolver returns the registered account resolver.
  3140  func (s *Server) AccountResolver() AccountResolver {
  3141  	s.mu.Lock()
  3142  	ar := s.accResolver
  3143  	s.mu.Unlock()
  3144  	return ar
  3145  }
  3146  
  3147  // isClaimAccount returns if this account is backed by a JWT claim.
  3148  // Lock should be held.
  3149  func (a *Account) isClaimAccount() bool {
  3150  	return a.claimJWT != _EMPTY_
  3151  }
  3152  
  3153  // UpdateAccountClaims will update an existing account with new claims.
  3154  // This will replace any exports or imports previously defined.
  3155  // Lock MUST NOT be held upon entry.
  3156  func (s *Server) UpdateAccountClaims(a *Account, ac *jwt.AccountClaims) {
  3157  	s.updateAccountClaimsWithRefresh(a, ac, true)
  3158  }
  3159  
  3160  func (a *Account) traceLabel() string {
  3161  	if a == nil {
  3162  		return _EMPTY_
  3163  	}
  3164  	if a.nameTag != _EMPTY_ {
  3165  		return fmt.Sprintf("%s/%s", a.Name, a.nameTag)
  3166  	}
  3167  	return a.Name
  3168  }
  3169  
  3170  // Check if an account has external auth set.
  3171  // Operator/Account Resolver only.
  3172  func (a *Account) hasExternalAuth() bool {
  3173  	if a == nil {
  3174  		return false
  3175  	}
  3176  	a.mu.RLock()
  3177  	defer a.mu.RUnlock()
  3178  	return a.extAuth != nil
  3179  }
  3180  
  3181  // Deterimine if this is an external auth user.
  3182  func (a *Account) isExternalAuthUser(userID string) bool {
  3183  	if a == nil {
  3184  		return false
  3185  	}
  3186  	a.mu.RLock()
  3187  	defer a.mu.RUnlock()
  3188  	if a.extAuth != nil {
  3189  		for _, u := range a.extAuth.AuthUsers {
  3190  			if userID == u {
  3191  				return true
  3192  			}
  3193  		}
  3194  	}
  3195  	return false
  3196  }
  3197  
  3198  // Return the external authorization xkey if external authorization is enabled and the xkey is set.
  3199  // Operator/Account Resolver only.
  3200  func (a *Account) externalAuthXKey() string {
  3201  	if a == nil {
  3202  		return _EMPTY_
  3203  	}
  3204  	a.mu.RLock()
  3205  	defer a.mu.RUnlock()
  3206  	if a.extAuth != nil && a.extAuth.XKey != _EMPTY_ {
  3207  		return a.extAuth.XKey
  3208  	}
  3209  	return _EMPTY_
  3210  }
  3211  
  3212  // Check if an account switch for external authorization is allowed.
  3213  func (a *Account) isAllowedAcount(acc string) bool {
  3214  	if a == nil {
  3215  		return false
  3216  	}
  3217  	a.mu.RLock()
  3218  	defer a.mu.RUnlock()
  3219  	if a.extAuth != nil {
  3220  		// if we have a single allowed account, and we have a wildcard
  3221  		// we accept it
  3222  		if len(a.extAuth.AllowedAccounts) == 1 &&
  3223  			a.extAuth.AllowedAccounts[0] == jwt.AnyAccount {
  3224  			return true
  3225  		}
  3226  		// otherwise must match exactly
  3227  		for _, a := range a.extAuth.AllowedAccounts {
  3228  			if a == acc {
  3229  				return true
  3230  			}
  3231  		}
  3232  	}
  3233  	return false
  3234  }
  3235  
  3236  // updateAccountClaimsWithRefresh will update an existing account with new claims.
  3237  // If refreshImportingAccounts is true it will also update incomplete dependent accounts
  3238  // This will replace any exports or imports previously defined.
  3239  // Lock MUST NOT be held upon entry.
  3240  func (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaims, refreshImportingAccounts bool) {
  3241  	if a == nil {
  3242  		return
  3243  	}
  3244  	s.Debugf("Updating account claims: %s/%s", a.Name, ac.Name)
  3245  	a.checkExpiration(ac.Claims())
  3246  
  3247  	a.mu.Lock()
  3248  	// Clone to update, only select certain fields.
  3249  	old := &Account{Name: a.Name, exports: a.exports, limits: a.limits, signingKeys: a.signingKeys}
  3250  
  3251  	// overwrite claim meta data
  3252  	a.nameTag = ac.Name
  3253  	a.tags = ac.Tags
  3254  
  3255  	var td string
  3256  	var tds int
  3257  	if ac.Trace != nil {
  3258  		// Update trace destination and sampling
  3259  		td, tds = string(ac.Trace.Destination), ac.Trace.Sampling
  3260  		if !IsValidPublishSubject(td) {
  3261  			td, tds = _EMPTY_, 0
  3262  		} else if tds <= 0 || tds > 100 {
  3263  			tds = 100
  3264  		}
  3265  	}
  3266  	a.traceDest, a.traceDestSampling = td, tds
  3267  
  3268  	// Check for external authorization.
  3269  	if ac.HasExternalAuthorization() {
  3270  		a.extAuth = &jwt.ExternalAuthorization{}
  3271  		a.extAuth.AuthUsers.Add(ac.Authorization.AuthUsers...)
  3272  		a.extAuth.AllowedAccounts.Add(ac.Authorization.AllowedAccounts...)
  3273  		a.extAuth.XKey = ac.Authorization.XKey
  3274  	}
  3275  
  3276  	// Reset exports and imports here.
  3277  
  3278  	// Exports is creating a whole new map.
  3279  	a.exports = exportMap{}
  3280  
  3281  	// Imports are checked unlocked in processInbound, so we can't change out the struct here. Need to process inline.
  3282  	if a.imports.streams != nil {
  3283  		old.imports.streams = a.imports.streams
  3284  		a.imports.streams = nil
  3285  	}
  3286  	if a.imports.services != nil {
  3287  		old.imports.services = make(map[string]*serviceImport, len(a.imports.services))
  3288  	}
  3289  	for k, v := range a.imports.services {
  3290  		old.imports.services[k] = v
  3291  		delete(a.imports.services, k)
  3292  	}
  3293  
  3294  	alteredScope := map[string]struct{}{}
  3295  
  3296  	// update account signing keys
  3297  	a.signingKeys = nil
  3298  	_, strict := s.strictSigningKeyUsage[a.Issuer]
  3299  	if len(ac.SigningKeys) > 0 || !strict {
  3300  		a.signingKeys = make(map[string]jwt.Scope)
  3301  	}
  3302  	signersChanged := false
  3303  	for k, scope := range ac.SigningKeys {
  3304  		a.signingKeys[k] = scope
  3305  	}
  3306  	if !strict {
  3307  		a.signingKeys[a.Name] = nil
  3308  	}
  3309  	if len(a.signingKeys) != len(old.signingKeys) {
  3310  		signersChanged = true
  3311  	}
  3312  	for k, scope := range a.signingKeys {
  3313  		if oldScope, ok := old.signingKeys[k]; !ok {
  3314  			signersChanged = true
  3315  		} else if !reflect.DeepEqual(scope, oldScope) {
  3316  			signersChanged = true
  3317  			alteredScope[k] = struct{}{}
  3318  		}
  3319  	}
  3320  	// collect mappings that need to be removed
  3321  	removeList := []string{}
  3322  	for _, m := range a.mappings {
  3323  		if _, ok := ac.Mappings[jwt.Subject(m.src)]; !ok {
  3324  			removeList = append(removeList, m.src)
  3325  		}
  3326  	}
  3327  	a.mu.Unlock()
  3328  
  3329  	for sub, wm := range ac.Mappings {
  3330  		mappings := make([]*MapDest, len(wm))
  3331  		for i, m := range wm {
  3332  			mappings[i] = &MapDest{
  3333  				Subject: string(m.Subject),
  3334  				Weight:  m.GetWeight(),
  3335  				Cluster: m.Cluster,
  3336  			}
  3337  		}
  3338  		// This will overwrite existing entries
  3339  		a.AddWeightedMappings(string(sub), mappings...)
  3340  	}
  3341  	// remove mappings
  3342  	for _, rmMapping := range removeList {
  3343  		a.RemoveMapping(rmMapping)
  3344  	}
  3345  
  3346  	// Re-register system exports/imports.
  3347  	if a == s.SystemAccount() {
  3348  		s.addSystemAccountExports(a)
  3349  	} else {
  3350  		s.registerSystemImports(a)
  3351  	}
  3352  
  3353  	jsEnabled := s.JetStreamEnabled()
  3354  
  3355  	streamTokenExpirationChanged := false
  3356  	serviceTokenExpirationChanged := false
  3357  
  3358  	for _, e := range ac.Exports {
  3359  		switch e.Type {
  3360  		case jwt.Stream:
  3361  			s.Debugf("Adding stream export %q for %s", e.Subject, a.traceLabel())
  3362  			if err := a.addStreamExportWithAccountPos(
  3363  				string(e.Subject), authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil {
  3364  				s.Debugf("Error adding stream export to account [%s]: %v", a.traceLabel(), err.Error())
  3365  			}
  3366  		case jwt.Service:
  3367  			s.Debugf("Adding service export %q for %s", e.Subject, a.traceLabel())
  3368  			rt := Singleton
  3369  			switch e.ResponseType {
  3370  			case jwt.ResponseTypeStream:
  3371  				rt = Streamed
  3372  			case jwt.ResponseTypeChunked:
  3373  				rt = Chunked
  3374  			}
  3375  			if err := a.addServiceExportWithResponseAndAccountPos(
  3376  				string(e.Subject), rt, authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil {
  3377  				s.Debugf("Error adding service export to account [%s]: %v", a.traceLabel(), err)
  3378  				continue
  3379  			}
  3380  			sub := string(e.Subject)
  3381  			if e.Latency != nil {
  3382  				if err := a.TrackServiceExportWithSampling(sub, string(e.Latency.Results), int(e.Latency.Sampling)); err != nil {
  3383  					hdrNote := _EMPTY_
  3384  					if e.Latency.Sampling == jwt.Headers {
  3385  						hdrNote = " (using headers)"
  3386  					}
  3387  					s.Debugf("Error adding latency tracking%s for service export to account [%s]: %v", hdrNote, a.traceLabel(), err)
  3388  				}
  3389  			}
  3390  			if e.ResponseThreshold != 0 {
  3391  				// Response threshold was set in options.
  3392  				if err := a.SetServiceExportResponseThreshold(sub, e.ResponseThreshold); err != nil {
  3393  					s.Debugf("Error adding service export response threshold for [%s]: %v", a.traceLabel(), err)
  3394  				}
  3395  			}
  3396  			if err := a.SetServiceExportAllowTrace(sub, e.AllowTrace); err != nil {
  3397  				s.Debugf("Error adding allow_trace for %q: %v", sub, err)
  3398  			}
  3399  		}
  3400  
  3401  		var revocationChanged *bool
  3402  		var ea *exportAuth
  3403  
  3404  		a.mu.Lock()
  3405  		switch e.Type {
  3406  		case jwt.Stream:
  3407  			revocationChanged = &streamTokenExpirationChanged
  3408  			if se, ok := a.exports.streams[string(e.Subject)]; ok && se != nil {
  3409  				ea = &se.exportAuth
  3410  			}
  3411  		case jwt.Service:
  3412  			revocationChanged = &serviceTokenExpirationChanged
  3413  			if se, ok := a.exports.services[string(e.Subject)]; ok && se != nil {
  3414  				ea = &se.exportAuth
  3415  			}
  3416  		}
  3417  		if ea != nil {
  3418  			oldRevocations := ea.actsRevoked
  3419  			if len(e.Revocations) == 0 {
  3420  				// remove all, no need to evaluate existing imports
  3421  				ea.actsRevoked = nil
  3422  			} else if len(oldRevocations) == 0 {
  3423  				// add all, existing imports need to be re evaluated
  3424  				ea.actsRevoked = e.Revocations
  3425  				*revocationChanged = true
  3426  			} else {
  3427  				ea.actsRevoked = e.Revocations
  3428  				// diff, existing imports need to be conditionally re evaluated, depending on:
  3429  				// if a key was added, or it's timestamp increased
  3430  				for k, t := range e.Revocations {
  3431  					if tOld, ok := oldRevocations[k]; !ok || tOld < t {
  3432  						*revocationChanged = true
  3433  					}
  3434  				}
  3435  			}
  3436  		}
  3437  		a.mu.Unlock()
  3438  	}
  3439  	var incompleteImports []*jwt.Import
  3440  	for _, i := range ac.Imports {
  3441  		// check tmpAccounts with priority
  3442  		var acc *Account
  3443  		var err error
  3444  		if v, ok := s.tmpAccounts.Load(i.Account); ok {
  3445  			acc = v.(*Account)
  3446  		} else {
  3447  			acc, err = s.lookupAccount(i.Account)
  3448  		}
  3449  		if acc == nil || err != nil {
  3450  			s.Errorf("Can't locate account [%s] for import of [%v] %s (err=%v)", i.Account, i.Subject, i.Type, err)
  3451  			incompleteImports = append(incompleteImports, i)
  3452  			continue
  3453  		}
  3454  		from := string(i.Subject)
  3455  		to := i.GetTo()
  3456  		switch i.Type {
  3457  		case jwt.Stream:
  3458  			if i.LocalSubject != _EMPTY_ {
  3459  				// set local subject implies to is empty
  3460  				to = string(i.LocalSubject)
  3461  				s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to)
  3462  				err = a.AddMappedStreamImportWithClaim(acc, from, to, i)
  3463  			} else {
  3464  				s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to)
  3465  				err = a.AddStreamImportWithClaim(acc, from, to, i)
  3466  			}
  3467  			if err != nil {
  3468  				s.Debugf("Error adding stream import to account [%s]: %v", a.traceLabel(), err.Error())
  3469  				incompleteImports = append(incompleteImports, i)
  3470  			}
  3471  		case jwt.Service:
  3472  			if i.LocalSubject != _EMPTY_ {
  3473  				from = string(i.LocalSubject)
  3474  				to = string(i.Subject)
  3475  			}
  3476  			s.Debugf("Adding service import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to)
  3477  			if err := a.AddServiceImportWithClaim(acc, from, to, i); err != nil {
  3478  				s.Debugf("Error adding service import to account [%s]: %v", a.traceLabel(), err.Error())
  3479  				incompleteImports = append(incompleteImports, i)
  3480  			}
  3481  		}
  3482  	}
  3483  	// Now let's apply any needed changes from import/export changes.
  3484  	if !a.checkStreamImportsEqual(old) {
  3485  		awcsti := map[string]struct{}{a.Name: {}}
  3486  		for _, c := range a.getClients() {
  3487  			c.processSubsOnConfigReload(awcsti)
  3488  		}
  3489  	}
  3490  	// Now check if stream exports have changed.
  3491  	if !a.checkStreamExportsEqual(old) || signersChanged || streamTokenExpirationChanged {
  3492  		clients := map[*client]struct{}{}
  3493  		// We need to check all accounts that have an import claim from this account.
  3494  		awcsti := map[string]struct{}{}
  3495  		s.accounts.Range(func(k, v interface{}) bool {
  3496  			acc := v.(*Account)
  3497  			// Move to the next if this account is actually account "a".
  3498  			if acc.Name == a.Name {
  3499  				return true
  3500  			}
  3501  			// TODO: checkStreamImportAuthorized() stack should not be trying
  3502  			// to lock "acc". If we find that to be needed, we will need to
  3503  			// rework this to ensure we don't lock acc.
  3504  			acc.mu.Lock()
  3505  			for _, im := range acc.imports.streams {
  3506  				if im != nil && im.acc.Name == a.Name {
  3507  					// Check for if we are still authorized for an import.
  3508  					im.invalid = !a.checkStreamImportAuthorized(acc, im.from, im.claim)
  3509  					awcsti[acc.Name] = struct{}{}
  3510  					for c := range acc.clients {
  3511  						clients[c] = struct{}{}
  3512  					}
  3513  				}
  3514  			}
  3515  			acc.mu.Unlock()
  3516  			return true
  3517  		})
  3518  		// Now walk clients.
  3519  		for c := range clients {
  3520  			c.processSubsOnConfigReload(awcsti)
  3521  		}
  3522  	}
  3523  	// Now check if service exports have changed.
  3524  	if !a.checkServiceExportsEqual(old) || signersChanged || serviceTokenExpirationChanged {
  3525  		s.accounts.Range(func(k, v interface{}) bool {
  3526  			acc := v.(*Account)
  3527  			// Move to the next if this account is actually account "a".
  3528  			if acc.Name == a.Name {
  3529  				return true
  3530  			}
  3531  			// TODO: checkServiceImportAuthorized() stack should not be trying
  3532  			// to lock "acc". If we find that to be needed, we will need to
  3533  			// rework this to ensure we don't lock acc.
  3534  			acc.mu.Lock()
  3535  			for _, si := range acc.imports.services {
  3536  				if si != nil && si.acc.Name == a.Name {
  3537  					// Check for if we are still authorized for an import.
  3538  					si.invalid = !a.checkServiceImportAuthorized(acc, si.to, si.claim)
  3539  					// Make sure we should still be tracking latency and if we
  3540  					// are allowed to trace.
  3541  					if !si.response {
  3542  						if se := a.getServiceExport(si.to); se != nil {
  3543  							if si.latency != nil {
  3544  								si.latency = se.latency
  3545  							}
  3546  							// Update allow trace.
  3547  							si.atrc = se.atrc
  3548  						}
  3549  					}
  3550  				}
  3551  			}
  3552  			acc.mu.Unlock()
  3553  			return true
  3554  		})
  3555  	}
  3556  
  3557  	// Now make sure we shutdown the old service import subscriptions.
  3558  	var sids [][]byte
  3559  	a.mu.RLock()
  3560  	c := a.ic
  3561  	for _, si := range old.imports.services {
  3562  		if c != nil && si.sid != nil {
  3563  			sids = append(sids, si.sid)
  3564  		}
  3565  	}
  3566  	a.mu.RUnlock()
  3567  	for _, sid := range sids {
  3568  		c.processUnsub(sid)
  3569  	}
  3570  
  3571  	// Now do limits if they are present.
  3572  	a.mu.Lock()
  3573  	a.msubs = int32(ac.Limits.Subs)
  3574  	a.mpay = int32(ac.Limits.Payload)
  3575  	a.mconns = int32(ac.Limits.Conn)
  3576  	a.mleafs = int32(ac.Limits.LeafNodeConn)
  3577  	a.disallowBearer = ac.Limits.DisallowBearer
  3578  	// Check for any revocations
  3579  	if len(ac.Revocations) > 0 {
  3580  		// We will always replace whatever we had with most current, so no
  3581  		// need to look at what we have.
  3582  		a.usersRevoked = make(map[string]int64, len(ac.Revocations))
  3583  		for pk, t := range ac.Revocations {
  3584  			a.usersRevoked[pk] = t
  3585  		}
  3586  	} else {
  3587  		a.usersRevoked = nil
  3588  	}
  3589  	a.defaultPerms = buildPermissionsFromJwt(&ac.DefaultPermissions)
  3590  	a.incomplete = len(incompleteImports) != 0
  3591  	for _, i := range incompleteImports {
  3592  		s.incompleteAccExporterMap.Store(i.Account, struct{}{})
  3593  	}
  3594  	if a.srv == nil {
  3595  		a.srv = s
  3596  	}
  3597  
  3598  	if ac.Limits.IsJSEnabled() {
  3599  		toUnlimited := func(value int64) int64 {
  3600  			if value > 0 {
  3601  				return value
  3602  			}
  3603  			return -1
  3604  		}
  3605  		if ac.Limits.JetStreamLimits.DiskStorage != 0 || ac.Limits.JetStreamLimits.MemoryStorage != 0 {
  3606  			// JetStreamAccountLimits and jwt.JetStreamLimits use same value for unlimited
  3607  			a.jsLimits = map[string]JetStreamAccountLimits{
  3608  				_EMPTY_: {
  3609  					MaxMemory:            ac.Limits.JetStreamLimits.MemoryStorage,
  3610  					MaxStore:             ac.Limits.JetStreamLimits.DiskStorage,
  3611  					MaxStreams:           int(ac.Limits.JetStreamLimits.Streams),
  3612  					MaxConsumers:         int(ac.Limits.JetStreamLimits.Consumer),
  3613  					MemoryMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.MemoryMaxStreamBytes),
  3614  					StoreMaxStreamBytes:  toUnlimited(ac.Limits.JetStreamLimits.DiskMaxStreamBytes),
  3615  					MaxBytesRequired:     ac.Limits.JetStreamLimits.MaxBytesRequired,
  3616  					MaxAckPending:        int(toUnlimited(ac.Limits.JetStreamLimits.MaxAckPending)),
  3617  				},
  3618  			}
  3619  		} else {
  3620  			a.jsLimits = map[string]JetStreamAccountLimits{}
  3621  			for t, l := range ac.Limits.JetStreamTieredLimits {
  3622  				a.jsLimits[t] = JetStreamAccountLimits{
  3623  					MaxMemory:            l.MemoryStorage,
  3624  					MaxStore:             l.DiskStorage,
  3625  					MaxStreams:           int(l.Streams),
  3626  					MaxConsumers:         int(l.Consumer),
  3627  					MemoryMaxStreamBytes: toUnlimited(l.MemoryMaxStreamBytes),
  3628  					StoreMaxStreamBytes:  toUnlimited(l.DiskMaxStreamBytes),
  3629  					MaxBytesRequired:     l.MaxBytesRequired,
  3630  					MaxAckPending:        int(toUnlimited(l.MaxAckPending)),
  3631  				}
  3632  			}
  3633  		}
  3634  	} else if a.jsLimits != nil {
  3635  		// covers failed update followed by disable
  3636  		a.jsLimits = nil
  3637  	}
  3638  
  3639  	a.updated = time.Now()
  3640  	clients := a.getClientsLocked()
  3641  	a.mu.Unlock()
  3642  
  3643  	// Sort if we are over the limit.
  3644  	if a.MaxTotalConnectionsReached() {
  3645  		sort.Slice(clients, func(i, j int) bool {
  3646  			return clients[i].start.After(clients[j].start)
  3647  		})
  3648  	}
  3649  
  3650  	// If JetStream is enabled for this server we will call into configJetStream for the account
  3651  	// regardless of enabled or disabled. It handles both cases.
  3652  	if jsEnabled {
  3653  		if err := s.configJetStream(a); err != nil {
  3654  			s.Errorf("Error configuring jetstream for account [%s]: %v", a.traceLabel(), err.Error())
  3655  			a.mu.Lock()
  3656  			// Absent reload of js server cfg, this is going to be broken until js is disabled
  3657  			a.incomplete = true
  3658  			a.mu.Unlock()
  3659  		}
  3660  	} else if a.jsLimits != nil {
  3661  		// We do not have JS enabled for this server, but the account has it enabled so setup
  3662  		// our imports properly. This allows this server to proxy JS traffic correctly.
  3663  		s.checkJetStreamExports()
  3664  		a.enableAllJetStreamServiceImportsAndMappings()
  3665  	}
  3666  
  3667  	for i, c := range clients {
  3668  		a.mu.RLock()
  3669  		exceeded := a.mconns != jwt.NoLimit && i >= int(a.mconns)
  3670  		a.mu.RUnlock()
  3671  		if exceeded {
  3672  			c.maxAccountConnExceeded()
  3673  			continue
  3674  		}
  3675  		c.mu.Lock()
  3676  		c.applyAccountLimits()
  3677  		theJWT := c.opts.JWT
  3678  		c.mu.Unlock()
  3679  		// Check for being revoked here. We use ac one to avoid the account lock.
  3680  		if (ac.Revocations != nil || ac.Limits.DisallowBearer) && theJWT != _EMPTY_ {
  3681  			if juc, err := jwt.DecodeUserClaims(theJWT); err != nil {
  3682  				c.Debugf("User JWT not valid: %v", err)
  3683  				c.authViolation()
  3684  				continue
  3685  			} else if juc.BearerToken && ac.Limits.DisallowBearer {
  3686  				c.Debugf("Bearer User JWT not allowed")
  3687  				c.authViolation()
  3688  				continue
  3689  			} else if ok := ac.IsClaimRevoked(juc); ok {
  3690  				c.sendErrAndDebug("User Authentication Revoked")
  3691  				c.closeConnection(Revocation)
  3692  				continue
  3693  			}
  3694  		}
  3695  	}
  3696  
  3697  	// Check if the signing keys changed, might have to evict
  3698  	if signersChanged {
  3699  		for _, c := range clients {
  3700  			c.mu.Lock()
  3701  			if c.user == nil {
  3702  				c.mu.Unlock()
  3703  				continue
  3704  			}
  3705  			sk := c.user.SigningKey
  3706  			c.mu.Unlock()
  3707  			if sk == _EMPTY_ {
  3708  				continue
  3709  			}
  3710  			if _, ok := alteredScope[sk]; ok {
  3711  				c.closeConnection(AuthenticationViolation)
  3712  			} else if _, ok := a.hasIssuer(sk); !ok {
  3713  				c.closeConnection(AuthenticationViolation)
  3714  			}
  3715  		}
  3716  	}
  3717  
  3718  	if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok && refreshImportingAccounts {
  3719  		s.incompleteAccExporterMap.Delete(old.Name)
  3720  		s.accounts.Range(func(key, value interface{}) bool {
  3721  			acc := value.(*Account)
  3722  			acc.mu.RLock()
  3723  			incomplete := acc.incomplete
  3724  			name := acc.Name
  3725  			label := acc.traceLabel()
  3726  			// Must use jwt in account or risk failing on fetch
  3727  			// This jwt may not be the same that caused exportingAcc to be in incompleteAccExporterMap
  3728  			claimJWT := acc.claimJWT
  3729  			acc.mu.RUnlock()
  3730  			if incomplete && name != old.Name {
  3731  				if accClaims, _, err := s.verifyAccountClaims(claimJWT); err == nil {
  3732  					// Since claimJWT has not changed, acc can become complete
  3733  					// but it won't alter incomplete for it's dependents accounts.
  3734  					s.updateAccountClaimsWithRefresh(acc, accClaims, false)
  3735  					// old.Name was deleted before ranging over accounts
  3736  					// If it exists again, UpdateAccountClaims set it for failed imports of acc.
  3737  					// So there was one import of acc that imported this account and failed again.
  3738  					// Since this account just got updated, the import itself may be in error. So trace that.
  3739  					if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok {
  3740  						s.incompleteAccExporterMap.Delete(old.Name)
  3741  						s.Errorf("Account %s has issues importing account %s", label, old.Name)
  3742  					}
  3743  				}
  3744  			}
  3745  			return true
  3746  		})
  3747  	}
  3748  }
  3749  
  3750  // Helper to build an internal account structure from a jwt.AccountClaims.
  3751  // Lock MUST NOT be held upon entry.
  3752  func (s *Server) buildInternalAccount(ac *jwt.AccountClaims) *Account {
  3753  	acc := NewAccount(ac.Subject)
  3754  	acc.Issuer = ac.Issuer
  3755  	// Set this here since we are placing in s.tmpAccounts below and may be
  3756  	// referenced by an route RS+, etc.
  3757  	s.setAccountSublist(acc)
  3758  
  3759  	// We don't want to register an account that is in the process of
  3760  	// being built, however, to solve circular import dependencies, we
  3761  	// need to store it here.
  3762  	s.tmpAccounts.Store(ac.Subject, acc)
  3763  	s.UpdateAccountClaims(acc, ac)
  3764  	return acc
  3765  }
  3766  
  3767  // Helper to build Permissions from jwt.Permissions
  3768  // or return nil if none were specified
  3769  func buildPermissionsFromJwt(uc *jwt.Permissions) *Permissions {
  3770  	if uc == nil {
  3771  		return nil
  3772  	}
  3773  	var p *Permissions
  3774  	if len(uc.Pub.Allow) > 0 || len(uc.Pub.Deny) > 0 {
  3775  		p = &Permissions{}
  3776  		p.Publish = &SubjectPermission{}
  3777  		p.Publish.Allow = uc.Pub.Allow
  3778  		p.Publish.Deny = uc.Pub.Deny
  3779  	}
  3780  	if len(uc.Sub.Allow) > 0 || len(uc.Sub.Deny) > 0 {
  3781  		if p == nil {
  3782  			p = &Permissions{}
  3783  		}
  3784  		p.Subscribe = &SubjectPermission{}
  3785  		p.Subscribe.Allow = uc.Sub.Allow
  3786  		p.Subscribe.Deny = uc.Sub.Deny
  3787  	}
  3788  	if uc.Resp != nil {
  3789  		if p == nil {
  3790  			p = &Permissions{}
  3791  		}
  3792  		p.Response = &ResponsePermission{
  3793  			MaxMsgs: uc.Resp.MaxMsgs,
  3794  			Expires: uc.Resp.Expires,
  3795  		}
  3796  		validateResponsePermissions(p)
  3797  	}
  3798  	return p
  3799  }
  3800  
  3801  // Helper to build internal NKeyUser.
  3802  func buildInternalNkeyUser(uc *jwt.UserClaims, acts map[string]struct{}, acc *Account) *NkeyUser {
  3803  	nu := &NkeyUser{Nkey: uc.Subject, Account: acc, AllowedConnectionTypes: acts}
  3804  	if uc.IssuerAccount != _EMPTY_ {
  3805  		nu.SigningKey = uc.Issuer
  3806  	}
  3807  
  3808  	// Now check for permissions.
  3809  	var p = buildPermissionsFromJwt(&uc.Permissions)
  3810  	if p == nil && acc.defaultPerms != nil {
  3811  		p = acc.defaultPerms.clone()
  3812  	}
  3813  	nu.Permissions = p
  3814  	return nu
  3815  }
  3816  
  3817  func fetchAccount(res AccountResolver, name string) (string, error) {
  3818  	if !nkeys.IsValidPublicAccountKey(name) {
  3819  		return _EMPTY_, fmt.Errorf("will only fetch valid account keys")
  3820  	}
  3821  	return res.Fetch(name)
  3822  }
  3823  
  3824  // AccountResolver interface. This is to fetch Account JWTs by public nkeys
  3825  type AccountResolver interface {
  3826  	Fetch(name string) (string, error)
  3827  	Store(name, jwt string) error
  3828  	IsReadOnly() bool
  3829  	Start(server *Server) error
  3830  	IsTrackingUpdate() bool
  3831  	Reload() error
  3832  	Close()
  3833  }
  3834  
  3835  // Default implementations of IsReadOnly/Start so only need to be written when changed
  3836  type resolverDefaultsOpsImpl struct{}
  3837  
  3838  func (*resolverDefaultsOpsImpl) IsReadOnly() bool {
  3839  	return true
  3840  }
  3841  
  3842  func (*resolverDefaultsOpsImpl) IsTrackingUpdate() bool {
  3843  	return false
  3844  }
  3845  
  3846  func (*resolverDefaultsOpsImpl) Start(*Server) error {
  3847  	return nil
  3848  }
  3849  
  3850  func (*resolverDefaultsOpsImpl) Reload() error {
  3851  	return nil
  3852  }
  3853  
  3854  func (*resolverDefaultsOpsImpl) Close() {
  3855  }
  3856  
  3857  func (*resolverDefaultsOpsImpl) Store(_, _ string) error {
  3858  	return fmt.Errorf("store operation not supported for URL Resolver")
  3859  }
  3860  
  3861  // MemAccResolver is a memory only resolver.
  3862  // Mostly for testing.
  3863  type MemAccResolver struct {
  3864  	sm sync.Map
  3865  	resolverDefaultsOpsImpl
  3866  }
  3867  
  3868  // Fetch will fetch the account jwt claims from the internal sync.Map.
  3869  func (m *MemAccResolver) Fetch(name string) (string, error) {
  3870  	if j, ok := m.sm.Load(name); ok {
  3871  		return j.(string), nil
  3872  	}
  3873  	return _EMPTY_, ErrMissingAccount
  3874  }
  3875  
  3876  // Store will store the account jwt claims in the internal sync.Map.
  3877  func (m *MemAccResolver) Store(name, jwt string) error {
  3878  	m.sm.Store(name, jwt)
  3879  	return nil
  3880  }
  3881  
  3882  func (m *MemAccResolver) IsReadOnly() bool {
  3883  	return false
  3884  }
  3885  
  3886  // URLAccResolver implements an http fetcher.
  3887  type URLAccResolver struct {
  3888  	url string
  3889  	c   *http.Client
  3890  	resolverDefaultsOpsImpl
  3891  }
  3892  
  3893  // NewURLAccResolver returns a new resolver for the given base URL.
  3894  func NewURLAccResolver(url string) (*URLAccResolver, error) {
  3895  	if !strings.HasSuffix(url, "/") {
  3896  		url += "/"
  3897  	}
  3898  	// FIXME(dlc) - Make timeout and others configurable.
  3899  	// We create our own transport to amortize TLS.
  3900  	tr := &http.Transport{
  3901  		MaxIdleConns:    10,
  3902  		IdleConnTimeout: 30 * time.Second,
  3903  	}
  3904  	ur := &URLAccResolver{
  3905  		url: url,
  3906  		c:   &http.Client{Timeout: DEFAULT_ACCOUNT_FETCH_TIMEOUT, Transport: tr},
  3907  	}
  3908  	return ur, nil
  3909  }
  3910  
  3911  // Fetch will fetch the account jwt claims from the base url, appending the
  3912  // account name onto the end.
  3913  func (ur *URLAccResolver) Fetch(name string) (string, error) {
  3914  	url := ur.url + name
  3915  	resp, err := ur.c.Get(url)
  3916  	if err != nil {
  3917  		return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), err)
  3918  	} else if resp == nil {
  3919  		return _EMPTY_, fmt.Errorf("could not fetch <%q>: no response", redactURLString(url))
  3920  	}
  3921  	defer resp.Body.Close()
  3922  	if resp.StatusCode != http.StatusOK {
  3923  		return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), resp.Status)
  3924  	}
  3925  	body, err := io.ReadAll(resp.Body)
  3926  	if err != nil {
  3927  		return _EMPTY_, err
  3928  	}
  3929  	return string(body), nil
  3930  }
  3931  
  3932  // Resolver based on nats for synchronization and backing directory for storage.
  3933  type DirAccResolver struct {
  3934  	*DirJWTStore
  3935  	*Server
  3936  	syncInterval time.Duration
  3937  	fetchTimeout time.Duration
  3938  }
  3939  
  3940  func (dr *DirAccResolver) IsTrackingUpdate() bool {
  3941  	return true
  3942  }
  3943  
  3944  func (dr *DirAccResolver) Reload() error {
  3945  	return dr.DirJWTStore.Reload()
  3946  }
  3947  
  3948  // ServerAPIClaimUpdateResponse is the response to $SYS.REQ.ACCOUNT.<id>.CLAIMS.UPDATE and $SYS.REQ.CLAIMS.UPDATE
  3949  type ServerAPIClaimUpdateResponse struct {
  3950  	Server *ServerInfo        `json:"server"`
  3951  	Data   *ClaimUpdateStatus `json:"data,omitempty"`
  3952  	Error  *ClaimUpdateError  `json:"error,omitempty"`
  3953  }
  3954  
  3955  type ClaimUpdateError struct {
  3956  	Account     string `json:"account,omitempty"`
  3957  	Code        int    `json:"code"`
  3958  	Description string `json:"description,omitempty"`
  3959  }
  3960  
  3961  type ClaimUpdateStatus struct {
  3962  	Account string `json:"account,omitempty"`
  3963  	Code    int    `json:"code,omitempty"`
  3964  	Message string `json:"message,omitempty"`
  3965  }
  3966  
  3967  func respondToUpdate(s *Server, respSubj string, acc string, message string, err error) {
  3968  	if err == nil {
  3969  		if acc == _EMPTY_ {
  3970  			s.Debugf("%s", message)
  3971  		} else {
  3972  			s.Debugf("%s - %s", message, acc)
  3973  		}
  3974  	} else {
  3975  		if acc == _EMPTY_ {
  3976  			s.Errorf("%s - %s", message, err)
  3977  		} else {
  3978  			s.Errorf("%s - %s - %s", message, acc, err)
  3979  		}
  3980  	}
  3981  	if respSubj == _EMPTY_ {
  3982  		return
  3983  	}
  3984  
  3985  	response := ServerAPIClaimUpdateResponse{
  3986  		Server: &ServerInfo{},
  3987  	}
  3988  
  3989  	if err == nil {
  3990  		response.Data = &ClaimUpdateStatus{
  3991  			Account: acc,
  3992  			Code:    http.StatusOK,
  3993  			Message: message,
  3994  		}
  3995  	} else {
  3996  		response.Error = &ClaimUpdateError{
  3997  			Account:     acc,
  3998  			Code:        http.StatusInternalServerError,
  3999  			Description: fmt.Sprintf("%s - %v", message, err),
  4000  		}
  4001  	}
  4002  
  4003  	s.sendInternalMsgLocked(respSubj, _EMPTY_, response.Server, response)
  4004  }
  4005  
  4006  func handleListRequest(store *DirJWTStore, s *Server, reply string) {
  4007  	if reply == _EMPTY_ {
  4008  		return
  4009  	}
  4010  	accIds := make([]string, 0, 1024)
  4011  	if err := store.PackWalk(1, func(partialPackMsg string) {
  4012  		if tk := strings.Split(partialPackMsg, "|"); len(tk) == 2 {
  4013  			accIds = append(accIds, tk[0])
  4014  		}
  4015  	}); err != nil {
  4016  		// let them timeout
  4017  		s.Errorf("list request error: %v", err)
  4018  	} else {
  4019  		s.Debugf("list request responded with %d account ids", len(accIds))
  4020  		server := &ServerInfo{}
  4021  		response := map[string]interface{}{"server": server, "data": accIds}
  4022  		s.sendInternalMsgLocked(reply, _EMPTY_, server, response)
  4023  	}
  4024  }
  4025  
  4026  func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) {
  4027  	var accIds []interface{}
  4028  	var subj, sysAccName string
  4029  	if sysAcc := s.SystemAccount(); sysAcc != nil {
  4030  		sysAccName = sysAcc.GetName()
  4031  	}
  4032  	// Only operator and operator signing key are allowed to delete
  4033  	gk, err := jwt.DecodeGeneric(string(msg))
  4034  	if err == nil {
  4035  		subj = gk.Subject
  4036  		if store.deleteType == NoDelete {
  4037  			err = fmt.Errorf("delete must be enabled in server config")
  4038  		} else if subj != gk.Issuer {
  4039  			err = fmt.Errorf("not self signed")
  4040  		} else if _, ok := store.operator[gk.Issuer]; !ok {
  4041  			err = fmt.Errorf("not trusted")
  4042  		} else if list, ok := gk.Data["accounts"]; !ok {
  4043  			err = fmt.Errorf("malformed request")
  4044  		} else if accIds, ok = list.([]interface{}); !ok {
  4045  			err = fmt.Errorf("malformed request")
  4046  		} else {
  4047  			for _, entry := range accIds {
  4048  				if acc, ok := entry.(string); !ok ||
  4049  					acc == _EMPTY_ || !nkeys.IsValidPublicAccountKey(acc) {
  4050  					err = fmt.Errorf("malformed request")
  4051  					break
  4052  				} else if acc == sysAccName {
  4053  					err = fmt.Errorf("not allowed to delete system account")
  4054  					break
  4055  				}
  4056  			}
  4057  		}
  4058  	}
  4059  	if err != nil {
  4060  		respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("delete accounts request by %s failed", subj), err)
  4061  		return
  4062  	}
  4063  	errs := []string{}
  4064  	passCnt := 0
  4065  	for _, acc := range accIds {
  4066  		if err := store.delete(acc.(string)); err != nil {
  4067  			errs = append(errs, err.Error())
  4068  		} else {
  4069  			passCnt++
  4070  		}
  4071  	}
  4072  	if len(errs) == 0 {
  4073  		respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts", passCnt), nil)
  4074  	} else {
  4075  		respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts, failed for %d", passCnt, len(errs)),
  4076  			errors.New(strings.Join(errs, "\n")))
  4077  	}
  4078  }
  4079  
  4080  func getOperatorKeys(s *Server) (string, map[string]struct{}, bool, error) {
  4081  	var op string
  4082  	var strict bool
  4083  	keys := make(map[string]struct{})
  4084  	if opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 {
  4085  		op = opts.TrustedOperators[0].Subject
  4086  		strict = opts.TrustedOperators[0].StrictSigningKeyUsage
  4087  		if !strict {
  4088  			keys[opts.TrustedOperators[0].Subject] = struct{}{}
  4089  		}
  4090  		for _, key := range opts.TrustedOperators[0].SigningKeys {
  4091  			keys[key] = struct{}{}
  4092  		}
  4093  	}
  4094  	if len(keys) == 0 {
  4095  		return _EMPTY_, nil, false, fmt.Errorf("no operator key found")
  4096  	}
  4097  	return op, keys, strict, nil
  4098  }
  4099  
  4100  func claimValidate(claim *jwt.AccountClaims) error {
  4101  	vr := &jwt.ValidationResults{}
  4102  	claim.Validate(vr)
  4103  	if vr.IsBlocking(false) {
  4104  		return fmt.Errorf("validation errors: %v", vr.Errors())
  4105  	}
  4106  	return nil
  4107  }
  4108  
  4109  func removeCb(s *Server, pubKey string) {
  4110  	v, ok := s.accounts.Load(pubKey)
  4111  	if !ok {
  4112  		return
  4113  	}
  4114  	a := v.(*Account)
  4115  	s.Debugf("Disable account %s due to remove", pubKey)
  4116  	a.mu.Lock()
  4117  	// lock out new clients
  4118  	a.msubs = 0
  4119  	a.mpay = 0
  4120  	a.mconns = 0
  4121  	a.mleafs = 0
  4122  	a.updated = time.Now()
  4123  	jsa := a.js
  4124  	a.mu.Unlock()
  4125  	// set the account to be expired and disconnect clients
  4126  	a.expiredTimeout()
  4127  	// For JS, we need also to disable it.
  4128  	if js := s.getJetStream(); js != nil && jsa != nil {
  4129  		js.disableJetStream(jsa)
  4130  		// Remove JetStream state in memory, this will be reset
  4131  		// on the changed callback from the account in case it is
  4132  		// enabled again.
  4133  		a.js = nil
  4134  	}
  4135  	// We also need to remove all ServerImport subscriptions
  4136  	a.removeAllServiceImportSubs()
  4137  	a.mu.Lock()
  4138  	a.clearExpirationTimer()
  4139  	a.mu.Unlock()
  4140  }
  4141  
  4142  func (dr *DirAccResolver) Start(s *Server) error {
  4143  	op, opKeys, strict, err := getOperatorKeys(s)
  4144  	if err != nil {
  4145  		return err
  4146  	}
  4147  	dr.Lock()
  4148  	defer dr.Unlock()
  4149  	dr.Server = s
  4150  	dr.operator = opKeys
  4151  	dr.DirJWTStore.changed = func(pubKey string) {
  4152  		if v, ok := s.accounts.Load(pubKey); ok {
  4153  			if theJwt, err := dr.LoadAcc(pubKey); err != nil {
  4154  				s.Errorf("DirResolver - Update got error on load: %v", err)
  4155  			} else {
  4156  				acc := v.(*Account)
  4157  				if err = s.updateAccountWithClaimJWT(acc, theJwt); err != nil {
  4158  					s.Errorf("DirResolver - Update for account %q resulted in error %v", pubKey, err)
  4159  				} else {
  4160  					if _, jsa, err := acc.checkForJetStream(); err != nil {
  4161  						if !IsNatsErr(err, JSNotEnabledForAccountErr) {
  4162  							s.Warnf("DirResolver - Error checking for JetStream support for account %q: %v", pubKey, err)
  4163  						}
  4164  					} else if jsa == nil {
  4165  						if err = s.configJetStream(acc); err != nil {
  4166  							s.Errorf("DirResolver - Error configuring JetStream for account %q: %v", pubKey, err)
  4167  						}
  4168  					}
  4169  				}
  4170  			}
  4171  		}
  4172  	}
  4173  	dr.DirJWTStore.deleted = func(pubKey string) {
  4174  		removeCb(s, pubKey)
  4175  	}
  4176  	packRespIb := s.newRespInbox()
  4177  	for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {
  4178  		// subscribe to account jwt update requests
  4179  		if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) {
  4180  			var pubKey string
  4181  			tk := strings.Split(subj, tsep)
  4182  			if len(tk) == accUpdateTokensNew {
  4183  				pubKey = tk[accReqAccIndex]
  4184  			} else if len(tk) == accUpdateTokensOld {
  4185  				pubKey = tk[accUpdateAccIdxOld]
  4186  			} else {
  4187  				s.Debugf("DirResolver - jwt update skipped due to bad subject %q", subj)
  4188  				return
  4189  			}
  4190  			if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4191  				respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err)
  4192  			} else if err := claimValidate(claim); err != nil {
  4193  				respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err)
  4194  			} else if claim.Subject != pubKey {
  4195  				err := errors.New("subject does not match jwt content")
  4196  				respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err)
  4197  			} else if claim.Issuer == op && strict {
  4198  				err := errors.New("operator requires issuer to be a signing key")
  4199  				respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err)
  4200  			} else if err := dr.save(pubKey, string(msg)); err != nil {
  4201  				respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err)
  4202  			} else {
  4203  				respondToUpdate(s, resp, pubKey, "jwt updated", nil)
  4204  			}
  4205  		}); err != nil {
  4206  			return fmt.Errorf("error setting up update handling: %v", err)
  4207  		}
  4208  	}
  4209  	if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) {
  4210  		// As this is a raw message, we need to extract payload and only decode claims from it,
  4211  		// in case request is sent with headers.
  4212  		_, msg = c.msgParts(msg)
  4213  		if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4214  			respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err)
  4215  		} else if claim.Issuer == op && strict {
  4216  			err := errors.New("operator requires issuer to be a signing key")
  4217  			respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err)
  4218  		} else if err := claimValidate(claim); err != nil {
  4219  			respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err)
  4220  		} else if err := dr.save(claim.Subject, string(msg)); err != nil {
  4221  			respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err)
  4222  		} else {
  4223  			respondToUpdate(s, resp, claim.Subject, "jwt updated", nil)
  4224  		}
  4225  	}); err != nil {
  4226  		return fmt.Errorf("error setting up update handling: %v", err)
  4227  	}
  4228  	// respond to lookups with our version
  4229  	if _, err := s.sysSubscribe(fmt.Sprintf(accLookupReqSubj, "*"), func(_ *subscription, _ *client, _ *Account, subj, reply string, msg []byte) {
  4230  		if reply == _EMPTY_ {
  4231  			return
  4232  		}
  4233  		tk := strings.Split(subj, tsep)
  4234  		if len(tk) != accLookupReqTokens {
  4235  			return
  4236  		}
  4237  		accName := tk[accReqAccIndex]
  4238  		if theJWT, err := dr.DirJWTStore.LoadAcc(accName); err != nil {
  4239  			if errors.Is(err, fs.ErrNotExist) {
  4240  				s.Debugf("DirResolver - Could not find account %q", accName)
  4241  				// Reply with empty response to signal absence of JWT to others.
  4242  				s.sendInternalMsgLocked(reply, _EMPTY_, nil, nil)
  4243  			} else {
  4244  				s.Errorf("DirResolver - Error looking up account %q: %v", accName, err)
  4245  			}
  4246  		} else {
  4247  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(theJWT))
  4248  		}
  4249  	}); err != nil {
  4250  		return fmt.Errorf("error setting up lookup request handling: %v", err)
  4251  	}
  4252  	// respond to pack requests with one or more pack messages
  4253  	// an empty message signifies the end of the response responder.
  4254  	if _, err := s.sysSubscribeQ(accPackReqSubj, "responder", func(_ *subscription, _ *client, _ *Account, _, reply string, theirHash []byte) {
  4255  		if reply == _EMPTY_ {
  4256  			return
  4257  		}
  4258  		ourHash := dr.DirJWTStore.Hash()
  4259  		if bytes.Equal(theirHash, ourHash[:]) {
  4260  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{})
  4261  			s.Debugf("DirResolver - Pack request matches hash %x", ourHash[:])
  4262  		} else if err := dr.DirJWTStore.PackWalk(1, func(partialPackMsg string) {
  4263  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(partialPackMsg))
  4264  		}); err != nil {
  4265  			// let them timeout
  4266  			s.Errorf("DirResolver - Pack request error: %v", err)
  4267  		} else {
  4268  			s.Debugf("DirResolver - Pack request hash %x - finished responding with hash %x", theirHash, ourHash)
  4269  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{})
  4270  		}
  4271  	}); err != nil {
  4272  		return fmt.Errorf("error setting up pack request handling: %v", err)
  4273  	}
  4274  	// respond to list requests with one message containing all account ids
  4275  	if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) {
  4276  		handleListRequest(dr.DirJWTStore, s, reply)
  4277  	}); err != nil {
  4278  		return fmt.Errorf("error setting up list request handling: %v", err)
  4279  	}
  4280  	if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) {
  4281  		handleDeleteRequest(dr.DirJWTStore, s, msg, reply)
  4282  	}); err != nil {
  4283  		return fmt.Errorf("error setting up delete request handling: %v", err)
  4284  	}
  4285  	// embed pack responses into store
  4286  	if _, err := s.sysSubscribe(packRespIb, func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) {
  4287  		hash := dr.DirJWTStore.Hash()
  4288  		if len(msg) == 0 { // end of response stream
  4289  			s.Debugf("DirResolver - Merging finished and resulting in: %x", dr.DirJWTStore.Hash())
  4290  			return
  4291  		} else if err := dr.DirJWTStore.Merge(string(msg)); err != nil {
  4292  			s.Errorf("DirResolver - Merging resulted in error: %v", err)
  4293  		} else {
  4294  			s.Debugf("DirResolver - Merging succeeded and changed %x to %x", hash, dr.DirJWTStore.Hash())
  4295  		}
  4296  	}); err != nil {
  4297  		return fmt.Errorf("error setting up pack response handling: %v", err)
  4298  	}
  4299  	// periodically send out pack message
  4300  	quit := s.quitCh
  4301  	s.startGoRoutine(func() {
  4302  		defer s.grWG.Done()
  4303  		ticker := time.NewTicker(dr.syncInterval)
  4304  		for {
  4305  			select {
  4306  			case <-quit:
  4307  				ticker.Stop()
  4308  				return
  4309  			case <-ticker.C:
  4310  			}
  4311  			ourHash := dr.DirJWTStore.Hash()
  4312  			s.Debugf("DirResolver - Checking store state: %x", ourHash)
  4313  			s.sendInternalMsgLocked(accPackReqSubj, packRespIb, nil, ourHash[:])
  4314  		}
  4315  	})
  4316  	s.Noticef("Managing all jwt in exclusive directory %s", dr.directory)
  4317  	return nil
  4318  }
  4319  
  4320  func (dr *DirAccResolver) Fetch(name string) (string, error) {
  4321  	if theJWT, err := dr.LoadAcc(name); theJWT != _EMPTY_ {
  4322  		return theJWT, nil
  4323  	} else {
  4324  		dr.Lock()
  4325  		srv := dr.Server
  4326  		to := dr.fetchTimeout
  4327  		dr.Unlock()
  4328  		if srv == nil {
  4329  			return _EMPTY_, err
  4330  		}
  4331  		return srv.fetch(dr, name, to) // lookup from other server
  4332  	}
  4333  }
  4334  
  4335  func (dr *DirAccResolver) Store(name, jwt string) error {
  4336  	return dr.saveIfNewer(name, jwt)
  4337  }
  4338  
  4339  type DirResOption func(s *DirAccResolver) error
  4340  
  4341  // limits the amount of time spent waiting for an account fetch to complete
  4342  func FetchTimeout(to time.Duration) DirResOption {
  4343  	return func(r *DirAccResolver) error {
  4344  		if to <= time.Duration(0) {
  4345  			return fmt.Errorf("Fetch timeout %v is too smal", to)
  4346  		}
  4347  		r.fetchTimeout = to
  4348  		return nil
  4349  	}
  4350  }
  4351  
  4352  func (dr *DirAccResolver) apply(opts ...DirResOption) error {
  4353  	for _, o := range opts {
  4354  		if err := o(dr); err != nil {
  4355  			return err
  4356  		}
  4357  	}
  4358  	return nil
  4359  }
  4360  
  4361  func NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete deleteType, opts ...DirResOption) (*DirAccResolver, error) {
  4362  	if limit == 0 {
  4363  		limit = math.MaxInt64
  4364  	}
  4365  	if syncInterval <= 0 {
  4366  		syncInterval = time.Minute
  4367  	}
  4368  	store, err := NewExpiringDirJWTStore(path, false, true, delete, 0, limit, false, 0, nil)
  4369  	if err != nil {
  4370  		return nil, err
  4371  	}
  4372  
  4373  	res := &DirAccResolver{store, nil, syncInterval, DEFAULT_ACCOUNT_FETCH_TIMEOUT}
  4374  	if err := res.apply(opts...); err != nil {
  4375  		return nil, err
  4376  	}
  4377  	return res, nil
  4378  }
  4379  
  4380  // Caching resolver using nats for lookups and making use of a directory for storage
  4381  type CacheDirAccResolver struct {
  4382  	DirAccResolver
  4383  	ttl time.Duration
  4384  }
  4385  
  4386  func (s *Server) fetch(res AccountResolver, name string, timeout time.Duration) (string, error) {
  4387  	if s == nil {
  4388  		return _EMPTY_, ErrNoAccountResolver
  4389  	}
  4390  	respC := make(chan []byte, 1)
  4391  	accountLookupRequest := fmt.Sprintf(accLookupReqSubj, name)
  4392  	s.mu.Lock()
  4393  	if s.sys == nil || s.sys.replies == nil {
  4394  		s.mu.Unlock()
  4395  		return _EMPTY_, fmt.Errorf("eventing shut down")
  4396  	}
  4397  	// Resolver will wait for detected active servers to reply
  4398  	// before serving an error in case there weren't any found.
  4399  	expectedServers := len(s.sys.servers)
  4400  	replySubj := s.newRespInbox()
  4401  	replies := s.sys.replies
  4402  
  4403  	// Store our handler.
  4404  	replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) {
  4405  		var clone []byte
  4406  		isEmpty := len(msg) == 0
  4407  		if !isEmpty {
  4408  			clone = make([]byte, len(msg))
  4409  			copy(clone, msg)
  4410  		}
  4411  		s.mu.Lock()
  4412  		defer s.mu.Unlock()
  4413  		expectedServers--
  4414  		// Skip empty responses until getting all the available servers.
  4415  		if isEmpty && expectedServers > 0 {
  4416  			return
  4417  		}
  4418  		// Use the first valid response if there is still interest or
  4419  		// one of the empty responses to signal that it was not found.
  4420  		if _, ok := replies[replySubj]; ok {
  4421  			select {
  4422  			case respC <- clone:
  4423  			default:
  4424  			}
  4425  		}
  4426  	}
  4427  	s.sendInternalMsg(accountLookupRequest, replySubj, nil, []byte{})
  4428  	quit := s.quitCh
  4429  	s.mu.Unlock()
  4430  	var err error
  4431  	var theJWT string
  4432  	select {
  4433  	case <-quit:
  4434  		err = errors.New("fetching jwt failed due to shutdown")
  4435  	case <-time.After(timeout):
  4436  		err = errors.New("fetching jwt timed out")
  4437  	case m := <-respC:
  4438  		if len(m) == 0 {
  4439  			err = errors.New("account jwt not found")
  4440  		} else if err = res.Store(name, string(m)); err == nil {
  4441  			theJWT = string(m)
  4442  		}
  4443  	}
  4444  	s.mu.Lock()
  4445  	delete(replies, replySubj)
  4446  	s.mu.Unlock()
  4447  	close(respC)
  4448  	return theJWT, err
  4449  }
  4450  
  4451  func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, opts ...DirResOption) (*CacheDirAccResolver, error) {
  4452  	if limit <= 0 {
  4453  		limit = 1_000
  4454  	}
  4455  	store, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil)
  4456  	if err != nil {
  4457  		return nil, err
  4458  	}
  4459  	res := &CacheDirAccResolver{DirAccResolver{store, nil, 0, DEFAULT_ACCOUNT_FETCH_TIMEOUT}, ttl}
  4460  	if err := res.apply(opts...); err != nil {
  4461  		return nil, err
  4462  	}
  4463  	return res, nil
  4464  }
  4465  
  4466  func (dr *CacheDirAccResolver) Start(s *Server) error {
  4467  	op, opKeys, strict, err := getOperatorKeys(s)
  4468  	if err != nil {
  4469  		return err
  4470  	}
  4471  	dr.Lock()
  4472  	defer dr.Unlock()
  4473  	dr.Server = s
  4474  	dr.operator = opKeys
  4475  	dr.DirJWTStore.changed = func(pubKey string) {
  4476  		if v, ok := s.accounts.Load(pubKey); !ok {
  4477  		} else if theJwt, err := dr.LoadAcc(pubKey); err != nil {
  4478  			s.Errorf("DirResolver - Update got error on load: %v", err)
  4479  		} else if err := s.updateAccountWithClaimJWT(v.(*Account), theJwt); err != nil {
  4480  			s.Errorf("DirResolver - Update resulted in error %v", err)
  4481  		}
  4482  	}
  4483  	dr.DirJWTStore.deleted = func(pubKey string) {
  4484  		removeCb(s, pubKey)
  4485  	}
  4486  	for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {
  4487  		// subscribe to account jwt update requests
  4488  		if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) {
  4489  			var pubKey string
  4490  			tk := strings.Split(subj, tsep)
  4491  			if len(tk) == accUpdateTokensNew {
  4492  				pubKey = tk[accReqAccIndex]
  4493  			} else if len(tk) == accUpdateTokensOld {
  4494  				pubKey = tk[accUpdateAccIdxOld]
  4495  			} else {
  4496  				s.Debugf("DirResolver - jwt update cache skipped due to bad subject %q", subj)
  4497  				return
  4498  			}
  4499  			if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4500  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4501  			} else if claim.Subject != pubKey {
  4502  				err := errors.New("subject does not match jwt content")
  4503  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4504  			} else if claim.Issuer == op && strict {
  4505  				err := errors.New("operator requires issuer to be a signing key")
  4506  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4507  			} else if _, ok := s.accounts.Load(pubKey); !ok {
  4508  				respondToUpdate(s, resp, pubKey, "jwt update cache skipped", nil)
  4509  			} else if err := claimValidate(claim); err != nil {
  4510  				respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err)
  4511  			} else if err := dr.save(pubKey, string(msg)); err != nil {
  4512  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4513  			} else {
  4514  				respondToUpdate(s, resp, pubKey, "jwt updated cache", nil)
  4515  			}
  4516  		}); err != nil {
  4517  			return fmt.Errorf("error setting up update handling: %v", err)
  4518  		}
  4519  	}
  4520  	if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) {
  4521  		// As this is a raw message, we need to extract payload and only decode claims from it,
  4522  		// in case request is sent with headers.
  4523  		_, msg = c.msgParts(msg)
  4524  		if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4525  			respondToUpdate(s, resp, "n/a", "jwt update cache resulted in error", err)
  4526  		} else if claim.Issuer == op && strict {
  4527  			err := errors.New("operator requires issuer to be a signing key")
  4528  			respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err)
  4529  		} else if _, ok := s.accounts.Load(claim.Subject); !ok {
  4530  			respondToUpdate(s, resp, claim.Subject, "jwt update cache skipped", nil)
  4531  		} else if err := claimValidate(claim); err != nil {
  4532  			respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err)
  4533  		} else if err := dr.save(claim.Subject, string(msg)); err != nil {
  4534  			respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err)
  4535  		} else {
  4536  			respondToUpdate(s, resp, claim.Subject, "jwt updated cache", nil)
  4537  		}
  4538  	}); err != nil {
  4539  		return fmt.Errorf("error setting up update handling: %v", err)
  4540  	}
  4541  	// respond to list requests with one message containing all account ids
  4542  	if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) {
  4543  		handleListRequest(dr.DirJWTStore, s, reply)
  4544  	}); err != nil {
  4545  		return fmt.Errorf("error setting up list request handling: %v", err)
  4546  	}
  4547  	if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) {
  4548  		handleDeleteRequest(dr.DirJWTStore, s, msg, reply)
  4549  	}); err != nil {
  4550  		return fmt.Errorf("error setting up list request handling: %v", err)
  4551  	}
  4552  	s.Noticef("Managing some jwt in exclusive directory %s", dr.directory)
  4553  	return nil
  4554  }
  4555  
  4556  func (dr *CacheDirAccResolver) Reload() error {
  4557  	return dr.DirAccResolver.Reload()
  4558  }