github.com/nats-io/nats-server/v2@v2.11.0-preview.2/server/accounts.go (about)

     1  // Copyright 2018-2024 The NATS Authors
     2  // Licensed under the Apache License, Version 2.0 (the "License");
     3  // you may not use this file except in compliance with the License.
     4  // You may obtain a copy of the License at
     5  //
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package server
    15  
    16  import (
    17  	"bytes"
    18  	"encoding/hex"
    19  	"errors"
    20  	"fmt"
    21  	"io"
    22  	"io/fs"
    23  	"math"
    24  	"math/rand"
    25  	"net/http"
    26  	"net/textproto"
    27  	"reflect"
    28  	"sort"
    29  	"strconv"
    30  	"strings"
    31  	"sync"
    32  	"sync/atomic"
    33  	"time"
    34  
    35  	"github.com/nats-io/jwt/v2"
    36  	"github.com/nats-io/nats-server/v2/internal/fastrand"
    37  	"github.com/nats-io/nkeys"
    38  	"github.com/nats-io/nuid"
    39  )
    40  
    41  // For backwards compatibility with NATS < 2.0, users who are not explicitly defined into an
    42  // account will be grouped in the default global account.
    43  const globalAccountName = DEFAULT_GLOBAL_ACCOUNT
    44  
    45  const defaultMaxSubLimitReportThreshold = int64(2 * time.Second)
    46  
    47  var maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold
    48  
    49  // Account are subject namespace definitions. By default no messages are shared between accounts.
    50  // You can share via Exports and Imports of Streams and Services.
    51  type Account struct {
    52  	stats
    53  	gwReplyMapping
    54  	Name         string
    55  	LogicalName  string
    56  	Nkey         string
    57  	Issuer       string
    58  	claimJWT     string
    59  	updated      time.Time
    60  	mu           sync.RWMutex
    61  	sqmu         sync.Mutex
    62  	sl           *Sublist
    63  	ic           *client
    64  	isid         uint64
    65  	etmr         *time.Timer
    66  	ctmr         *time.Timer
    67  	strack       map[string]sconns
    68  	nrclients    int32
    69  	sysclients   int32
    70  	nleafs       int32
    71  	nrleafs      int32
    72  	clients      map[*client]struct{}
    73  	rm           map[string]int32
    74  	lqws         map[string]int32
    75  	usersRevoked map[string]int64
    76  	mappings     []*mapping
    77  	hasMapped    atomic.Bool
    78  	lmu          sync.RWMutex
    79  	lleafs       []*client
    80  	leafClusters map[string]uint64
    81  	imports      importMap
    82  	exports      exportMap
    83  	js           *jsAccount
    84  	jsLimits     map[string]JetStreamAccountLimits
    85  	limits
    86  	expired      bool
    87  	incomplete   bool
    88  	signingKeys  map[string]jwt.Scope
    89  	extAuth      *jwt.ExternalAuthorization
    90  	srv          *Server // server this account is registered with (possibly nil)
    91  	lds          string  // loop detection subject for leaf nodes
    92  	siReply      []byte  // service reply prefix, will form wildcard subscription.
    93  	eventIds     *nuid.NUID
    94  	eventIdsMu   sync.Mutex
    95  	defaultPerms *Permissions
    96  	tags         jwt.TagList
    97  	nameTag      string
    98  	lastLimErr   int64
    99  	routePoolIdx int
   100  	// If the trace destination is specified and a message with a traceParentHdr
   101  	// is received, and has the least significant bit of the last token set to 1,
   102  	// then if traceDestSampling is > 0 and < 100, a random value will be selected
   103  	// and if it falls between 0 and that value, message tracing will be triggered.
   104  	traceDest         string
   105  	traceDestSampling int
   106  }
   107  
   108  const (
   109  	accDedicatedRoute                = -1
   110  	accTransitioningToDedicatedRoute = -2
   111  )
   112  
   113  // Account based limits.
   114  type limits struct {
   115  	mpay           int32
   116  	msubs          int32
   117  	mconns         int32
   118  	mleafs         int32
   119  	disallowBearer bool
   120  }
   121  
   122  // Used to track remote clients and leafnodes per remote server.
   123  type sconns struct {
   124  	conns int32
   125  	leafs int32
   126  }
   127  
   128  // Import stream mapping struct
   129  type streamImport struct {
   130  	acc     *Account
   131  	from    string
   132  	to      string
   133  	tr      *subjectTransform
   134  	rtr     *subjectTransform
   135  	claim   *jwt.Import
   136  	usePub  bool
   137  	invalid bool
   138  	// This is `allow_trace` and when true and message tracing is happening,
   139  	// we will trace egresses past the account boundary, if `false`, we stop
   140  	// at the account boundary.
   141  	atrc bool
   142  }
   143  
   144  const ClientInfoHdr = "Nats-Request-Info"
   145  
   146  // Import service mapping struct
   147  type serviceImport struct {
   148  	acc         *Account
   149  	claim       *jwt.Import
   150  	se          *serviceExport
   151  	sid         []byte
   152  	from        string
   153  	to          string
   154  	tr          *subjectTransform
   155  	ts          int64
   156  	rt          ServiceRespType
   157  	latency     *serviceLatency
   158  	m1          *ServiceLatency
   159  	rc          *client
   160  	usePub      bool
   161  	response    bool
   162  	invalid     bool
   163  	share       bool
   164  	tracking    bool
   165  	didDeliver  bool
   166  	atrc        bool        // allow trace (got from service export)
   167  	trackingHdr http.Header // header from request
   168  }
   169  
   170  // This is used to record when we create a mapping for implicit service
   171  // imports. We use this to clean up entries that are not singletons when
   172  // we detect that interest is no longer present. The key to the map will
   173  // be the actual interest. We record the mapped subject and the account.
   174  type serviceRespEntry struct {
   175  	acc  *Account
   176  	msub string
   177  }
   178  
   179  // ServiceRespType represents the types of service request response types.
   180  type ServiceRespType uint8
   181  
   182  // Service response types. Defaults to a singleton.
   183  const (
   184  	Singleton ServiceRespType = iota
   185  	Streamed
   186  	Chunked
   187  )
   188  
   189  // String helper.
   190  func (rt ServiceRespType) String() string {
   191  	switch rt {
   192  	case Singleton:
   193  		return "Singleton"
   194  	case Streamed:
   195  		return "Streamed"
   196  	case Chunked:
   197  		return "Chunked"
   198  	}
   199  	return "Unknown ServiceResType"
   200  }
   201  
   202  // exportAuth holds configured approvals or boolean indicating an
   203  // auth token is required for import.
   204  type exportAuth struct {
   205  	tokenReq    bool
   206  	accountPos  uint
   207  	approved    map[string]*Account
   208  	actsRevoked map[string]int64
   209  }
   210  
   211  // streamExport
   212  type streamExport struct {
   213  	exportAuth
   214  }
   215  
   216  // serviceExport holds additional information for exported services.
   217  type serviceExport struct {
   218  	exportAuth
   219  	acc        *Account
   220  	respType   ServiceRespType
   221  	latency    *serviceLatency
   222  	rtmr       *time.Timer
   223  	respThresh time.Duration
   224  	// This is `allow_trace` and when true and message tracing is happening,
   225  	// when processing a service import we will go through account boundary
   226  	// and trace egresses on that other account. If `false`, we stop at the
   227  	// account boundary.
   228  	atrc bool
   229  }
   230  
   231  // Used to track service latency.
   232  type serviceLatency struct {
   233  	sampling int8 // percentage from 1-100 or 0 to indicate triggered by header
   234  	subject  string
   235  }
   236  
   237  // exportMap tracks the exported streams and services.
   238  type exportMap struct {
   239  	streams   map[string]*streamExport
   240  	services  map[string]*serviceExport
   241  	responses map[string]*serviceImport
   242  }
   243  
   244  // importMap tracks the imported streams and services.
   245  // For services we will also track the response mappings as well.
   246  type importMap struct {
   247  	streams  []*streamImport
   248  	services map[string]*serviceImport
   249  	rrMap    map[string][]*serviceRespEntry
   250  }
   251  
   252  // NewAccount creates a new unlimited account with the given name.
   253  func NewAccount(name string) *Account {
   254  	a := &Account{
   255  		Name:        name,
   256  		LogicalName: name,
   257  		limits:      limits{-1, -1, -1, -1, false},
   258  		eventIds:    nuid.New(),
   259  	}
   260  	return a
   261  }
   262  
   263  func (a *Account) String() string {
   264  	return a.Name
   265  }
   266  
   267  func (a *Account) setTraceDest(dest string) {
   268  	a.mu.Lock()
   269  	a.traceDest = dest
   270  	a.mu.Unlock()
   271  }
   272  
   273  func (a *Account) getTraceDestAndSampling() (string, int) {
   274  	a.mu.RLock()
   275  	dest := a.traceDest
   276  	sampling := a.traceDestSampling
   277  	a.mu.RUnlock()
   278  	return dest, sampling
   279  }
   280  
   281  // Used to create shallow copies of accounts for transfer
   282  // from opts to real accounts in server struct.
   283  // Account `na` write lock is expected to be held on entry
   284  // while account `a` is the one from the Options struct
   285  // being loaded/reloaded and do not need locking.
   286  func (a *Account) shallowCopy(na *Account) {
   287  	na.Nkey = a.Nkey
   288  	na.Issuer = a.Issuer
   289  	na.traceDest, na.traceDestSampling = a.traceDest, a.traceDestSampling
   290  
   291  	if a.imports.streams != nil {
   292  		na.imports.streams = make([]*streamImport, 0, len(a.imports.streams))
   293  		for _, v := range a.imports.streams {
   294  			si := *v
   295  			na.imports.streams = append(na.imports.streams, &si)
   296  		}
   297  	}
   298  	if a.imports.services != nil {
   299  		na.imports.services = make(map[string]*serviceImport)
   300  		for k, v := range a.imports.services {
   301  			si := *v
   302  			na.imports.services[k] = &si
   303  		}
   304  	}
   305  	if a.exports.streams != nil {
   306  		na.exports.streams = make(map[string]*streamExport)
   307  		for k, v := range a.exports.streams {
   308  			if v != nil {
   309  				se := *v
   310  				na.exports.streams[k] = &se
   311  			} else {
   312  				na.exports.streams[k] = nil
   313  			}
   314  		}
   315  	}
   316  	if a.exports.services != nil {
   317  		na.exports.services = make(map[string]*serviceExport)
   318  		for k, v := range a.exports.services {
   319  			if v != nil {
   320  				se := *v
   321  				na.exports.services[k] = &se
   322  			} else {
   323  				na.exports.services[k] = nil
   324  			}
   325  		}
   326  	}
   327  	na.mappings = a.mappings
   328  	na.hasMapped.Store(len(na.mappings) > 0)
   329  
   330  	// JetStream
   331  	na.jsLimits = a.jsLimits
   332  	// Server config account limits.
   333  	na.limits = a.limits
   334  }
   335  
   336  // nextEventID uses its own lock for better concurrency.
   337  func (a *Account) nextEventID() string {
   338  	a.eventIdsMu.Lock()
   339  	id := a.eventIds.Next()
   340  	a.eventIdsMu.Unlock()
   341  	return id
   342  }
   343  
   344  // Returns a slice of clients stored in the account, or nil if none is present.
   345  // Lock is held on entry.
   346  func (a *Account) getClientsLocked() []*client {
   347  	if len(a.clients) == 0 {
   348  		return nil
   349  	}
   350  	clients := make([]*client, 0, len(a.clients))
   351  	for c := range a.clients {
   352  		clients = append(clients, c)
   353  	}
   354  	return clients
   355  }
   356  
   357  // Returns a slice of clients stored in the account, or nil if none is present.
   358  func (a *Account) getClients() []*client {
   359  	a.mu.RLock()
   360  	clients := a.getClientsLocked()
   361  	a.mu.RUnlock()
   362  	return clients
   363  }
   364  
   365  // Called to track a remote server and connections and leafnodes it
   366  // has for this account.
   367  func (a *Account) updateRemoteServer(m *AccountNumConns) []*client {
   368  	a.mu.Lock()
   369  	if a.strack == nil {
   370  		a.strack = make(map[string]sconns)
   371  	}
   372  	// This does not depend on receiving all updates since each one is idempotent.
   373  	// FIXME(dlc) - We should cleanup when these both go to zero.
   374  	prev := a.strack[m.Server.ID]
   375  	a.strack[m.Server.ID] = sconns{conns: int32(m.Conns), leafs: int32(m.LeafNodes)}
   376  	a.nrclients += int32(m.Conns) - prev.conns
   377  	a.nrleafs += int32(m.LeafNodes) - prev.leafs
   378  
   379  	mtce := a.mconns != jwt.NoLimit && (len(a.clients)-int(a.sysclients)+int(a.nrclients) > int(a.mconns))
   380  	// If we are over here some have snuck in and we need to rebalance.
   381  	// All others will probably be doing the same thing but better to be
   382  	// conservative and bit harsh here. Clients will reconnect if we over compensate.
   383  	var clients []*client
   384  	if mtce {
   385  		clients := a.getClientsLocked()
   386  		sort.Slice(clients, func(i, j int) bool {
   387  			return clients[i].start.After(clients[j].start)
   388  		})
   389  		over := (len(a.clients) - int(a.sysclients) + int(a.nrclients)) - int(a.mconns)
   390  		if over < len(clients) {
   391  			clients = clients[:over]
   392  		}
   393  	}
   394  	// Now check leafnodes.
   395  	mtlce := a.mleafs != jwt.NoLimit && (a.nleafs+a.nrleafs > a.mleafs)
   396  	if mtlce {
   397  		// Take ones from the end.
   398  		a.lmu.RLock()
   399  		leafs := a.lleafs
   400  		over := int(a.nleafs + a.nrleafs - a.mleafs)
   401  		if over < len(leafs) {
   402  			leafs = leafs[len(leafs)-over:]
   403  		}
   404  		clients = append(clients, leafs...)
   405  		a.lmu.RUnlock()
   406  	}
   407  	a.mu.Unlock()
   408  
   409  	// If we have exceeded our max clients this will be populated.
   410  	return clients
   411  }
   412  
   413  // Removes tracking for a remote server that has shutdown.
   414  func (a *Account) removeRemoteServer(sid string) {
   415  	a.mu.Lock()
   416  	if a.strack != nil {
   417  		prev := a.strack[sid]
   418  		delete(a.strack, sid)
   419  		a.nrclients -= prev.conns
   420  		a.nrleafs -= prev.leafs
   421  	}
   422  	a.mu.Unlock()
   423  }
   424  
   425  // When querying for subject interest this is the number of
   426  // expected responses. We need to actually check that the entry
   427  // has active connections.
   428  func (a *Account) expectedRemoteResponses() (expected int32) {
   429  	a.mu.RLock()
   430  	for _, sc := range a.strack {
   431  		if sc.conns > 0 || sc.leafs > 0 {
   432  			expected++
   433  		}
   434  	}
   435  	a.mu.RUnlock()
   436  	return
   437  }
   438  
   439  // Clears eventing and tracking for this account.
   440  func (a *Account) clearEventing() {
   441  	a.mu.Lock()
   442  	a.nrclients = 0
   443  	// Now clear state
   444  	clearTimer(&a.etmr)
   445  	clearTimer(&a.ctmr)
   446  	a.clients = nil
   447  	a.strack = nil
   448  	a.mu.Unlock()
   449  }
   450  
   451  // GetName will return the accounts name.
   452  func (a *Account) GetName() string {
   453  	if a == nil {
   454  		return "n/a"
   455  	}
   456  	a.mu.RLock()
   457  	name := a.Name
   458  	a.mu.RUnlock()
   459  	return name
   460  }
   461  
   462  // NumConnections returns active number of clients for this account for
   463  // all known servers.
   464  func (a *Account) NumConnections() int {
   465  	a.mu.RLock()
   466  	nc := len(a.clients) - int(a.sysclients) + int(a.nrclients)
   467  	a.mu.RUnlock()
   468  	return nc
   469  }
   470  
   471  // NumRemoteConnections returns the number of client or leaf connections that
   472  // are not on this server.
   473  func (a *Account) NumRemoteConnections() int {
   474  	a.mu.RLock()
   475  	nc := int(a.nrclients + a.nrleafs)
   476  	a.mu.RUnlock()
   477  	return nc
   478  }
   479  
   480  // NumLocalConnections returns active number of clients for this account
   481  // on this server.
   482  func (a *Account) NumLocalConnections() int {
   483  	a.mu.RLock()
   484  	nlc := a.numLocalConnections()
   485  	a.mu.RUnlock()
   486  	return nlc
   487  }
   488  
   489  // Do not account for the system accounts.
   490  func (a *Account) numLocalConnections() int {
   491  	return len(a.clients) - int(a.sysclients) - int(a.nleafs)
   492  }
   493  
   494  // This is for extended local interest.
   495  // Lock should not be held.
   496  func (a *Account) numLocalAndLeafConnections() int {
   497  	a.mu.RLock()
   498  	nlc := len(a.clients) - int(a.sysclients)
   499  	a.mu.RUnlock()
   500  	return nlc
   501  }
   502  
   503  func (a *Account) numLocalLeafNodes() int {
   504  	return int(a.nleafs)
   505  }
   506  
   507  // MaxTotalConnectionsReached returns if we have reached our limit for number of connections.
   508  func (a *Account) MaxTotalConnectionsReached() bool {
   509  	var mtce bool
   510  	a.mu.RLock()
   511  	if a.mconns != jwt.NoLimit {
   512  		mtce = len(a.clients)-int(a.sysclients)+int(a.nrclients) >= int(a.mconns)
   513  	}
   514  	a.mu.RUnlock()
   515  	return mtce
   516  }
   517  
   518  // MaxActiveConnections return the set limit for the account system
   519  // wide for total number of active connections.
   520  func (a *Account) MaxActiveConnections() int {
   521  	a.mu.RLock()
   522  	mconns := int(a.mconns)
   523  	a.mu.RUnlock()
   524  	return mconns
   525  }
   526  
   527  // MaxTotalLeafNodesReached returns if we have reached our limit for number of leafnodes.
   528  func (a *Account) MaxTotalLeafNodesReached() bool {
   529  	a.mu.RLock()
   530  	mtc := a.maxTotalLeafNodesReached()
   531  	a.mu.RUnlock()
   532  	return mtc
   533  }
   534  
   535  func (a *Account) maxTotalLeafNodesReached() bool {
   536  	if a.mleafs != jwt.NoLimit {
   537  		return a.nleafs+a.nrleafs >= a.mleafs
   538  	}
   539  	return false
   540  }
   541  
   542  // NumLeafNodes returns the active number of local and remote
   543  // leaf node connections.
   544  func (a *Account) NumLeafNodes() int {
   545  	a.mu.RLock()
   546  	nln := int(a.nleafs + a.nrleafs)
   547  	a.mu.RUnlock()
   548  	return nln
   549  }
   550  
   551  // NumRemoteLeafNodes returns the active number of remote
   552  // leaf node connections.
   553  func (a *Account) NumRemoteLeafNodes() int {
   554  	a.mu.RLock()
   555  	nrn := int(a.nrleafs)
   556  	a.mu.RUnlock()
   557  	return nrn
   558  }
   559  
   560  // MaxActiveLeafNodes return the set limit for the account system
   561  // wide for total number of leavenode connections.
   562  // NOTE: these are tracked separately.
   563  func (a *Account) MaxActiveLeafNodes() int {
   564  	a.mu.RLock()
   565  	mleafs := int(a.mleafs)
   566  	a.mu.RUnlock()
   567  	return mleafs
   568  }
   569  
   570  // RoutedSubs returns how many subjects we would send across a route when first
   571  // connected or expressing interest. Local client subs.
   572  func (a *Account) RoutedSubs() int {
   573  	a.mu.RLock()
   574  	defer a.mu.RUnlock()
   575  	return len(a.rm)
   576  }
   577  
   578  // TotalSubs returns total number of Subscriptions for this account.
   579  func (a *Account) TotalSubs() int {
   580  	a.mu.RLock()
   581  	defer a.mu.RUnlock()
   582  	if a.sl == nil {
   583  		return 0
   584  	}
   585  	return int(a.sl.Count())
   586  }
   587  
   588  func (a *Account) shouldLogMaxSubErr() bool {
   589  	if a == nil {
   590  		return true
   591  	}
   592  	a.mu.RLock()
   593  	last := a.lastLimErr
   594  	a.mu.RUnlock()
   595  	if now := time.Now().UnixNano(); now-last >= maxSubLimitReportThreshold {
   596  		a.mu.Lock()
   597  		a.lastLimErr = now
   598  		a.mu.Unlock()
   599  		return true
   600  	}
   601  	return false
   602  }
   603  
   604  // MapDest is for mapping published subjects for clients.
   605  type MapDest struct {
   606  	Subject string `json:"subject"`
   607  	Weight  uint8  `json:"weight"`
   608  	Cluster string `json:"cluster,omitempty"`
   609  }
   610  
   611  func NewMapDest(subject string, weight uint8) *MapDest {
   612  	return &MapDest{subject, weight, _EMPTY_}
   613  }
   614  
   615  // destination is for internal representation for a weighted mapped destination.
   616  type destination struct {
   617  	tr     *subjectTransform
   618  	weight uint8
   619  }
   620  
   621  // mapping is an internal entry for mapping subjects.
   622  type mapping struct {
   623  	src    string
   624  	wc     bool
   625  	dests  []*destination
   626  	cdests map[string][]*destination
   627  }
   628  
   629  // AddMapping adds in a simple route mapping from src subject to dest subject
   630  // for inbound client messages.
   631  func (a *Account) AddMapping(src, dest string) error {
   632  	return a.AddWeightedMappings(src, NewMapDest(dest, 100))
   633  }
   634  
   635  // AddWeightedMappings will add in a weighted mappings for the destinations.
   636  func (a *Account) AddWeightedMappings(src string, dests ...*MapDest) error {
   637  	a.mu.Lock()
   638  	defer a.mu.Unlock()
   639  
   640  	if !IsValidSubject(src) {
   641  		return ErrBadSubject
   642  	}
   643  
   644  	m := &mapping{src: src, wc: subjectHasWildcard(src), dests: make([]*destination, 0, len(dests)+1)}
   645  	seen := make(map[string]struct{})
   646  
   647  	var tw = make(map[string]uint8)
   648  	for _, d := range dests {
   649  		if _, ok := seen[d.Subject]; ok {
   650  			return fmt.Errorf("duplicate entry for %q", d.Subject)
   651  		}
   652  		seen[d.Subject] = struct{}{}
   653  		if d.Weight > 100 {
   654  			return fmt.Errorf("individual weights need to be <= 100")
   655  		}
   656  		tw[d.Cluster] += d.Weight
   657  		if tw[d.Cluster] > 100 {
   658  			return fmt.Errorf("total weight needs to be <= 100")
   659  		}
   660  		err := ValidateMappingDestination(d.Subject)
   661  		if err != nil {
   662  			return err
   663  		}
   664  		tr, err := NewSubjectTransform(src, d.Subject)
   665  		if err != nil {
   666  			return err
   667  		}
   668  		if d.Cluster == _EMPTY_ {
   669  			m.dests = append(m.dests, &destination{tr, d.Weight})
   670  		} else {
   671  			// We have a cluster scoped filter.
   672  			if m.cdests == nil {
   673  				m.cdests = make(map[string][]*destination)
   674  			}
   675  			ad := m.cdests[d.Cluster]
   676  			ad = append(ad, &destination{tr, d.Weight})
   677  			m.cdests[d.Cluster] = ad
   678  		}
   679  	}
   680  
   681  	processDestinations := func(dests []*destination) ([]*destination, error) {
   682  		var ltw uint8
   683  		for _, d := range dests {
   684  			ltw += d.weight
   685  		}
   686  		// Auto add in original at weight difference if all entries weight does not total to 100.
   687  		// Iff the src was not already added in explicitly, meaning they want loss.
   688  		_, haveSrc := seen[src]
   689  		if ltw != 100 && !haveSrc {
   690  			dest := src
   691  			if m.wc {
   692  				// We need to make the appropriate markers for the wildcards etc.
   693  				dest = transformTokenize(dest)
   694  			}
   695  			tr, err := NewSubjectTransform(src, dest)
   696  			if err != nil {
   697  				return nil, err
   698  			}
   699  			aw := 100 - ltw
   700  			if len(dests) == 0 {
   701  				aw = 100
   702  			}
   703  			dests = append(dests, &destination{tr, aw})
   704  		}
   705  		sort.Slice(dests, func(i, j int) bool { return dests[i].weight < dests[j].weight })
   706  
   707  		var lw uint8
   708  		for _, d := range dests {
   709  			d.weight += lw
   710  			lw = d.weight
   711  		}
   712  		return dests, nil
   713  	}
   714  
   715  	var err error
   716  	if m.dests, err = processDestinations(m.dests); err != nil {
   717  		return err
   718  	}
   719  
   720  	// Option cluster scoped destinations
   721  	for cluster, dests := range m.cdests {
   722  		if dests, err = processDestinations(dests); err != nil {
   723  			return err
   724  		}
   725  		m.cdests[cluster] = dests
   726  	}
   727  
   728  	// Replace an old one if it exists.
   729  	for i, em := range a.mappings {
   730  		if em.src == src {
   731  			a.mappings[i] = m
   732  			return nil
   733  		}
   734  	}
   735  	// If we did not replace add to the end.
   736  	a.mappings = append(a.mappings, m)
   737  	a.hasMapped.Store(len(a.mappings) > 0)
   738  
   739  	// If we have connected leafnodes make sure to update.
   740  	if a.nleafs > 0 {
   741  		// Need to release because lock ordering is client -> account
   742  		a.mu.Unlock()
   743  		// Now grab the leaf list lock. We can hold client lock under this one.
   744  		a.lmu.RLock()
   745  		for _, lc := range a.lleafs {
   746  			lc.forceAddToSmap(src)
   747  		}
   748  		a.lmu.RUnlock()
   749  		a.mu.Lock()
   750  	}
   751  	return nil
   752  }
   753  
   754  // RemoveMapping will remove an existing mapping.
   755  func (a *Account) RemoveMapping(src string) bool {
   756  	a.mu.Lock()
   757  	defer a.mu.Unlock()
   758  	for i, m := range a.mappings {
   759  		if m.src == src {
   760  			// Swap last one into this spot. Its ok to change order.
   761  			a.mappings[i] = a.mappings[len(a.mappings)-1]
   762  			a.mappings[len(a.mappings)-1] = nil // gc
   763  			a.mappings = a.mappings[:len(a.mappings)-1]
   764  			a.hasMapped.Store(len(a.mappings) > 0)
   765  			// If we have connected leafnodes make sure to update.
   766  			if a.nleafs > 0 {
   767  				// Need to release because lock ordering is client -> account
   768  				a.mu.Unlock()
   769  				// Now grab the leaf list lock. We can hold client lock under this one.
   770  				a.lmu.RLock()
   771  				for _, lc := range a.lleafs {
   772  					lc.forceRemoveFromSmap(src)
   773  				}
   774  				a.lmu.RUnlock()
   775  				a.mu.Lock()
   776  			}
   777  			return true
   778  		}
   779  	}
   780  	return false
   781  }
   782  
   783  // Indicates we have mapping entries.
   784  func (a *Account) hasMappings() bool {
   785  	if a == nil {
   786  		return false
   787  	}
   788  	return a.hasMapped.Load()
   789  }
   790  
   791  // This performs the logic to map to a new dest subject based on mappings.
   792  // Should only be called from processInboundClientMsg or service import processing.
   793  func (a *Account) selectMappedSubject(dest string) (string, bool) {
   794  	if !a.hasMappings() {
   795  		return dest, false
   796  	}
   797  
   798  	a.mu.Lock()
   799  	// In case we have to tokenize for subset matching.
   800  	tsa := [32]string{}
   801  	tts := tsa[:0]
   802  
   803  	var m *mapping
   804  	for _, rm := range a.mappings {
   805  		if !rm.wc && rm.src == dest {
   806  			m = rm
   807  			break
   808  		} else {
   809  			// tokenize and reuse for subset matching.
   810  			if len(tts) == 0 {
   811  				start := 0
   812  				subject := dest
   813  				for i := 0; i < len(subject); i++ {
   814  					if subject[i] == btsep {
   815  						tts = append(tts, subject[start:i])
   816  						start = i + 1
   817  					}
   818  				}
   819  				tts = append(tts, subject[start:])
   820  			}
   821  			if isSubsetMatch(tts, rm.src) {
   822  				m = rm
   823  				break
   824  			}
   825  		}
   826  	}
   827  
   828  	if m == nil {
   829  		a.mu.Unlock()
   830  		return dest, false
   831  	}
   832  
   833  	// The selected destination for the mapping.
   834  	var d *destination
   835  	var ndest string
   836  
   837  	dests := m.dests
   838  	if len(m.cdests) > 0 {
   839  		cn := a.srv.cachedClusterName()
   840  		dests = m.cdests[cn]
   841  		if dests == nil {
   842  			// Fallback to main if we do not match the cluster.
   843  			dests = m.dests
   844  		}
   845  	}
   846  
   847  	// Optimize for single entry case.
   848  	if len(dests) == 1 && dests[0].weight == 100 {
   849  		d = dests[0]
   850  	} else {
   851  		w := uint8(fastrand.Uint32n(100))
   852  		for _, rm := range dests {
   853  			if w < rm.weight {
   854  				d = rm
   855  				break
   856  			}
   857  		}
   858  	}
   859  
   860  	if d != nil {
   861  		if len(d.tr.dtokmftokindexesargs) == 0 {
   862  			ndest = d.tr.dest
   863  		} else {
   864  			ndest = d.tr.TransformTokenizedSubject(tts)
   865  		}
   866  	}
   867  
   868  	a.mu.Unlock()
   869  	return ndest, true
   870  }
   871  
   872  // SubscriptionInterest returns true if this account has a matching subscription
   873  // for the given `subject`.
   874  func (a *Account) SubscriptionInterest(subject string) bool {
   875  	return a.Interest(subject) > 0
   876  }
   877  
   878  // Interest returns the number of subscriptions for a given subject that match.
   879  func (a *Account) Interest(subject string) int {
   880  	var nms int
   881  	a.mu.RLock()
   882  	if a.sl != nil {
   883  		res := a.sl.Match(subject)
   884  		nms = len(res.psubs) + len(res.qsubs)
   885  	}
   886  	a.mu.RUnlock()
   887  	return nms
   888  }
   889  
   890  // addClient keeps our accounting of local active clients or leafnodes updated.
   891  // Returns previous total.
   892  func (a *Account) addClient(c *client) int {
   893  	a.mu.Lock()
   894  	n := len(a.clients)
   895  	if a.clients != nil {
   896  		a.clients[c] = struct{}{}
   897  	}
   898  	// If we did not add it, we are done
   899  	if n == len(a.clients) {
   900  		a.mu.Unlock()
   901  		return n
   902  	}
   903  	if c.kind != CLIENT && c.kind != LEAF {
   904  		a.sysclients++
   905  	} else if c.kind == LEAF {
   906  		a.nleafs++
   907  	}
   908  	a.mu.Unlock()
   909  
   910  	// If we added a new leaf use the list lock and add it to the list.
   911  	if c.kind == LEAF {
   912  		a.lmu.Lock()
   913  		a.lleafs = append(a.lleafs, c)
   914  		a.lmu.Unlock()
   915  	}
   916  
   917  	if c != nil && c.srv != nil {
   918  		c.srv.accConnsUpdate(a)
   919  	}
   920  
   921  	return n
   922  }
   923  
   924  // For registering clusters for remote leafnodes.
   925  // We only register as the hub.
   926  func (a *Account) registerLeafNodeCluster(cluster string) {
   927  	a.mu.Lock()
   928  	defer a.mu.Unlock()
   929  	if a.leafClusters == nil {
   930  		a.leafClusters = make(map[string]uint64)
   931  	}
   932  	a.leafClusters[cluster]++
   933  }
   934  
   935  // Check to see if we already have this cluster registered.
   936  func (a *Account) hasLeafNodeCluster(cluster string) bool {
   937  	a.mu.RLock()
   938  	defer a.mu.RUnlock()
   939  	return a.leafClusters[cluster] > 0
   940  }
   941  
   942  // Check to see if this cluster is isolated, meaning the only one.
   943  // Read Lock should be held.
   944  func (a *Account) isLeafNodeClusterIsolated(cluster string) bool {
   945  	if cluster == _EMPTY_ {
   946  		return false
   947  	}
   948  	if len(a.leafClusters) > 1 {
   949  		return false
   950  	}
   951  	return a.leafClusters[cluster] == uint64(a.nleafs)
   952  }
   953  
   954  // Helper function to remove leaf nodes. If number of leafnodes gets large
   955  // this may need to be optimized out of linear search but believe number
   956  // of active leafnodes per account scope to be small and therefore cache friendly.
   957  // Lock should not be held on general account lock.
   958  func (a *Account) removeLeafNode(c *client) {
   959  	// Make sure we hold the list lock as well.
   960  	a.lmu.Lock()
   961  	defer a.lmu.Unlock()
   962  
   963  	ll := len(a.lleafs)
   964  	for i, l := range a.lleafs {
   965  		if l == c {
   966  			a.lleafs[i] = a.lleafs[ll-1]
   967  			if ll == 1 {
   968  				a.lleafs = nil
   969  			} else {
   970  				a.lleafs = a.lleafs[:ll-1]
   971  			}
   972  			return
   973  		}
   974  	}
   975  }
   976  
   977  // removeClient keeps our accounting of local active clients updated.
   978  func (a *Account) removeClient(c *client) int {
   979  	a.mu.Lock()
   980  	n := len(a.clients)
   981  	delete(a.clients, c)
   982  	// If we did not actually remove it, we are done.
   983  	if n == len(a.clients) {
   984  		a.mu.Unlock()
   985  		return n
   986  	}
   987  	if c.kind != CLIENT && c.kind != LEAF {
   988  		a.sysclients--
   989  	} else if c.kind == LEAF {
   990  		a.nleafs--
   991  		// Need to do cluster accounting here.
   992  		// Do cluster accounting if we are a hub.
   993  		if c.isHubLeafNode() {
   994  			cluster := c.remoteCluster()
   995  			if count := a.leafClusters[cluster]; count > 1 {
   996  				a.leafClusters[cluster]--
   997  			} else if count == 1 {
   998  				delete(a.leafClusters, cluster)
   999  			}
  1000  		}
  1001  	}
  1002  	a.mu.Unlock()
  1003  
  1004  	if c.kind == LEAF {
  1005  		a.removeLeafNode(c)
  1006  	}
  1007  
  1008  	if c != nil && c.srv != nil {
  1009  		c.srv.accConnsUpdate(a)
  1010  	}
  1011  
  1012  	return n
  1013  }
  1014  
  1015  func setExportAuth(ea *exportAuth, subject string, accounts []*Account, accountPos uint) error {
  1016  	if accountPos > 0 {
  1017  		token := strings.Split(subject, tsep)
  1018  		if len(token) < int(accountPos) || token[accountPos-1] != "*" {
  1019  			return ErrInvalidSubject
  1020  		}
  1021  	}
  1022  	ea.accountPos = accountPos
  1023  	// empty means auth required but will be import token.
  1024  	if accounts == nil {
  1025  		return nil
  1026  	}
  1027  	if len(accounts) == 0 {
  1028  		ea.tokenReq = true
  1029  		return nil
  1030  	}
  1031  	if ea.approved == nil {
  1032  		ea.approved = make(map[string]*Account, len(accounts))
  1033  	}
  1034  	for _, acc := range accounts {
  1035  		ea.approved[acc.Name] = acc
  1036  	}
  1037  	return nil
  1038  }
  1039  
  1040  // AddServiceExport will configure the account with the defined export.
  1041  func (a *Account) AddServiceExport(subject string, accounts []*Account) error {
  1042  	return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, 0)
  1043  }
  1044  
  1045  // AddServiceExport will configure the account with the defined export.
  1046  func (a *Account) addServiceExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error {
  1047  	return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, accountPos)
  1048  }
  1049  
  1050  // AddServiceExportWithResponse will configure the account with the defined export and response type.
  1051  func (a *Account) AddServiceExportWithResponse(subject string, respType ServiceRespType, accounts []*Account) error {
  1052  	return a.addServiceExportWithResponseAndAccountPos(subject, respType, accounts, 0)
  1053  }
  1054  
  1055  // AddServiceExportWithresponse will configure the account with the defined export and response type.
  1056  func (a *Account) addServiceExportWithResponseAndAccountPos(
  1057  	subject string, respType ServiceRespType, accounts []*Account, accountPos uint) error {
  1058  	if a == nil {
  1059  		return ErrMissingAccount
  1060  	}
  1061  
  1062  	a.mu.Lock()
  1063  	if a.exports.services == nil {
  1064  		a.exports.services = make(map[string]*serviceExport)
  1065  	}
  1066  
  1067  	se := a.exports.services[subject]
  1068  	// Always  create a service export
  1069  	if se == nil {
  1070  		se = &serviceExport{}
  1071  	}
  1072  
  1073  	if respType != Singleton {
  1074  		se.respType = respType
  1075  	}
  1076  
  1077  	if accounts != nil || accountPos > 0 {
  1078  		if err := setExportAuth(&se.exportAuth, subject, accounts, accountPos); err != nil {
  1079  			a.mu.Unlock()
  1080  			return err
  1081  		}
  1082  	}
  1083  	lrt := a.lowestServiceExportResponseTime()
  1084  	se.acc = a
  1085  	se.respThresh = DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD
  1086  	a.exports.services[subject] = se
  1087  
  1088  	var clients []*client
  1089  	nlrt := a.lowestServiceExportResponseTime()
  1090  	if nlrt != lrt && len(a.clients) > 0 {
  1091  		clients = a.getClientsLocked()
  1092  	}
  1093  	// Need to release because lock ordering is client -> Account
  1094  	a.mu.Unlock()
  1095  	if len(clients) > 0 {
  1096  		updateAllClientsServiceExportResponseTime(clients, nlrt)
  1097  	}
  1098  	return nil
  1099  }
  1100  
  1101  // TrackServiceExport will enable latency tracking of the named service.
  1102  // Results will be published in this account to the given results subject.
  1103  func (a *Account) TrackServiceExport(service, results string) error {
  1104  	return a.TrackServiceExportWithSampling(service, results, DEFAULT_SERVICE_LATENCY_SAMPLING)
  1105  }
  1106  
  1107  // TrackServiceExportWithSampling will enable latency tracking of the named service for the given
  1108  // sampling rate (1-100). Results will be published in this account to the given results subject.
  1109  func (a *Account) TrackServiceExportWithSampling(service, results string, sampling int) error {
  1110  	if a == nil {
  1111  		return ErrMissingAccount
  1112  	}
  1113  
  1114  	if sampling != 0 { // 0 means triggered by header
  1115  		if sampling < 1 || sampling > 100 {
  1116  			return ErrBadSampling
  1117  		}
  1118  	}
  1119  	if !IsValidPublishSubject(results) {
  1120  		return ErrBadPublishSubject
  1121  	}
  1122  	// Don't loop back on outselves.
  1123  	if a.IsExportService(results) {
  1124  		return ErrBadPublishSubject
  1125  	}
  1126  
  1127  	if a.srv != nil && !a.srv.EventsEnabled() {
  1128  		return ErrNoSysAccount
  1129  	}
  1130  
  1131  	a.mu.Lock()
  1132  	if a.exports.services == nil {
  1133  		a.mu.Unlock()
  1134  		return ErrMissingService
  1135  	}
  1136  	ea, ok := a.exports.services[service]
  1137  	if !ok {
  1138  		a.mu.Unlock()
  1139  		return ErrMissingService
  1140  	}
  1141  	if ea == nil {
  1142  		ea = &serviceExport{}
  1143  		a.exports.services[service] = ea
  1144  	} else if ea.respType != Singleton {
  1145  		a.mu.Unlock()
  1146  		return ErrBadServiceType
  1147  	}
  1148  	ea.latency = &serviceLatency{
  1149  		sampling: int8(sampling),
  1150  		subject:  results,
  1151  	}
  1152  	s := a.srv
  1153  	a.mu.Unlock()
  1154  
  1155  	if s == nil {
  1156  		return nil
  1157  	}
  1158  
  1159  	// Now track down the imports and add in latency as needed to enable.
  1160  	s.accounts.Range(func(k, v any) bool {
  1161  		acc := v.(*Account)
  1162  		acc.mu.Lock()
  1163  		for _, im := range acc.imports.services {
  1164  			if im != nil && im.acc.Name == a.Name && subjectIsSubsetMatch(im.to, service) {
  1165  				im.latency = ea.latency
  1166  			}
  1167  		}
  1168  		acc.mu.Unlock()
  1169  		return true
  1170  	})
  1171  
  1172  	return nil
  1173  }
  1174  
  1175  // UnTrackServiceExport will disable latency tracking of the named service.
  1176  func (a *Account) UnTrackServiceExport(service string) {
  1177  	if a == nil || (a.srv != nil && !a.srv.EventsEnabled()) {
  1178  		return
  1179  	}
  1180  
  1181  	a.mu.Lock()
  1182  	if a.exports.services == nil {
  1183  		a.mu.Unlock()
  1184  		return
  1185  	}
  1186  	ea, ok := a.exports.services[service]
  1187  	if !ok || ea == nil || ea.latency == nil {
  1188  		a.mu.Unlock()
  1189  		return
  1190  	}
  1191  	// We have latency here.
  1192  	ea.latency = nil
  1193  	s := a.srv
  1194  	a.mu.Unlock()
  1195  
  1196  	if s == nil {
  1197  		return
  1198  	}
  1199  
  1200  	// Now track down the imports and clean them up.
  1201  	s.accounts.Range(func(k, v any) bool {
  1202  		acc := v.(*Account)
  1203  		acc.mu.Lock()
  1204  		for _, im := range acc.imports.services {
  1205  			if im != nil && im.acc.Name == a.Name {
  1206  				if subjectIsSubsetMatch(im.to, service) {
  1207  					im.latency, im.m1 = nil, nil
  1208  				}
  1209  			}
  1210  		}
  1211  		acc.mu.Unlock()
  1212  		return true
  1213  	})
  1214  }
  1215  
  1216  // IsExportService will indicate if this service exists. Will check wildcard scenarios.
  1217  func (a *Account) IsExportService(service string) bool {
  1218  	a.mu.RLock()
  1219  	defer a.mu.RUnlock()
  1220  	_, ok := a.exports.services[service]
  1221  	if ok {
  1222  		return true
  1223  	}
  1224  	tokens := strings.Split(service, tsep)
  1225  	for subj := range a.exports.services {
  1226  		if isSubsetMatch(tokens, subj) {
  1227  			return true
  1228  		}
  1229  	}
  1230  	return false
  1231  }
  1232  
  1233  // IsExportServiceTracking will indicate if given publish subject is an export service with tracking enabled.
  1234  func (a *Account) IsExportServiceTracking(service string) bool {
  1235  	a.mu.RLock()
  1236  	ea, ok := a.exports.services[service]
  1237  	if ok && ea == nil {
  1238  		a.mu.RUnlock()
  1239  		return false
  1240  	}
  1241  	if ok && ea != nil && ea.latency != nil {
  1242  		a.mu.RUnlock()
  1243  		return true
  1244  	}
  1245  	// FIXME(dlc) - Might want to cache this is in the hot path checking for latency tracking.
  1246  	tokens := strings.Split(service, tsep)
  1247  	for subj, ea := range a.exports.services {
  1248  		if isSubsetMatch(tokens, subj) && ea != nil && ea.latency != nil {
  1249  			a.mu.RUnlock()
  1250  			return true
  1251  		}
  1252  	}
  1253  	a.mu.RUnlock()
  1254  	return false
  1255  }
  1256  
  1257  // ServiceLatency is the JSON message sent out in response to latency tracking for
  1258  // an accounts exported services. Additional client info is available in requestor
  1259  // and responder. Note that for a requestor, the only information shared by default
  1260  // is the RTT used to calculate the total latency. The requestor's account can
  1261  // designate to share the additional information in the service import.
  1262  type ServiceLatency struct {
  1263  	TypedEvent
  1264  	Status         int           `json:"status"`
  1265  	Error          string        `json:"description,omitempty"`
  1266  	Requestor      *ClientInfo   `json:"requestor,omitempty"`
  1267  	Responder      *ClientInfo   `json:"responder,omitempty"`
  1268  	RequestHeader  http.Header   `json:"header,omitempty"` // only contains header(s) triggering the measurement
  1269  	RequestStart   time.Time     `json:"start"`
  1270  	ServiceLatency time.Duration `json:"service"`
  1271  	SystemLatency  time.Duration `json:"system"`
  1272  	TotalLatency   time.Duration `json:"total"`
  1273  }
  1274  
  1275  // ServiceLatencyType is the NATS Event Type for ServiceLatency
  1276  const ServiceLatencyType = "io.nats.server.metric.v1.service_latency"
  1277  
  1278  // NATSTotalTime is a helper function that totals the NATS latencies.
  1279  func (m1 *ServiceLatency) NATSTotalTime() time.Duration {
  1280  	return m1.Requestor.RTT + m1.Responder.RTT + m1.SystemLatency
  1281  }
  1282  
  1283  // Merge function to merge m1 and m2 (requestor and responder) measurements
  1284  // when there are two samples. This happens when the requestor and responder
  1285  // are on different servers.
  1286  //
  1287  // m2 ServiceLatency is correct, so use that.
  1288  // m1 TotalLatency is correct, so use that.
  1289  // Will use those to back into NATS latency.
  1290  func (m1 *ServiceLatency) merge(m2 *ServiceLatency) {
  1291  	rtt := time.Duration(0)
  1292  	if m2.Responder != nil {
  1293  		rtt = m2.Responder.RTT
  1294  	}
  1295  	m1.SystemLatency = m1.ServiceLatency - (m2.ServiceLatency + rtt)
  1296  	m1.ServiceLatency = m2.ServiceLatency
  1297  	m1.Responder = m2.Responder
  1298  	sanitizeLatencyMetric(m1)
  1299  }
  1300  
  1301  // sanitizeLatencyMetric adjusts latency metric values that could go
  1302  // negative in some edge conditions since we estimate client RTT
  1303  // for both requestor and responder.
  1304  // These numbers are never meant to be negative, it just could be
  1305  // how we back into the values based on estimated RTT.
  1306  func sanitizeLatencyMetric(sl *ServiceLatency) {
  1307  	if sl.ServiceLatency < 0 {
  1308  		sl.ServiceLatency = 0
  1309  	}
  1310  	if sl.SystemLatency < 0 {
  1311  		sl.SystemLatency = 0
  1312  	}
  1313  }
  1314  
  1315  // Used for transporting remote latency measurements.
  1316  type remoteLatency struct {
  1317  	Account    string         `json:"account"`
  1318  	ReqId      string         `json:"req_id"`
  1319  	M2         ServiceLatency `json:"m2"`
  1320  	respThresh time.Duration
  1321  }
  1322  
  1323  // sendLatencyResult will send a latency result and clear the si of the requestor(rc).
  1324  func (a *Account) sendLatencyResult(si *serviceImport, sl *ServiceLatency) {
  1325  	sl.Type = ServiceLatencyType
  1326  	sl.ID = a.nextEventID()
  1327  	sl.Time = time.Now().UTC()
  1328  	a.mu.Lock()
  1329  	lsubj := si.latency.subject
  1330  	si.rc = nil
  1331  	a.mu.Unlock()
  1332  
  1333  	a.srv.sendInternalAccountMsg(a, lsubj, sl)
  1334  }
  1335  
  1336  // Used to send a bad request metric when we do not have a reply subject
  1337  func (a *Account) sendBadRequestTrackingLatency(si *serviceImport, requestor *client, header http.Header) {
  1338  	sl := &ServiceLatency{
  1339  		Status:    400,
  1340  		Error:     "Bad Request",
  1341  		Requestor: requestor.getClientInfo(si.share),
  1342  	}
  1343  	sl.RequestHeader = header
  1344  	sl.RequestStart = time.Now().Add(-sl.Requestor.RTT).UTC()
  1345  	a.sendLatencyResult(si, sl)
  1346  }
  1347  
  1348  // Used to send a latency result when the requestor interest was lost before the
  1349  // response could be delivered.
  1350  func (a *Account) sendReplyInterestLostTrackLatency(si *serviceImport) {
  1351  	sl := &ServiceLatency{
  1352  		Status: 408,
  1353  		Error:  "Request Timeout",
  1354  	}
  1355  	a.mu.RLock()
  1356  	rc, share, ts := si.rc, si.share, si.ts
  1357  	sl.RequestHeader = si.trackingHdr
  1358  	a.mu.RUnlock()
  1359  	if rc != nil {
  1360  		sl.Requestor = rc.getClientInfo(share)
  1361  	}
  1362  	sl.RequestStart = time.Unix(0, ts-int64(sl.Requestor.RTT)).UTC()
  1363  	a.sendLatencyResult(si, sl)
  1364  }
  1365  
  1366  func (a *Account) sendBackendErrorTrackingLatency(si *serviceImport, reason rsiReason) {
  1367  	sl := &ServiceLatency{}
  1368  	a.mu.RLock()
  1369  	rc, share, ts := si.rc, si.share, si.ts
  1370  	sl.RequestHeader = si.trackingHdr
  1371  	a.mu.RUnlock()
  1372  	if rc != nil {
  1373  		sl.Requestor = rc.getClientInfo(share)
  1374  	}
  1375  	var reqRTT time.Duration
  1376  	if sl.Requestor != nil {
  1377  		reqRTT = sl.Requestor.RTT
  1378  	}
  1379  	sl.RequestStart = time.Unix(0, ts-int64(reqRTT)).UTC()
  1380  	if reason == rsiNoDelivery {
  1381  		sl.Status = 503
  1382  		sl.Error = "Service Unavailable"
  1383  	} else if reason == rsiTimeout {
  1384  		sl.Status = 504
  1385  		sl.Error = "Service Timeout"
  1386  	}
  1387  	a.sendLatencyResult(si, sl)
  1388  }
  1389  
  1390  // sendTrackingLatency will send out the appropriate tracking information for the
  1391  // service request/response latency. This is called when the requestor's server has
  1392  // received the response.
  1393  // TODO(dlc) - holding locks for RTTs may be too much long term. Should revisit.
  1394  func (a *Account) sendTrackingLatency(si *serviceImport, responder *client) bool {
  1395  	a.mu.RLock()
  1396  	rc := si.rc
  1397  	a.mu.RUnlock()
  1398  	if rc == nil {
  1399  		return true
  1400  	}
  1401  
  1402  	ts := time.Now()
  1403  	serviceRTT := time.Duration(ts.UnixNano() - si.ts)
  1404  	requestor := si.rc
  1405  
  1406  	sl := &ServiceLatency{
  1407  		Status:    200,
  1408  		Requestor: requestor.getClientInfo(si.share),
  1409  		Responder: responder.getClientInfo(true),
  1410  	}
  1411  	var respRTT, reqRTT time.Duration
  1412  	if sl.Responder != nil {
  1413  		respRTT = sl.Responder.RTT
  1414  	}
  1415  	if sl.Requestor != nil {
  1416  		reqRTT = sl.Requestor.RTT
  1417  	}
  1418  	sl.RequestStart = time.Unix(0, si.ts-int64(reqRTT)).UTC()
  1419  	sl.ServiceLatency = serviceRTT - respRTT
  1420  	sl.TotalLatency = reqRTT + serviceRTT
  1421  	if respRTT > 0 {
  1422  		sl.SystemLatency = time.Since(ts)
  1423  		sl.TotalLatency += sl.SystemLatency
  1424  	}
  1425  	sl.RequestHeader = si.trackingHdr
  1426  	sanitizeLatencyMetric(sl)
  1427  
  1428  	sl.Type = ServiceLatencyType
  1429  	sl.ID = a.nextEventID()
  1430  	sl.Time = time.Now().UTC()
  1431  
  1432  	// If we are expecting a remote measurement, store our sl here.
  1433  	// We need to account for the race between this and us receiving the
  1434  	// remote measurement.
  1435  	// FIXME(dlc) - We need to clean these up but this should happen
  1436  	// already with the auto-expire logic.
  1437  	if responder != nil && responder.kind != CLIENT {
  1438  		si.acc.mu.Lock()
  1439  		if si.m1 != nil {
  1440  			m1, m2 := sl, si.m1
  1441  			m1.merge(m2)
  1442  			si.acc.mu.Unlock()
  1443  			a.srv.sendInternalAccountMsg(a, si.latency.subject, m1)
  1444  			a.mu.Lock()
  1445  			si.rc = nil
  1446  			a.mu.Unlock()
  1447  			return true
  1448  		}
  1449  		si.m1 = sl
  1450  		si.acc.mu.Unlock()
  1451  		return false
  1452  	} else {
  1453  		a.srv.sendInternalAccountMsg(a, si.latency.subject, sl)
  1454  		a.mu.Lock()
  1455  		si.rc = nil
  1456  		a.mu.Unlock()
  1457  	}
  1458  	return true
  1459  }
  1460  
  1461  // This will check to make sure our response lower threshold is set
  1462  // properly in any clients doing rrTracking.
  1463  func updateAllClientsServiceExportResponseTime(clients []*client, lrt time.Duration) {
  1464  	for _, c := range clients {
  1465  		c.mu.Lock()
  1466  		if c.rrTracking != nil && lrt != c.rrTracking.lrt {
  1467  			c.rrTracking.lrt = lrt
  1468  			if c.rrTracking.ptmr.Stop() {
  1469  				c.rrTracking.ptmr.Reset(lrt)
  1470  			}
  1471  		}
  1472  		c.mu.Unlock()
  1473  	}
  1474  }
  1475  
  1476  // Will select the lowest respThresh from all service exports.
  1477  // Read lock should be held.
  1478  func (a *Account) lowestServiceExportResponseTime() time.Duration {
  1479  	// Lowest we will allow is 5 minutes. Its an upper bound for this function.
  1480  	lrt := 5 * time.Minute
  1481  	for _, se := range a.exports.services {
  1482  		if se.respThresh < lrt {
  1483  			lrt = se.respThresh
  1484  		}
  1485  	}
  1486  	return lrt
  1487  }
  1488  
  1489  // AddServiceImportWithClaim will add in the service import via the jwt claim.
  1490  func (a *Account) AddServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import) error {
  1491  	return a.addServiceImportWithClaim(destination, from, to, imClaim, false)
  1492  }
  1493  
  1494  // addServiceImportWithClaim will add in the service import via the jwt claim.
  1495  // It will also skip the authorization check in cases where internal is true
  1496  func (a *Account) addServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import, internal bool) error {
  1497  	if destination == nil {
  1498  		return ErrMissingAccount
  1499  	}
  1500  	// Empty means use from.
  1501  	if to == _EMPTY_ {
  1502  		to = from
  1503  	}
  1504  	if !IsValidSubject(from) || !IsValidSubject(to) {
  1505  		return ErrInvalidSubject
  1506  	}
  1507  
  1508  	// First check to see if the account has authorized us to route to the "to" subject.
  1509  	if !internal && !destination.checkServiceImportAuthorized(a, to, imClaim) {
  1510  		return ErrServiceImportAuthorization
  1511  	}
  1512  
  1513  	// Check if this introduces a cycle before proceeding.
  1514  	if err := a.serviceImportFormsCycle(destination, from); err != nil {
  1515  		return err
  1516  	}
  1517  
  1518  	_, err := a.addServiceImport(destination, from, to, imClaim)
  1519  
  1520  	return err
  1521  }
  1522  
  1523  const MaxAccountCycleSearchDepth = 1024
  1524  
  1525  func (a *Account) serviceImportFormsCycle(dest *Account, from string) error {
  1526  	return dest.checkServiceImportsForCycles(from, map[string]bool{a.Name: true})
  1527  }
  1528  
  1529  func (a *Account) checkServiceImportsForCycles(from string, visited map[string]bool) error {
  1530  	if len(visited) >= MaxAccountCycleSearchDepth {
  1531  		return ErrCycleSearchDepth
  1532  	}
  1533  	a.mu.RLock()
  1534  	for _, si := range a.imports.services {
  1535  		if SubjectsCollide(from, si.to) {
  1536  			a.mu.RUnlock()
  1537  			if visited[si.acc.Name] {
  1538  				return ErrImportFormsCycle
  1539  			}
  1540  			// Push ourselves and check si.acc
  1541  			visited[a.Name] = true
  1542  			if subjectIsSubsetMatch(si.from, from) {
  1543  				from = si.from
  1544  			}
  1545  			if err := si.acc.checkServiceImportsForCycles(from, visited); err != nil {
  1546  				return err
  1547  			}
  1548  			a.mu.RLock()
  1549  		}
  1550  	}
  1551  	a.mu.RUnlock()
  1552  	return nil
  1553  }
  1554  
  1555  func (a *Account) streamImportFormsCycle(dest *Account, to string) error {
  1556  	return dest.checkStreamImportsForCycles(to, map[string]bool{a.Name: true})
  1557  }
  1558  
  1559  // Lock should be held.
  1560  func (a *Account) hasServiceExportMatching(to string) bool {
  1561  	for subj := range a.exports.services {
  1562  		if subjectIsSubsetMatch(to, subj) {
  1563  			return true
  1564  		}
  1565  	}
  1566  	return false
  1567  }
  1568  
  1569  // Lock should be held.
  1570  func (a *Account) hasStreamExportMatching(to string) bool {
  1571  	for subj := range a.exports.streams {
  1572  		if subjectIsSubsetMatch(to, subj) {
  1573  			return true
  1574  		}
  1575  	}
  1576  	return false
  1577  }
  1578  
  1579  func (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool) error {
  1580  	if len(visited) >= MaxAccountCycleSearchDepth {
  1581  		return ErrCycleSearchDepth
  1582  	}
  1583  
  1584  	a.mu.RLock()
  1585  
  1586  	if !a.hasStreamExportMatching(to) {
  1587  		a.mu.RUnlock()
  1588  		return nil
  1589  	}
  1590  
  1591  	for _, si := range a.imports.streams {
  1592  		if SubjectsCollide(to, si.to) {
  1593  			a.mu.RUnlock()
  1594  			if visited[si.acc.Name] {
  1595  				return ErrImportFormsCycle
  1596  			}
  1597  			// Push ourselves and check si.acc
  1598  			visited[a.Name] = true
  1599  			if subjectIsSubsetMatch(si.to, to) {
  1600  				to = si.to
  1601  			}
  1602  			if err := si.acc.checkStreamImportsForCycles(to, visited); err != nil {
  1603  				return err
  1604  			}
  1605  			a.mu.RLock()
  1606  		}
  1607  	}
  1608  	a.mu.RUnlock()
  1609  	return nil
  1610  }
  1611  
  1612  // SetServiceImportSharing will allow sharing of information about requests with the export account.
  1613  // Used for service latency tracking at the moment.
  1614  func (a *Account) SetServiceImportSharing(destination *Account, to string, allow bool) error {
  1615  	return a.setServiceImportSharing(destination, to, true, allow)
  1616  }
  1617  
  1618  // setServiceImportSharing will allow sharing of information about requests with the export account.
  1619  func (a *Account) setServiceImportSharing(destination *Account, to string, check, allow bool) error {
  1620  	a.mu.Lock()
  1621  	defer a.mu.Unlock()
  1622  	if check && a.isClaimAccount() {
  1623  		return fmt.Errorf("claim based accounts can not be updated directly")
  1624  	}
  1625  	for _, si := range a.imports.services {
  1626  		if si.acc == destination && si.to == to {
  1627  			si.share = allow
  1628  			return nil
  1629  		}
  1630  	}
  1631  	return fmt.Errorf("service import not found")
  1632  }
  1633  
  1634  // AddServiceImport will add a route to an account to send published messages / requests
  1635  // to the destination account. From is the local subject to map, To is the
  1636  // subject that will appear on the destination account. Destination will need
  1637  // to have an import rule to allow access via addService.
  1638  func (a *Account) AddServiceImport(destination *Account, from, to string) error {
  1639  	return a.AddServiceImportWithClaim(destination, from, to, nil)
  1640  }
  1641  
  1642  // NumPendingReverseResponses returns the number of response mappings we have for all outstanding
  1643  // requests for service imports.
  1644  func (a *Account) NumPendingReverseResponses() int {
  1645  	a.mu.RLock()
  1646  	defer a.mu.RUnlock()
  1647  	return len(a.imports.rrMap)
  1648  }
  1649  
  1650  // NumPendingAllResponses return the number of all responses outstanding for service exports.
  1651  func (a *Account) NumPendingAllResponses() int {
  1652  	return a.NumPendingResponses(_EMPTY_)
  1653  }
  1654  
  1655  // NumPendingResponses returns the number of responses outstanding for service exports
  1656  // on this account. An empty filter string returns all responses regardless of which export.
  1657  // If you specify the filter we will only return ones that are for that export.
  1658  // NOTE this is only for what this server is tracking.
  1659  func (a *Account) NumPendingResponses(filter string) int {
  1660  	a.mu.RLock()
  1661  	defer a.mu.RUnlock()
  1662  	if filter == _EMPTY_ {
  1663  		return len(a.exports.responses)
  1664  	}
  1665  	se := a.getServiceExport(filter)
  1666  	if se == nil {
  1667  		return 0
  1668  	}
  1669  	var nre int
  1670  	for _, si := range a.exports.responses {
  1671  		if si.se == se {
  1672  			nre++
  1673  		}
  1674  	}
  1675  	return nre
  1676  }
  1677  
  1678  // NumServiceImports returns the number of service imports we have configured.
  1679  func (a *Account) NumServiceImports() int {
  1680  	a.mu.RLock()
  1681  	defer a.mu.RUnlock()
  1682  	return len(a.imports.services)
  1683  }
  1684  
  1685  // Reason why we are removing this response serviceImport.
  1686  type rsiReason int
  1687  
  1688  const (
  1689  	rsiOk = rsiReason(iota)
  1690  	rsiNoDelivery
  1691  	rsiTimeout
  1692  )
  1693  
  1694  // removeRespServiceImport removes a response si mapping and the reverse entries for interest detection.
  1695  func (a *Account) removeRespServiceImport(si *serviceImport, reason rsiReason) {
  1696  	if si == nil {
  1697  		return
  1698  	}
  1699  
  1700  	a.mu.Lock()
  1701  	c := a.ic
  1702  	delete(a.exports.responses, si.from)
  1703  	dest, to, tracking, rc, didDeliver := si.acc, si.to, si.tracking, si.rc, si.didDeliver
  1704  	a.mu.Unlock()
  1705  
  1706  	// If we have a sid make sure to unsub.
  1707  	if len(si.sid) > 0 && c != nil {
  1708  		c.processUnsub(si.sid)
  1709  	}
  1710  
  1711  	if tracking && rc != nil && !didDeliver {
  1712  		a.sendBackendErrorTrackingLatency(si, reason)
  1713  	}
  1714  
  1715  	dest.checkForReverseEntry(to, si, false)
  1716  }
  1717  
  1718  // removeServiceImport will remove the route by subject.
  1719  func (a *Account) removeServiceImport(subject string) {
  1720  	a.mu.Lock()
  1721  	si, ok := a.imports.services[subject]
  1722  	delete(a.imports.services, subject)
  1723  
  1724  	var sid []byte
  1725  	c := a.ic
  1726  
  1727  	if ok && si != nil {
  1728  		if a.ic != nil && si.sid != nil {
  1729  			sid = si.sid
  1730  		}
  1731  	}
  1732  	a.mu.Unlock()
  1733  
  1734  	if sid != nil {
  1735  		c.processUnsub(sid)
  1736  	}
  1737  }
  1738  
  1739  // This tracks responses to service requests mappings. This is used for cleanup.
  1740  func (a *Account) addReverseRespMapEntry(acc *Account, reply, from string) {
  1741  	a.mu.Lock()
  1742  	if a.imports.rrMap == nil {
  1743  		a.imports.rrMap = make(map[string][]*serviceRespEntry)
  1744  	}
  1745  	sre := &serviceRespEntry{acc, from}
  1746  	sra := a.imports.rrMap[reply]
  1747  	a.imports.rrMap[reply] = append(sra, sre)
  1748  	a.mu.Unlock()
  1749  }
  1750  
  1751  // checkForReverseEntries is for when we are trying to match reverse entries to a wildcard.
  1752  // This will be called from checkForReverseEntry when the reply arg is a wildcard subject.
  1753  // This will usually be called in a go routine since we need to walk all the entries.
  1754  func (a *Account) checkForReverseEntries(reply string, checkInterest, recursed bool) {
  1755  	if subjectIsLiteral(reply) {
  1756  		a._checkForReverseEntry(reply, nil, checkInterest, recursed)
  1757  		return
  1758  	}
  1759  
  1760  	a.mu.RLock()
  1761  	if len(a.imports.rrMap) == 0 {
  1762  		a.mu.RUnlock()
  1763  		return
  1764  	}
  1765  
  1766  	var _rs [64]string
  1767  	rs := _rs[:0]
  1768  	if n := len(a.imports.rrMap); n > cap(rs) {
  1769  		rs = make([]string, 0, n)
  1770  	}
  1771  
  1772  	for k := range a.imports.rrMap {
  1773  		rs = append(rs, k)
  1774  	}
  1775  	a.mu.RUnlock()
  1776  
  1777  	tsa := [32]string{}
  1778  	tts := tokenizeSubjectIntoSlice(tsa[:0], reply)
  1779  
  1780  	rsa := [32]string{}
  1781  	for _, r := range rs {
  1782  		rts := tokenizeSubjectIntoSlice(rsa[:0], r)
  1783  		//  isSubsetMatchTokenized is heavy so make sure we do this without the lock.
  1784  		if isSubsetMatchTokenized(rts, tts) {
  1785  			a._checkForReverseEntry(r, nil, checkInterest, recursed)
  1786  		}
  1787  	}
  1788  }
  1789  
  1790  // This checks for any response map entries. If you specify an si we will only match and
  1791  // clean up for that one, otherwise we remove them all.
  1792  func (a *Account) checkForReverseEntry(reply string, si *serviceImport, checkInterest bool) {
  1793  	a._checkForReverseEntry(reply, si, checkInterest, false)
  1794  }
  1795  
  1796  // Callers should use checkForReverseEntry instead. This function exists to help prevent
  1797  // infinite recursion.
  1798  func (a *Account) _checkForReverseEntry(reply string, si *serviceImport, checkInterest, recursed bool) {
  1799  	a.mu.RLock()
  1800  	if len(a.imports.rrMap) == 0 {
  1801  		a.mu.RUnlock()
  1802  		return
  1803  	}
  1804  
  1805  	if subjectHasWildcard(reply) {
  1806  		if recursed {
  1807  			// If we have reached this condition then it is because the reverse entries also
  1808  			// contain wildcards (that shouldn't happen but a client *could* provide an inbox
  1809  			// prefix that is illegal because it ends in a wildcard character), at which point
  1810  			// we will end up with infinite recursion between this func and checkForReverseEntries.
  1811  			// To avoid a stack overflow panic, we'll give up instead.
  1812  			a.mu.RUnlock()
  1813  			return
  1814  		}
  1815  
  1816  		doInline := len(a.imports.rrMap) <= 64
  1817  		a.mu.RUnlock()
  1818  
  1819  		if doInline {
  1820  			a.checkForReverseEntries(reply, checkInterest, true)
  1821  		} else {
  1822  			go a.checkForReverseEntries(reply, checkInterest, true)
  1823  		}
  1824  		return
  1825  	}
  1826  
  1827  	if sres := a.imports.rrMap[reply]; sres == nil {
  1828  		a.mu.RUnlock()
  1829  		return
  1830  	}
  1831  
  1832  	// If we are here we have an entry we should check.
  1833  	// If requested we will first check if there is any
  1834  	// interest for this subject for the entire account.
  1835  	// If there is we can not delete any entries yet.
  1836  	// Note that if we are here reply has to be a literal subject.
  1837  	if checkInterest {
  1838  		// If interest still exists we can not clean these up yet.
  1839  		if rr := a.sl.Match(reply); len(rr.psubs)+len(rr.qsubs) > 0 {
  1840  			a.mu.RUnlock()
  1841  			return
  1842  		}
  1843  	}
  1844  	a.mu.RUnlock()
  1845  
  1846  	// Delete the appropriate entries here based on optional si.
  1847  	a.mu.Lock()
  1848  	// We need a new lookup here because we have released the lock.
  1849  	sres := a.imports.rrMap[reply]
  1850  	if si == nil {
  1851  		delete(a.imports.rrMap, reply)
  1852  	} else if sres != nil {
  1853  		// Find the one we are looking for..
  1854  		for i, sre := range sres {
  1855  			if sre.msub == si.from {
  1856  				sres = append(sres[:i], sres[i+1:]...)
  1857  				break
  1858  			}
  1859  		}
  1860  		if len(sres) > 0 {
  1861  			a.imports.rrMap[si.to] = sres
  1862  		} else {
  1863  			delete(a.imports.rrMap, si.to)
  1864  		}
  1865  	}
  1866  	a.mu.Unlock()
  1867  
  1868  	// If we are here we no longer have interest and we have
  1869  	// response entries that we should clean up.
  1870  	if si == nil {
  1871  		// sres is now known to have been removed from a.imports.rrMap, so we
  1872  		// can safely (data race wise) iterate through.
  1873  		for _, sre := range sres {
  1874  			acc := sre.acc
  1875  			var trackingCleanup bool
  1876  			var rsi *serviceImport
  1877  			acc.mu.Lock()
  1878  			c := acc.ic
  1879  			if rsi = acc.exports.responses[sre.msub]; rsi != nil && !rsi.didDeliver {
  1880  				delete(acc.exports.responses, rsi.from)
  1881  				trackingCleanup = rsi.tracking && rsi.rc != nil
  1882  			}
  1883  			acc.mu.Unlock()
  1884  			// If we are doing explicit subs for all responses (e.g. bound to leafnode)
  1885  			// we will have a non-empty sid here.
  1886  			if rsi != nil && len(rsi.sid) > 0 && c != nil {
  1887  				c.processUnsub(rsi.sid)
  1888  			}
  1889  			if trackingCleanup {
  1890  				acc.sendReplyInterestLostTrackLatency(rsi)
  1891  			}
  1892  		}
  1893  	}
  1894  }
  1895  
  1896  // Checks to see if a potential service import subject is already overshadowed.
  1897  func (a *Account) serviceImportShadowed(from string) bool {
  1898  	a.mu.RLock()
  1899  	defer a.mu.RUnlock()
  1900  	if a.imports.services[from] != nil {
  1901  		return true
  1902  	}
  1903  	// We did not find a direct match, so check individually.
  1904  	for subj := range a.imports.services {
  1905  		if subjectIsSubsetMatch(from, subj) {
  1906  			return true
  1907  		}
  1908  	}
  1909  	return false
  1910  }
  1911  
  1912  // Internal check to see if a service import exists.
  1913  func (a *Account) serviceImportExists(from string) bool {
  1914  	a.mu.RLock()
  1915  	dup := a.imports.services[from]
  1916  	a.mu.RUnlock()
  1917  	return dup != nil
  1918  }
  1919  
  1920  // Add a service import.
  1921  // This does no checks and should only be called by the msg processing code.
  1922  // Use AddServiceImport from above if responding to user input or config changes, etc.
  1923  func (a *Account) addServiceImport(dest *Account, from, to string, claim *jwt.Import) (*serviceImport, error) {
  1924  	rt := Singleton
  1925  	var lat *serviceLatency
  1926  
  1927  	if dest == nil {
  1928  		return nil, ErrMissingAccount
  1929  	}
  1930  
  1931  	var atrc bool
  1932  	dest.mu.RLock()
  1933  	se := dest.getServiceExport(to)
  1934  	if se != nil {
  1935  		rt = se.respType
  1936  		lat = se.latency
  1937  		atrc = se.atrc
  1938  	}
  1939  	dest.mu.RUnlock()
  1940  
  1941  	a.mu.Lock()
  1942  	if a.imports.services == nil {
  1943  		a.imports.services = make(map[string]*serviceImport)
  1944  	} else if dup := a.imports.services[from]; dup != nil {
  1945  		a.mu.Unlock()
  1946  		return nil, fmt.Errorf("duplicate service import subject %q, previously used in import for account %q, subject %q",
  1947  			from, dup.acc.Name, dup.to)
  1948  	}
  1949  
  1950  	if to == _EMPTY_ {
  1951  		to = from
  1952  	}
  1953  	// Check to see if we have a wildcard
  1954  	var (
  1955  		usePub bool
  1956  		tr     *subjectTransform
  1957  		err    error
  1958  	)
  1959  	if subjectHasWildcard(to) {
  1960  		// If to and from match, then we use the published subject.
  1961  		if to == from {
  1962  			usePub = true
  1963  		} else {
  1964  			to, _ = transformUntokenize(to)
  1965  			// Create a transform. Do so in reverse such that $ symbols only exist in to
  1966  			if tr, err = NewSubjectTransformStrict(to, transformTokenize(from)); err != nil {
  1967  				a.mu.Unlock()
  1968  				return nil, fmt.Errorf("failed to create mapping transform for service import subject from %q to %q: %v",
  1969  					from, to, err)
  1970  			} else {
  1971  				// un-tokenize and reverse transform so we get the transform needed
  1972  				from, _ = transformUntokenize(from)
  1973  				tr = tr.reverse()
  1974  			}
  1975  		}
  1976  	}
  1977  	var share bool
  1978  	if claim != nil {
  1979  		share = claim.Share
  1980  	}
  1981  	si := &serviceImport{dest, claim, se, nil, from, to, tr, 0, rt, lat, nil, nil, usePub, false, false, share, false, false, atrc, nil}
  1982  	a.imports.services[from] = si
  1983  	a.mu.Unlock()
  1984  
  1985  	if err := a.addServiceImportSub(si); err != nil {
  1986  		a.removeServiceImport(si.from)
  1987  		return nil, err
  1988  	}
  1989  	return si, nil
  1990  }
  1991  
  1992  // Returns the internal client, will create one if not present.
  1993  // Lock should be held.
  1994  func (a *Account) internalClient() *client {
  1995  	if a.ic == nil && a.srv != nil {
  1996  		a.ic = a.srv.createInternalAccountClient()
  1997  		a.ic.acc = a
  1998  	}
  1999  	return a.ic
  2000  }
  2001  
  2002  // Internal account scoped subscriptions.
  2003  func (a *Account) subscribeInternal(subject string, cb msgHandler) (*subscription, error) {
  2004  	return a.subscribeInternalEx(subject, cb, false)
  2005  }
  2006  
  2007  // Unsubscribe from an internal account subscription.
  2008  func (a *Account) unsubscribeInternal(sub *subscription) {
  2009  	if ic := a.internalClient(); ic != nil {
  2010  		ic.processUnsub(sub.sid)
  2011  	}
  2012  }
  2013  
  2014  // Creates internal subscription for service import responses.
  2015  func (a *Account) subscribeServiceImportResponse(subject string) (*subscription, error) {
  2016  	return a.subscribeInternalEx(subject, a.processServiceImportResponse, true)
  2017  }
  2018  
  2019  func (a *Account) subscribeInternalEx(subject string, cb msgHandler, ri bool) (*subscription, error) {
  2020  	a.mu.Lock()
  2021  	a.isid++
  2022  	c, sid := a.internalClient(), strconv.FormatUint(a.isid, 10)
  2023  	a.mu.Unlock()
  2024  
  2025  	// This will happen in parsing when the account has not been properly setup.
  2026  	if c == nil {
  2027  		return nil, fmt.Errorf("no internal account client")
  2028  	}
  2029  
  2030  	return c.processSubEx([]byte(subject), nil, []byte(sid), cb, false, false, ri)
  2031  }
  2032  
  2033  // This will add an account subscription that matches the "from" from a service import entry.
  2034  func (a *Account) addServiceImportSub(si *serviceImport) error {
  2035  	a.mu.Lock()
  2036  	c := a.internalClient()
  2037  	// This will happen in parsing when the account has not been properly setup.
  2038  	if c == nil {
  2039  		a.mu.Unlock()
  2040  		return nil
  2041  	}
  2042  	if si.sid != nil {
  2043  		a.mu.Unlock()
  2044  		return fmt.Errorf("duplicate call to create subscription for service import")
  2045  	}
  2046  	a.isid++
  2047  	sid := strconv.FormatUint(a.isid, 10)
  2048  	si.sid = []byte(sid)
  2049  	subject := si.from
  2050  	a.mu.Unlock()
  2051  
  2052  	cb := func(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) {
  2053  		c.processServiceImport(si, acc, msg)
  2054  	}
  2055  	sub, err := c.processSubEx([]byte(subject), nil, []byte(sid), cb, true, true, false)
  2056  	if err != nil {
  2057  		return err
  2058  	}
  2059  	// Leafnodes introduce a new way to introduce messages into the system. Therefore forward import subscription
  2060  	// This is similar to what initLeafNodeSmapAndSendSubs does
  2061  	// TODO we need to consider performing this update as we get client subscriptions.
  2062  	//      This behavior would result in subscription propagation only where actually used.
  2063  	a.updateLeafNodes(sub, 1)
  2064  	return nil
  2065  }
  2066  
  2067  // Remove all the subscriptions associated with service imports.
  2068  func (a *Account) removeAllServiceImportSubs() {
  2069  	a.mu.RLock()
  2070  	var sids [][]byte
  2071  	for _, si := range a.imports.services {
  2072  		if si.sid != nil {
  2073  			sids = append(sids, si.sid)
  2074  			si.sid = nil
  2075  		}
  2076  	}
  2077  	c := a.ic
  2078  	a.ic = nil
  2079  	a.mu.RUnlock()
  2080  
  2081  	if c == nil {
  2082  		return
  2083  	}
  2084  	for _, sid := range sids {
  2085  		c.processUnsub(sid)
  2086  	}
  2087  	c.closeConnection(InternalClient)
  2088  }
  2089  
  2090  // Add in subscriptions for all registered service imports.
  2091  func (a *Account) addAllServiceImportSubs() {
  2092  	var sis [32]*serviceImport
  2093  	serviceImports := sis[:0]
  2094  	a.mu.RLock()
  2095  	for _, si := range a.imports.services {
  2096  		serviceImports = append(serviceImports, si)
  2097  	}
  2098  	a.mu.RUnlock()
  2099  	for _, si := range serviceImports {
  2100  		a.addServiceImportSub(si)
  2101  	}
  2102  }
  2103  
  2104  var (
  2105  	// header where all information is encoded in one value.
  2106  	trcUber = textproto.CanonicalMIMEHeaderKey("Uber-Trace-Id")
  2107  	trcCtx  = textproto.CanonicalMIMEHeaderKey("Traceparent")
  2108  	trcB3   = textproto.CanonicalMIMEHeaderKey("B3")
  2109  	// openzipkin header to check
  2110  	trcB3Sm = textproto.CanonicalMIMEHeaderKey("X-B3-Sampled")
  2111  	trcB3Id = textproto.CanonicalMIMEHeaderKey("X-B3-TraceId")
  2112  	// additional header needed to include when present
  2113  	trcB3PSId        = textproto.CanonicalMIMEHeaderKey("X-B3-ParentSpanId")
  2114  	trcB3SId         = textproto.CanonicalMIMEHeaderKey("X-B3-SpanId")
  2115  	trcCtxSt         = textproto.CanonicalMIMEHeaderKey("Tracestate")
  2116  	trcUberCtxPrefix = textproto.CanonicalMIMEHeaderKey("Uberctx-")
  2117  )
  2118  
  2119  func newB3Header(h http.Header) http.Header {
  2120  	retHdr := http.Header{}
  2121  	if v, ok := h[trcB3Sm]; ok {
  2122  		retHdr[trcB3Sm] = v
  2123  	}
  2124  	if v, ok := h[trcB3Id]; ok {
  2125  		retHdr[trcB3Id] = v
  2126  	}
  2127  	if v, ok := h[trcB3PSId]; ok {
  2128  		retHdr[trcB3PSId] = v
  2129  	}
  2130  	if v, ok := h[trcB3SId]; ok {
  2131  		retHdr[trcB3SId] = v
  2132  	}
  2133  	return retHdr
  2134  }
  2135  
  2136  func newUberHeader(h http.Header, tId []string) http.Header {
  2137  	retHdr := http.Header{trcUber: tId}
  2138  	for k, v := range h {
  2139  		if strings.HasPrefix(k, trcUberCtxPrefix) {
  2140  			retHdr[k] = v
  2141  		}
  2142  	}
  2143  	return retHdr
  2144  }
  2145  
  2146  func newTraceCtxHeader(h http.Header, tId []string) http.Header {
  2147  	retHdr := http.Header{trcCtx: tId}
  2148  	if v, ok := h[trcCtxSt]; ok {
  2149  		retHdr[trcCtxSt] = v
  2150  	}
  2151  	return retHdr
  2152  }
  2153  
  2154  // Helper to determine when to sample. When header has a value, sampling is driven by header
  2155  func shouldSample(l *serviceLatency, c *client) (bool, http.Header) {
  2156  	if l == nil {
  2157  		return false, nil
  2158  	}
  2159  	if l.sampling < 0 {
  2160  		return false, nil
  2161  	}
  2162  	if l.sampling >= 100 {
  2163  		return true, nil
  2164  	}
  2165  	if l.sampling > 0 && rand.Int31n(100) <= int32(l.sampling) {
  2166  		return true, nil
  2167  	}
  2168  	h := c.parseState.getHeader()
  2169  	if len(h) == 0 {
  2170  		return false, nil
  2171  	}
  2172  	if tId := h[trcUber]; len(tId) != 0 {
  2173  		// sample 479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1
  2174  		tk := strings.Split(tId[0], ":")
  2175  		if len(tk) == 4 && len(tk[3]) > 0 && len(tk[3]) <= 2 {
  2176  			dst := [2]byte{}
  2177  			src := [2]byte{'0', tk[3][0]}
  2178  			if len(tk[3]) == 2 {
  2179  				src[1] = tk[3][1]
  2180  			}
  2181  			if _, err := hex.Decode(dst[:], src[:]); err == nil && dst[0]&1 == 1 {
  2182  				return true, newUberHeader(h, tId)
  2183  			}
  2184  		}
  2185  		return false, nil
  2186  	} else if sampled := h[trcB3Sm]; len(sampled) != 0 && sampled[0] == "1" {
  2187  		return true, newB3Header(h) // allowed
  2188  	} else if len(sampled) != 0 && sampled[0] == "0" {
  2189  		return false, nil // denied
  2190  	} else if _, ok := h[trcB3Id]; ok {
  2191  		// sample 80f198ee56343ba864fe8b2a57d3eff7
  2192  		// presence (with X-B3-Sampled not being 0) means sampling left to recipient
  2193  		return true, newB3Header(h)
  2194  	} else if b3 := h[trcB3]; len(b3) != 0 {
  2195  		// sample 80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90
  2196  		// sample 0
  2197  		tk := strings.Split(b3[0], "-")
  2198  		if len(tk) > 2 && tk[2] == "0" {
  2199  			return false, nil // denied
  2200  		} else if len(tk) == 1 && tk[0] == "0" {
  2201  			return false, nil // denied
  2202  		}
  2203  		return true, http.Header{trcB3: b3} // sampling allowed or left to recipient of header
  2204  	} else if tId := h[trcCtx]; len(tId) != 0 {
  2205  		var sample bool
  2206  		// sample 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
  2207  		tk := strings.Split(tId[0], "-")
  2208  		if len(tk) == 4 && len([]byte(tk[3])) == 2 {
  2209  			if hexVal, err := strconv.ParseInt(tk[3], 16, 8); err == nil {
  2210  				sample = hexVal&0x1 == 0x1
  2211  			}
  2212  		}
  2213  		if sample {
  2214  			return true, newTraceCtxHeader(h, tId)
  2215  		} else {
  2216  			return false, nil
  2217  		}
  2218  	}
  2219  	return false, nil
  2220  }
  2221  
  2222  // Used to mimic client like replies.
  2223  const (
  2224  	replyPrefix    = "_R_."
  2225  	replyPrefixLen = len(replyPrefix)
  2226  	baseServerLen  = 10
  2227  	replyLen       = 6
  2228  	minReplyLen    = 15
  2229  	digits         = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
  2230  	base           = 62
  2231  )
  2232  
  2233  // This is where all service export responses are handled.
  2234  func (a *Account) processServiceImportResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {
  2235  	a.mu.RLock()
  2236  	if a.expired || len(a.exports.responses) == 0 {
  2237  		a.mu.RUnlock()
  2238  		return
  2239  	}
  2240  	si := a.exports.responses[subject]
  2241  
  2242  	if si == nil || si.invalid {
  2243  		a.mu.RUnlock()
  2244  		return
  2245  	}
  2246  	a.mu.RUnlock()
  2247  
  2248  	// Send for normal processing.
  2249  	c.processServiceImport(si, a, msg)
  2250  }
  2251  
  2252  // Will create the response prefix for fast generation of responses.
  2253  // A wildcard subscription may be used handle interest graph propagation
  2254  // for all service replies, unless we are bound to a leafnode.
  2255  // Lock should be held.
  2256  func (a *Account) createRespWildcard() {
  2257  	var b = [baseServerLen]byte{'_', 'R', '_', '.'}
  2258  	rn := fastrand.Uint64()
  2259  	for i, l := replyPrefixLen, rn; i < len(b); i++ {
  2260  		b[i] = digits[l%base]
  2261  		l /= base
  2262  	}
  2263  	a.siReply = append(b[:], '.')
  2264  }
  2265  
  2266  // Test whether this is a tracked reply.
  2267  func isTrackedReply(reply []byte) bool {
  2268  	lreply := len(reply) - 1
  2269  	return lreply > 3 && reply[lreply-1] == '.' && reply[lreply] == 'T'
  2270  }
  2271  
  2272  // Generate a new service reply from the wildcard prefix.
  2273  // FIXME(dlc) - probably do not have to use rand here. about 25ns per.
  2274  func (a *Account) newServiceReply(tracking bool) []byte {
  2275  	a.mu.Lock()
  2276  	s := a.srv
  2277  	rn := fastrand.Uint64()
  2278  
  2279  	// Check if we need to create the reply here.
  2280  	var createdSiReply bool
  2281  	if a.siReply == nil {
  2282  		a.createRespWildcard()
  2283  		createdSiReply = true
  2284  	}
  2285  	replyPre := a.siReply
  2286  	a.mu.Unlock()
  2287  
  2288  	// If we created the siReply and we are not bound to a leafnode
  2289  	// we need to do the wildcard subscription.
  2290  	if createdSiReply {
  2291  		a.subscribeServiceImportResponse(string(append(replyPre, '>')))
  2292  	}
  2293  
  2294  	var b [replyLen]byte
  2295  	for i, l := 0, rn; i < len(b); i++ {
  2296  		b[i] = digits[l%base]
  2297  		l /= base
  2298  	}
  2299  	// Make sure to copy.
  2300  	reply := make([]byte, 0, len(replyPre)+len(b))
  2301  	reply = append(reply, replyPre...)
  2302  	reply = append(reply, b[:]...)
  2303  
  2304  	if tracking && s.sys != nil {
  2305  		// Add in our tracking identifier. This allows the metrics to get back to only
  2306  		// this server without needless SUBS/UNSUBS.
  2307  		reply = append(reply, '.')
  2308  		reply = append(reply, s.sys.shash...)
  2309  		reply = append(reply, '.', 'T')
  2310  	}
  2311  
  2312  	return reply
  2313  }
  2314  
  2315  // Checks if a serviceImport was created to map responses.
  2316  func (si *serviceImport) isRespServiceImport() bool {
  2317  	return si != nil && si.response
  2318  }
  2319  
  2320  // Sets the response threshold timer for a service export.
  2321  // Account lock should be held
  2322  func (se *serviceExport) setResponseThresholdTimer() {
  2323  	if se.rtmr != nil {
  2324  		return // Already set
  2325  	}
  2326  	se.rtmr = time.AfterFunc(se.respThresh, se.checkExpiredResponses)
  2327  }
  2328  
  2329  // Account lock should be held
  2330  func (se *serviceExport) clearResponseThresholdTimer() bool {
  2331  	if se.rtmr == nil {
  2332  		return true
  2333  	}
  2334  	stopped := se.rtmr.Stop()
  2335  	se.rtmr = nil
  2336  	return stopped
  2337  }
  2338  
  2339  // checkExpiredResponses will check for any pending responses that need to
  2340  // be cleaned up.
  2341  func (se *serviceExport) checkExpiredResponses() {
  2342  	acc := se.acc
  2343  	if acc == nil {
  2344  		se.clearResponseThresholdTimer()
  2345  		return
  2346  	}
  2347  
  2348  	var expired []*serviceImport
  2349  	mints := time.Now().UnixNano() - int64(se.respThresh)
  2350  
  2351  	// TODO(dlc) - Should we release lock while doing this? Or only do these in batches?
  2352  	// Should we break this up for responses only from this service export?
  2353  	// Responses live on acc directly for fast inbound processsing for the _R_ wildcard.
  2354  	// We could do another indirection at this level but just to get to the service export?
  2355  	var totalResponses int
  2356  	acc.mu.RLock()
  2357  	for _, si := range acc.exports.responses {
  2358  		if si.se == se {
  2359  			totalResponses++
  2360  			if si.ts <= mints {
  2361  				expired = append(expired, si)
  2362  			}
  2363  		}
  2364  	}
  2365  	acc.mu.RUnlock()
  2366  
  2367  	for _, si := range expired {
  2368  		acc.removeRespServiceImport(si, rsiTimeout)
  2369  	}
  2370  
  2371  	// Pull out expired to determine if we have any left for timer.
  2372  	totalResponses -= len(expired)
  2373  
  2374  	// Redo timer as needed.
  2375  	acc.mu.Lock()
  2376  	if totalResponses > 0 && se.rtmr != nil {
  2377  		se.rtmr.Stop()
  2378  		se.rtmr.Reset(se.respThresh)
  2379  	} else {
  2380  		se.clearResponseThresholdTimer()
  2381  	}
  2382  	acc.mu.Unlock()
  2383  }
  2384  
  2385  // ServiceExportResponseThreshold returns the current threshold.
  2386  func (a *Account) ServiceExportResponseThreshold(export string) (time.Duration, error) {
  2387  	a.mu.Lock()
  2388  	defer a.mu.Unlock()
  2389  	se := a.getServiceExport(export)
  2390  	if se == nil {
  2391  		return 0, fmt.Errorf("no export defined for %q", export)
  2392  	}
  2393  	return se.respThresh, nil
  2394  }
  2395  
  2396  // SetServiceExportResponseThreshold sets the maximum time the system will a response to be delivered
  2397  // from a service export responder.
  2398  func (a *Account) SetServiceExportResponseThreshold(export string, maxTime time.Duration) error {
  2399  	a.mu.Lock()
  2400  	if a.isClaimAccount() {
  2401  		a.mu.Unlock()
  2402  		return fmt.Errorf("claim based accounts can not be updated directly")
  2403  	}
  2404  	lrt := a.lowestServiceExportResponseTime()
  2405  	se := a.getServiceExport(export)
  2406  	if se == nil {
  2407  		a.mu.Unlock()
  2408  		return fmt.Errorf("no export defined for %q", export)
  2409  	}
  2410  	se.respThresh = maxTime
  2411  
  2412  	var clients []*client
  2413  	nlrt := a.lowestServiceExportResponseTime()
  2414  	if nlrt != lrt && len(a.clients) > 0 {
  2415  		clients = a.getClientsLocked()
  2416  	}
  2417  	// Need to release because lock ordering is client -> Account
  2418  	a.mu.Unlock()
  2419  	if len(clients) > 0 {
  2420  		updateAllClientsServiceExportResponseTime(clients, nlrt)
  2421  	}
  2422  	return nil
  2423  }
  2424  
  2425  func (a *Account) SetServiceExportAllowTrace(export string, allowTrace bool) error {
  2426  	a.mu.Lock()
  2427  	se := a.getServiceExport(export)
  2428  	if se == nil {
  2429  		a.mu.Unlock()
  2430  		return fmt.Errorf("no export defined for %q", export)
  2431  	}
  2432  	se.atrc = allowTrace
  2433  	a.mu.Unlock()
  2434  	return nil
  2435  }
  2436  
  2437  // This is for internal service import responses.
  2438  func (a *Account) addRespServiceImport(dest *Account, to string, osi *serviceImport, tracking bool, header http.Header) *serviceImport {
  2439  	nrr := string(osi.acc.newServiceReply(tracking))
  2440  
  2441  	a.mu.Lock()
  2442  	rt := osi.rt
  2443  
  2444  	// dest is the requestor's account. a is the service responder with the export.
  2445  	// Marked as internal here, that is how we distinguish.
  2446  	si := &serviceImport{dest, nil, osi.se, nil, nrr, to, nil, 0, rt, nil, nil, nil, false, true, false, osi.share, false, false, false, nil}
  2447  
  2448  	if a.exports.responses == nil {
  2449  		a.exports.responses = make(map[string]*serviceImport)
  2450  	}
  2451  	a.exports.responses[nrr] = si
  2452  
  2453  	// Always grab time and make sure response threshold timer is running.
  2454  	si.ts = time.Now().UnixNano()
  2455  	if osi.se != nil {
  2456  		osi.se.setResponseThresholdTimer()
  2457  	}
  2458  
  2459  	if rt == Singleton && tracking {
  2460  		si.latency = osi.latency
  2461  		si.tracking = true
  2462  		si.trackingHdr = header
  2463  	}
  2464  	a.mu.Unlock()
  2465  
  2466  	// We do add in the reverse map such that we can detect loss of interest and do proper
  2467  	// cleanup of this si as interest goes away.
  2468  	dest.addReverseRespMapEntry(a, to, nrr)
  2469  
  2470  	return si
  2471  }
  2472  
  2473  // AddStreamImportWithClaim will add in the stream import from a specific account with optional token.
  2474  func (a *Account) AddStreamImportWithClaim(account *Account, from, prefix string, imClaim *jwt.Import) error {
  2475  	return a.addStreamImportWithClaim(account, from, prefix, false, imClaim)
  2476  }
  2477  
  2478  func (a *Account) addStreamImportWithClaim(account *Account, from, prefix string, allowTrace bool, imClaim *jwt.Import) error {
  2479  	if account == nil {
  2480  		return ErrMissingAccount
  2481  	}
  2482  
  2483  	// First check to see if the account has authorized export of the subject.
  2484  	if !account.checkStreamImportAuthorized(a, from, imClaim) {
  2485  		return ErrStreamImportAuthorization
  2486  	}
  2487  
  2488  	// Check prefix if it exists and make sure its a literal.
  2489  	// Append token separator if not already present.
  2490  	if prefix != _EMPTY_ {
  2491  		// Make sure there are no wildcards here, this prefix needs to be a literal
  2492  		// since it will be prepended to a publish subject.
  2493  		if !subjectIsLiteral(prefix) {
  2494  			return ErrStreamImportBadPrefix
  2495  		}
  2496  		if prefix[len(prefix)-1] != btsep {
  2497  			prefix = prefix + string(btsep)
  2498  		}
  2499  	}
  2500  
  2501  	return a.addMappedStreamImportWithClaim(account, from, prefix+from, allowTrace, imClaim)
  2502  }
  2503  
  2504  // AddMappedStreamImport helper for AddMappedStreamImportWithClaim
  2505  func (a *Account) AddMappedStreamImport(account *Account, from, to string) error {
  2506  	return a.AddMappedStreamImportWithClaim(account, from, to, nil)
  2507  }
  2508  
  2509  // AddMappedStreamImportWithClaim will add in the stream import from a specific account with optional token.
  2510  func (a *Account) AddMappedStreamImportWithClaim(account *Account, from, to string, imClaim *jwt.Import) error {
  2511  	return a.addMappedStreamImportWithClaim(account, from, to, false, imClaim)
  2512  }
  2513  
  2514  func (a *Account) addMappedStreamImportWithClaim(account *Account, from, to string, allowTrace bool, imClaim *jwt.Import) error {
  2515  	if account == nil {
  2516  		return ErrMissingAccount
  2517  	}
  2518  
  2519  	// First check to see if the account has authorized export of the subject.
  2520  	if !account.checkStreamImportAuthorized(a, from, imClaim) {
  2521  		return ErrStreamImportAuthorization
  2522  	}
  2523  
  2524  	if to == _EMPTY_ {
  2525  		to = from
  2526  	}
  2527  
  2528  	// Check if this forms a cycle.
  2529  	if err := a.streamImportFormsCycle(account, to); err != nil {
  2530  		return err
  2531  	}
  2532  
  2533  	var (
  2534  		usePub bool
  2535  		tr     *subjectTransform
  2536  		err    error
  2537  	)
  2538  	if subjectHasWildcard(from) {
  2539  		if to == from {
  2540  			usePub = true
  2541  		} else {
  2542  			// Create a transform
  2543  			if tr, err = NewSubjectTransformStrict(from, transformTokenize(to)); err != nil {
  2544  				return fmt.Errorf("failed to create mapping transform for stream import subject from %q to %q: %v",
  2545  					from, to, err)
  2546  			}
  2547  			to, _ = transformUntokenize(to)
  2548  		}
  2549  	}
  2550  
  2551  	a.mu.Lock()
  2552  	if a.isStreamImportDuplicate(account, from) {
  2553  		a.mu.Unlock()
  2554  		return ErrStreamImportDuplicate
  2555  	}
  2556  	if imClaim != nil {
  2557  		allowTrace = imClaim.AllowTrace
  2558  	}
  2559  	a.imports.streams = append(a.imports.streams, &streamImport{account, from, to, tr, nil, imClaim, usePub, false, allowTrace})
  2560  	a.mu.Unlock()
  2561  	return nil
  2562  }
  2563  
  2564  // isStreamImportDuplicate checks for duplicate.
  2565  // Lock should be held.
  2566  func (a *Account) isStreamImportDuplicate(acc *Account, from string) bool {
  2567  	for _, si := range a.imports.streams {
  2568  		if si.acc == acc && si.from == from {
  2569  			return true
  2570  		}
  2571  	}
  2572  	return false
  2573  }
  2574  
  2575  // AddStreamImport will add in the stream import from a specific account.
  2576  func (a *Account) AddStreamImport(account *Account, from, prefix string) error {
  2577  	return a.addStreamImportWithClaim(account, from, prefix, false, nil)
  2578  }
  2579  
  2580  // IsPublicExport is a placeholder to denote a public export.
  2581  var IsPublicExport = []*Account(nil)
  2582  
  2583  // AddStreamExport will add an export to the account. If accounts is nil
  2584  // it will signify a public export, meaning anyone can import.
  2585  func (a *Account) AddStreamExport(subject string, accounts []*Account) error {
  2586  	return a.addStreamExportWithAccountPos(subject, accounts, 0)
  2587  }
  2588  
  2589  // AddStreamExport will add an export to the account. If accounts is nil
  2590  // it will signify a public export, meaning anyone can import.
  2591  // if accountPos is > 0, all imports will be granted where the following holds:
  2592  // strings.Split(subject, tsep)[accountPos] == account id will be granted.
  2593  func (a *Account) addStreamExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error {
  2594  	if a == nil {
  2595  		return ErrMissingAccount
  2596  	}
  2597  
  2598  	a.mu.Lock()
  2599  	defer a.mu.Unlock()
  2600  
  2601  	if a.exports.streams == nil {
  2602  		a.exports.streams = make(map[string]*streamExport)
  2603  	}
  2604  	ea := a.exports.streams[subject]
  2605  	if accounts != nil || accountPos > 0 {
  2606  		if ea == nil {
  2607  			ea = &streamExport{}
  2608  		}
  2609  		if err := setExportAuth(&ea.exportAuth, subject, accounts, accountPos); err != nil {
  2610  			return err
  2611  		}
  2612  	}
  2613  	a.exports.streams[subject] = ea
  2614  	return nil
  2615  }
  2616  
  2617  // Check if another account is authorized to import from us.
  2618  func (a *Account) checkStreamImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool {
  2619  	// Find the subject in the exports list.
  2620  	a.mu.RLock()
  2621  	auth := a.checkStreamImportAuthorizedNoLock(account, subject, imClaim)
  2622  	a.mu.RUnlock()
  2623  	return auth
  2624  }
  2625  
  2626  func (a *Account) checkStreamImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool {
  2627  	if a.exports.streams == nil || !IsValidSubject(subject) {
  2628  		return false
  2629  	}
  2630  	return a.checkStreamExportApproved(account, subject, imClaim)
  2631  }
  2632  
  2633  func (a *Account) checkAuth(ea *exportAuth, account *Account, imClaim *jwt.Import, tokens []string) bool {
  2634  	// if ea is nil or ea.approved is nil, that denotes a public export
  2635  	if ea == nil || (len(ea.approved) == 0 && !ea.tokenReq && ea.accountPos == 0) {
  2636  		return true
  2637  	}
  2638  	// Check if the export is protected and enforces presence of importing account identity
  2639  	if ea.accountPos > 0 {
  2640  		return ea.accountPos <= uint(len(tokens)) && tokens[ea.accountPos-1] == account.Name
  2641  	}
  2642  	// Check if token required
  2643  	if ea.tokenReq {
  2644  		return a.checkActivation(account, imClaim, ea, true)
  2645  	}
  2646  	if ea.approved == nil {
  2647  		return false
  2648  	}
  2649  	// If we have a matching account we are authorized
  2650  	_, ok := ea.approved[account.Name]
  2651  	return ok
  2652  }
  2653  
  2654  func (a *Account) checkStreamExportApproved(account *Account, subject string, imClaim *jwt.Import) bool {
  2655  	// Check direct match of subject first
  2656  	ea, ok := a.exports.streams[subject]
  2657  	if ok {
  2658  		// if ea is nil or eq.approved is nil, that denotes a public export
  2659  		if ea == nil {
  2660  			return true
  2661  		}
  2662  		return a.checkAuth(&ea.exportAuth, account, imClaim, nil)
  2663  	}
  2664  
  2665  	// ok if we are here we did not match directly so we need to test each one.
  2666  	// The import subject arg has to take precedence, meaning the export
  2667  	// has to be a true subset of the import claim. We already checked for
  2668  	// exact matches above.
  2669  	tokens := strings.Split(subject, tsep)
  2670  	for subj, ea := range a.exports.streams {
  2671  		if isSubsetMatch(tokens, subj) {
  2672  			if ea == nil {
  2673  				return true
  2674  			}
  2675  			return a.checkAuth(&ea.exportAuth, account, imClaim, tokens)
  2676  		}
  2677  	}
  2678  	return false
  2679  }
  2680  
  2681  func (a *Account) checkServiceExportApproved(account *Account, subject string, imClaim *jwt.Import) bool {
  2682  	// Check direct match of subject first
  2683  	se, ok := a.exports.services[subject]
  2684  	if ok {
  2685  		// if se is nil or eq.approved is nil, that denotes a public export
  2686  		if se == nil {
  2687  			return true
  2688  		}
  2689  		return a.checkAuth(&se.exportAuth, account, imClaim, nil)
  2690  	}
  2691  	// ok if we are here we did not match directly so we need to test each one.
  2692  	// The import subject arg has to take precedence, meaning the export
  2693  	// has to be a true subset of the import claim. We already checked for
  2694  	// exact matches above.
  2695  	tokens := strings.Split(subject, tsep)
  2696  	for subj, se := range a.exports.services {
  2697  		if isSubsetMatch(tokens, subj) {
  2698  			if se == nil {
  2699  				return true
  2700  			}
  2701  			return a.checkAuth(&se.exportAuth, account, imClaim, tokens)
  2702  		}
  2703  	}
  2704  	return false
  2705  }
  2706  
  2707  // Helper function to get a serviceExport.
  2708  // Lock should be held on entry.
  2709  func (a *Account) getServiceExport(subj string) *serviceExport {
  2710  	se, ok := a.exports.services[subj]
  2711  	// The export probably has a wildcard, so lookup that up.
  2712  	if !ok {
  2713  		se = a.getWildcardServiceExport(subj)
  2714  	}
  2715  	return se
  2716  }
  2717  
  2718  // This helper is used when trying to match a serviceExport record that is
  2719  // represented by a wildcard.
  2720  // Lock should be held on entry.
  2721  func (a *Account) getWildcardServiceExport(from string) *serviceExport {
  2722  	tokens := strings.Split(from, tsep)
  2723  	for subj, se := range a.exports.services {
  2724  		if isSubsetMatch(tokens, subj) {
  2725  			return se
  2726  		}
  2727  	}
  2728  	return nil
  2729  }
  2730  
  2731  // These are import stream specific versions for when an activation expires.
  2732  func (a *Account) streamActivationExpired(exportAcc *Account, subject string) {
  2733  	a.mu.RLock()
  2734  	if a.expired || a.imports.streams == nil {
  2735  		a.mu.RUnlock()
  2736  		return
  2737  	}
  2738  	var si *streamImport
  2739  	for _, si = range a.imports.streams {
  2740  		if si.acc == exportAcc && si.from == subject {
  2741  			break
  2742  		}
  2743  	}
  2744  
  2745  	if si == nil || si.invalid {
  2746  		a.mu.RUnlock()
  2747  		return
  2748  	}
  2749  	a.mu.RUnlock()
  2750  
  2751  	if si.acc.checkActivation(a, si.claim, nil, false) {
  2752  		// The token has been updated most likely and we are good to go.
  2753  		return
  2754  	}
  2755  
  2756  	a.mu.Lock()
  2757  	si.invalid = true
  2758  	clients := a.getClientsLocked()
  2759  	awcsti := map[string]struct{}{a.Name: {}}
  2760  	a.mu.Unlock()
  2761  	for _, c := range clients {
  2762  		c.processSubsOnConfigReload(awcsti)
  2763  	}
  2764  }
  2765  
  2766  // These are import service specific versions for when an activation expires.
  2767  func (a *Account) serviceActivationExpired(subject string) {
  2768  	a.mu.RLock()
  2769  	if a.expired || a.imports.services == nil {
  2770  		a.mu.RUnlock()
  2771  		return
  2772  	}
  2773  	si := a.imports.services[subject]
  2774  	if si == nil || si.invalid {
  2775  		a.mu.RUnlock()
  2776  		return
  2777  	}
  2778  	a.mu.RUnlock()
  2779  
  2780  	if si.acc.checkActivation(a, si.claim, nil, false) {
  2781  		// The token has been updated most likely and we are good to go.
  2782  		return
  2783  	}
  2784  
  2785  	a.mu.Lock()
  2786  	si.invalid = true
  2787  	a.mu.Unlock()
  2788  }
  2789  
  2790  // Fires for expired activation tokens. We could track this with timers etc.
  2791  // Instead we just re-analyze where we are and if we need to act.
  2792  func (a *Account) activationExpired(exportAcc *Account, subject string, kind jwt.ExportType) {
  2793  	switch kind {
  2794  	case jwt.Stream:
  2795  		a.streamActivationExpired(exportAcc, subject)
  2796  	case jwt.Service:
  2797  		a.serviceActivationExpired(subject)
  2798  	}
  2799  }
  2800  
  2801  func isRevoked(revocations map[string]int64, subject string, issuedAt int64) bool {
  2802  	if len(revocations) == 0 {
  2803  		return false
  2804  	}
  2805  	if t, ok := revocations[subject]; !ok || t < issuedAt {
  2806  		if t, ok := revocations[jwt.All]; !ok || t < issuedAt {
  2807  			return false
  2808  		}
  2809  	}
  2810  	return true
  2811  }
  2812  
  2813  // checkActivation will check the activation token for validity.
  2814  // ea may only be nil in cases where revocation may not be checked, say triggered by expiration timer.
  2815  func (a *Account) checkActivation(importAcc *Account, claim *jwt.Import, ea *exportAuth, expTimer bool) bool {
  2816  	if claim == nil || claim.Token == _EMPTY_ {
  2817  		return false
  2818  	}
  2819  	// Create a quick clone so we can inline Token JWT.
  2820  	clone := *claim
  2821  
  2822  	vr := jwt.CreateValidationResults()
  2823  	clone.Validate(importAcc.Name, vr)
  2824  	if vr.IsBlocking(true) {
  2825  		return false
  2826  	}
  2827  	act, err := jwt.DecodeActivationClaims(clone.Token)
  2828  	if err != nil {
  2829  		return false
  2830  	}
  2831  	if !a.isIssuerClaimTrusted(act) {
  2832  		return false
  2833  	}
  2834  	vr = jwt.CreateValidationResults()
  2835  	act.Validate(vr)
  2836  	if vr.IsBlocking(true) {
  2837  		return false
  2838  	}
  2839  	if act.Expires != 0 {
  2840  		tn := time.Now().Unix()
  2841  		if act.Expires <= tn {
  2842  			return false
  2843  		}
  2844  		if expTimer {
  2845  			expiresAt := time.Duration(act.Expires - tn)
  2846  			time.AfterFunc(expiresAt*time.Second, func() {
  2847  				importAcc.activationExpired(a, string(act.ImportSubject), claim.Type)
  2848  			})
  2849  		}
  2850  	}
  2851  	if ea == nil {
  2852  		return true
  2853  	}
  2854  	// Check for token revocation..
  2855  	return !isRevoked(ea.actsRevoked, act.Subject, act.IssuedAt)
  2856  }
  2857  
  2858  // Returns true if the activation claim is trusted. That is the issuer matches
  2859  // the account or is an entry in the signing keys.
  2860  func (a *Account) isIssuerClaimTrusted(claims *jwt.ActivationClaims) bool {
  2861  	// if no issuer account, issuer is the account
  2862  	if claims.IssuerAccount == _EMPTY_ {
  2863  		return true
  2864  	}
  2865  	// If the IssuerAccount is not us, then this is considered an error.
  2866  	if a.Name != claims.IssuerAccount {
  2867  		if a.srv != nil {
  2868  			a.srv.Errorf("Invalid issuer account %q in activation claim (subject: %q - type: %q) for account %q",
  2869  				claims.IssuerAccount, claims.Activation.ImportSubject, claims.Activation.ImportType, a.Name)
  2870  		}
  2871  		return false
  2872  	}
  2873  	_, ok := a.hasIssuerNoLock(claims.Issuer)
  2874  	return ok
  2875  }
  2876  
  2877  // Returns true if `a` and `b` stream imports are the same. Note that the
  2878  // check is done with the account's name, not the pointer. This is used
  2879  // during config reload where we are comparing current and new config
  2880  // in which pointers are different.
  2881  // No lock is acquired in this function, so it is assumed that the
  2882  // import maps are not changed while this executes.
  2883  func (a *Account) checkStreamImportsEqual(b *Account) bool {
  2884  	if len(a.imports.streams) != len(b.imports.streams) {
  2885  		return false
  2886  	}
  2887  	// Load the b imports into a map index by what we are looking for.
  2888  	bm := make(map[string]*streamImport, len(b.imports.streams))
  2889  	for _, bim := range b.imports.streams {
  2890  		bm[bim.acc.Name+bim.from+bim.to] = bim
  2891  	}
  2892  	for _, aim := range a.imports.streams {
  2893  		if bim, ok := bm[aim.acc.Name+aim.from+aim.to]; !ok {
  2894  			return false
  2895  		} else if aim.atrc != bim.atrc {
  2896  			return false
  2897  		}
  2898  	}
  2899  	return true
  2900  }
  2901  
  2902  // Returns true if `a` and `b` stream exports are the same.
  2903  // Acquires `a` read lock, but `b` is assumed to not be accessed
  2904  // by anyone but the caller (`b` is not registered anywhere).
  2905  func (a *Account) checkStreamExportsEqual(b *Account) bool {
  2906  	a.mu.RLock()
  2907  	defer a.mu.RUnlock()
  2908  	if len(a.exports.streams) != len(b.exports.streams) {
  2909  		return false
  2910  	}
  2911  	for subj, aea := range a.exports.streams {
  2912  		bea, ok := b.exports.streams[subj]
  2913  		if !ok {
  2914  			return false
  2915  		}
  2916  		if !isStreamExportEqual(aea, bea) {
  2917  			return false
  2918  		}
  2919  	}
  2920  	return true
  2921  }
  2922  
  2923  func isStreamExportEqual(a, b *streamExport) bool {
  2924  	if a == nil && b == nil {
  2925  		return true
  2926  	}
  2927  	if (a == nil && b != nil) || (a != nil && b == nil) {
  2928  		return false
  2929  	}
  2930  	return isExportAuthEqual(&a.exportAuth, &b.exportAuth)
  2931  }
  2932  
  2933  // Returns true if `a` and `b` service exports are the same.
  2934  // Acquires `a` read lock, but `b` is assumed to not be accessed
  2935  // by anyone but the caller (`b` is not registered anywhere).
  2936  func (a *Account) checkServiceExportsEqual(b *Account) bool {
  2937  	a.mu.RLock()
  2938  	defer a.mu.RUnlock()
  2939  	if len(a.exports.services) != len(b.exports.services) {
  2940  		return false
  2941  	}
  2942  	for subj, aea := range a.exports.services {
  2943  		bea, ok := b.exports.services[subj]
  2944  		if !ok {
  2945  			return false
  2946  		}
  2947  		if !isServiceExportEqual(aea, bea) {
  2948  			return false
  2949  		}
  2950  	}
  2951  	return true
  2952  }
  2953  
  2954  func isServiceExportEqual(a, b *serviceExport) bool {
  2955  	if a == nil && b == nil {
  2956  		return true
  2957  	}
  2958  	if (a == nil && b != nil) || (a != nil && b == nil) {
  2959  		return false
  2960  	}
  2961  	if !isExportAuthEqual(&a.exportAuth, &b.exportAuth) {
  2962  		return false
  2963  	}
  2964  	if a.acc.Name != b.acc.Name {
  2965  		return false
  2966  	}
  2967  	if a.respType != b.respType {
  2968  		return false
  2969  	}
  2970  	if a.latency != nil || b.latency != nil {
  2971  		if (a.latency != nil && b.latency == nil) || (a.latency == nil && b.latency != nil) {
  2972  			return false
  2973  		}
  2974  		if a.latency.sampling != b.latency.sampling {
  2975  			return false
  2976  		}
  2977  		if a.latency.subject != b.latency.subject {
  2978  			return false
  2979  		}
  2980  	}
  2981  	if a.atrc != b.atrc {
  2982  		return false
  2983  	}
  2984  	return true
  2985  }
  2986  
  2987  // Returns true if `a` and `b` exportAuth structures are equal.
  2988  // Both `a` and `b` are guaranteed to be non-nil.
  2989  // Locking is handled by the caller.
  2990  func isExportAuthEqual(a, b *exportAuth) bool {
  2991  	if a.tokenReq != b.tokenReq {
  2992  		return false
  2993  	}
  2994  	if a.accountPos != b.accountPos {
  2995  		return false
  2996  	}
  2997  	if len(a.approved) != len(b.approved) {
  2998  		return false
  2999  	}
  3000  	for ak, av := range a.approved {
  3001  		if bv, ok := b.approved[ak]; !ok || av.Name != bv.Name {
  3002  			return false
  3003  		}
  3004  	}
  3005  	if len(a.actsRevoked) != len(b.actsRevoked) {
  3006  		return false
  3007  	}
  3008  	for ak, av := range a.actsRevoked {
  3009  		if bv, ok := b.actsRevoked[ak]; !ok || av != bv {
  3010  			return false
  3011  		}
  3012  	}
  3013  	return true
  3014  }
  3015  
  3016  // Check if another account is authorized to route requests to this service.
  3017  func (a *Account) checkServiceImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool {
  3018  	a.mu.RLock()
  3019  	authorized := a.checkServiceImportAuthorizedNoLock(account, subject, imClaim)
  3020  	a.mu.RUnlock()
  3021  	return authorized
  3022  }
  3023  
  3024  // Check if another account is authorized to route requests to this service.
  3025  func (a *Account) checkServiceImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool {
  3026  	// Find the subject in the services list.
  3027  	if a.exports.services == nil {
  3028  		return false
  3029  	}
  3030  	return a.checkServiceExportApproved(account, subject, imClaim)
  3031  }
  3032  
  3033  // IsExpired returns expiration status.
  3034  func (a *Account) IsExpired() bool {
  3035  	a.mu.RLock()
  3036  	exp := a.expired
  3037  	a.mu.RUnlock()
  3038  	return exp
  3039  }
  3040  
  3041  // Called when an account has expired.
  3042  func (a *Account) expiredTimeout() {
  3043  	// Mark expired first.
  3044  	a.mu.Lock()
  3045  	a.expired = true
  3046  	a.mu.Unlock()
  3047  
  3048  	// Collect the clients and expire them.
  3049  	cs := a.getClients()
  3050  	for _, c := range cs {
  3051  		c.accountAuthExpired()
  3052  	}
  3053  }
  3054  
  3055  // Sets the expiration timer for an account JWT that has it set.
  3056  func (a *Account) setExpirationTimer(d time.Duration) {
  3057  	a.etmr = time.AfterFunc(d, a.expiredTimeout)
  3058  }
  3059  
  3060  // Lock should be held
  3061  func (a *Account) clearExpirationTimer() bool {
  3062  	if a.etmr == nil {
  3063  		return true
  3064  	}
  3065  	stopped := a.etmr.Stop()
  3066  	a.etmr = nil
  3067  	return stopped
  3068  }
  3069  
  3070  // checkUserRevoked will check if a user has been revoked.
  3071  func (a *Account) checkUserRevoked(nkey string, issuedAt int64) bool {
  3072  	a.mu.RLock()
  3073  	defer a.mu.RUnlock()
  3074  	return isRevoked(a.usersRevoked, nkey, issuedAt)
  3075  }
  3076  
  3077  // failBearer will return if bearer token are allowed (false) or disallowed (true)
  3078  func (a *Account) failBearer() bool {
  3079  	a.mu.RLock()
  3080  	defer a.mu.RUnlock()
  3081  	return a.disallowBearer
  3082  }
  3083  
  3084  // Check expiration and set the proper state as needed.
  3085  func (a *Account) checkExpiration(claims *jwt.ClaimsData) {
  3086  	a.mu.Lock()
  3087  	defer a.mu.Unlock()
  3088  
  3089  	a.clearExpirationTimer()
  3090  	if claims.Expires == 0 {
  3091  		a.expired = false
  3092  		return
  3093  	}
  3094  	tn := time.Now().Unix()
  3095  	if claims.Expires <= tn {
  3096  		a.expired = true
  3097  		return
  3098  	}
  3099  	expiresAt := time.Duration(claims.Expires - tn)
  3100  	a.setExpirationTimer(expiresAt * time.Second)
  3101  	a.expired = false
  3102  }
  3103  
  3104  // hasIssuer returns true if the issuer matches the account
  3105  // If the issuer is a scoped signing key, the scope will be returned as well
  3106  // issuer or it is a signing key for the account.
  3107  func (a *Account) hasIssuer(issuer string) (jwt.Scope, bool) {
  3108  	a.mu.RLock()
  3109  	scope, ok := a.hasIssuerNoLock(issuer)
  3110  	a.mu.RUnlock()
  3111  	return scope, ok
  3112  }
  3113  
  3114  // hasIssuerNoLock is the unlocked version of hasIssuer
  3115  func (a *Account) hasIssuerNoLock(issuer string) (jwt.Scope, bool) {
  3116  	scope, ok := a.signingKeys[issuer]
  3117  	return scope, ok
  3118  }
  3119  
  3120  // Returns the loop detection subject used for leafnodes
  3121  func (a *Account) getLDSubject() string {
  3122  	a.mu.RLock()
  3123  	lds := a.lds
  3124  	a.mu.RUnlock()
  3125  	return lds
  3126  }
  3127  
  3128  // Placeholder for signaling token auth required.
  3129  var tokenAuthReq = []*Account{}
  3130  
  3131  func authAccounts(tokenReq bool) []*Account {
  3132  	if tokenReq {
  3133  		return tokenAuthReq
  3134  	}
  3135  	return nil
  3136  }
  3137  
  3138  // SetAccountResolver will assign the account resolver.
  3139  func (s *Server) SetAccountResolver(ar AccountResolver) {
  3140  	s.mu.Lock()
  3141  	s.accResolver = ar
  3142  	s.mu.Unlock()
  3143  }
  3144  
  3145  // AccountResolver returns the registered account resolver.
  3146  func (s *Server) AccountResolver() AccountResolver {
  3147  	s.mu.Lock()
  3148  	ar := s.accResolver
  3149  	s.mu.Unlock()
  3150  	return ar
  3151  }
  3152  
  3153  // isClaimAccount returns if this account is backed by a JWT claim.
  3154  // Lock should be held.
  3155  func (a *Account) isClaimAccount() bool {
  3156  	return a.claimJWT != _EMPTY_
  3157  }
  3158  
  3159  // UpdateAccountClaims will update an existing account with new claims.
  3160  // This will replace any exports or imports previously defined.
  3161  // Lock MUST NOT be held upon entry.
  3162  func (s *Server) UpdateAccountClaims(a *Account, ac *jwt.AccountClaims) {
  3163  	s.updateAccountClaimsWithRefresh(a, ac, true)
  3164  }
  3165  
  3166  func (a *Account) traceLabel() string {
  3167  	if a == nil {
  3168  		return _EMPTY_
  3169  	}
  3170  	if a.nameTag != _EMPTY_ {
  3171  		return fmt.Sprintf("%s/%s", a.Name, a.nameTag)
  3172  	}
  3173  	return a.Name
  3174  }
  3175  
  3176  // Check if an account has external auth set.
  3177  // Operator/Account Resolver only.
  3178  func (a *Account) hasExternalAuth() bool {
  3179  	if a == nil {
  3180  		return false
  3181  	}
  3182  	a.mu.RLock()
  3183  	defer a.mu.RUnlock()
  3184  	return a.extAuth != nil
  3185  }
  3186  
  3187  // Deterimine if this is an external auth user.
  3188  func (a *Account) isExternalAuthUser(userID string) bool {
  3189  	if a == nil {
  3190  		return false
  3191  	}
  3192  	a.mu.RLock()
  3193  	defer a.mu.RUnlock()
  3194  	if a.extAuth != nil {
  3195  		for _, u := range a.extAuth.AuthUsers {
  3196  			if userID == u {
  3197  				return true
  3198  			}
  3199  		}
  3200  	}
  3201  	return false
  3202  }
  3203  
  3204  // Return the external authorization xkey if external authorization is enabled and the xkey is set.
  3205  // Operator/Account Resolver only.
  3206  func (a *Account) externalAuthXKey() string {
  3207  	if a == nil {
  3208  		return _EMPTY_
  3209  	}
  3210  	a.mu.RLock()
  3211  	defer a.mu.RUnlock()
  3212  	if a.extAuth != nil && a.extAuth.XKey != _EMPTY_ {
  3213  		return a.extAuth.XKey
  3214  	}
  3215  	return _EMPTY_
  3216  }
  3217  
  3218  // Check if an account switch for external authorization is allowed.
  3219  func (a *Account) isAllowedAcount(acc string) bool {
  3220  	if a == nil {
  3221  		return false
  3222  	}
  3223  	a.mu.RLock()
  3224  	defer a.mu.RUnlock()
  3225  	if a.extAuth != nil {
  3226  		// if we have a single allowed account, and we have a wildcard
  3227  		// we accept it
  3228  		if len(a.extAuth.AllowedAccounts) == 1 &&
  3229  			a.extAuth.AllowedAccounts[0] == jwt.AnyAccount {
  3230  			return true
  3231  		}
  3232  		// otherwise must match exactly
  3233  		for _, a := range a.extAuth.AllowedAccounts {
  3234  			if a == acc {
  3235  				return true
  3236  			}
  3237  		}
  3238  	}
  3239  	return false
  3240  }
  3241  
  3242  // updateAccountClaimsWithRefresh will update an existing account with new claims.
  3243  // If refreshImportingAccounts is true it will also update incomplete dependent accounts
  3244  // This will replace any exports or imports previously defined.
  3245  // Lock MUST NOT be held upon entry.
  3246  func (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaims, refreshImportingAccounts bool) {
  3247  	if a == nil {
  3248  		return
  3249  	}
  3250  	s.Debugf("Updating account claims: %s/%s", a.Name, ac.Name)
  3251  	a.checkExpiration(ac.Claims())
  3252  
  3253  	a.mu.Lock()
  3254  	// Clone to update, only select certain fields.
  3255  	old := &Account{Name: a.Name, exports: a.exports, limits: a.limits, signingKeys: a.signingKeys}
  3256  
  3257  	// overwrite claim meta data
  3258  	a.nameTag = ac.Name
  3259  	a.tags = ac.Tags
  3260  
  3261  	var td string
  3262  	var tds int
  3263  	if ac.Trace != nil {
  3264  		// Update trace destination and sampling
  3265  		td, tds = string(ac.Trace.Destination), ac.Trace.Sampling
  3266  		if !IsValidPublishSubject(td) {
  3267  			td, tds = _EMPTY_, 0
  3268  		} else if tds <= 0 || tds > 100 {
  3269  			tds = 100
  3270  		}
  3271  	}
  3272  	a.traceDest, a.traceDestSampling = td, tds
  3273  
  3274  	// Check for external authorization.
  3275  	if ac.HasExternalAuthorization() {
  3276  		a.extAuth = &jwt.ExternalAuthorization{}
  3277  		a.extAuth.AuthUsers.Add(ac.Authorization.AuthUsers...)
  3278  		a.extAuth.AllowedAccounts.Add(ac.Authorization.AllowedAccounts...)
  3279  		a.extAuth.XKey = ac.Authorization.XKey
  3280  	}
  3281  
  3282  	// Reset exports and imports here.
  3283  
  3284  	// Exports is creating a whole new map.
  3285  	a.exports = exportMap{}
  3286  
  3287  	// Imports are checked unlocked in processInbound, so we can't change out the struct here. Need to process inline.
  3288  	if a.imports.streams != nil {
  3289  		old.imports.streams = a.imports.streams
  3290  		a.imports.streams = nil
  3291  	}
  3292  	if a.imports.services != nil {
  3293  		old.imports.services = make(map[string]*serviceImport, len(a.imports.services))
  3294  	}
  3295  	for k, v := range a.imports.services {
  3296  		old.imports.services[k] = v
  3297  		delete(a.imports.services, k)
  3298  	}
  3299  
  3300  	alteredScope := map[string]struct{}{}
  3301  
  3302  	// update account signing keys
  3303  	a.signingKeys = nil
  3304  	_, strict := s.strictSigningKeyUsage[a.Issuer]
  3305  	if len(ac.SigningKeys) > 0 || !strict {
  3306  		a.signingKeys = make(map[string]jwt.Scope)
  3307  	}
  3308  	signersChanged := false
  3309  	for k, scope := range ac.SigningKeys {
  3310  		a.signingKeys[k] = scope
  3311  	}
  3312  	if !strict {
  3313  		a.signingKeys[a.Name] = nil
  3314  	}
  3315  	if len(a.signingKeys) != len(old.signingKeys) {
  3316  		signersChanged = true
  3317  	}
  3318  	for k, scope := range a.signingKeys {
  3319  		if oldScope, ok := old.signingKeys[k]; !ok {
  3320  			signersChanged = true
  3321  		} else if !reflect.DeepEqual(scope, oldScope) {
  3322  			signersChanged = true
  3323  			alteredScope[k] = struct{}{}
  3324  		}
  3325  	}
  3326  	// collect mappings that need to be removed
  3327  	removeList := []string{}
  3328  	for _, m := range a.mappings {
  3329  		if _, ok := ac.Mappings[jwt.Subject(m.src)]; !ok {
  3330  			removeList = append(removeList, m.src)
  3331  		}
  3332  	}
  3333  	a.mu.Unlock()
  3334  
  3335  	for sub, wm := range ac.Mappings {
  3336  		mappings := make([]*MapDest, len(wm))
  3337  		for i, m := range wm {
  3338  			mappings[i] = &MapDest{
  3339  				Subject: string(m.Subject),
  3340  				Weight:  m.GetWeight(),
  3341  				Cluster: m.Cluster,
  3342  			}
  3343  		}
  3344  		// This will overwrite existing entries
  3345  		a.AddWeightedMappings(string(sub), mappings...)
  3346  	}
  3347  	// remove mappings
  3348  	for _, rmMapping := range removeList {
  3349  		a.RemoveMapping(rmMapping)
  3350  	}
  3351  
  3352  	// Re-register system exports/imports.
  3353  	if a == s.SystemAccount() {
  3354  		s.addSystemAccountExports(a)
  3355  	} else {
  3356  		s.registerSystemImports(a)
  3357  	}
  3358  
  3359  	jsEnabled := s.JetStreamEnabled()
  3360  
  3361  	streamTokenExpirationChanged := false
  3362  	serviceTokenExpirationChanged := false
  3363  
  3364  	for _, e := range ac.Exports {
  3365  		switch e.Type {
  3366  		case jwt.Stream:
  3367  			s.Debugf("Adding stream export %q for %s", e.Subject, a.traceLabel())
  3368  			if err := a.addStreamExportWithAccountPos(
  3369  				string(e.Subject), authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil {
  3370  				s.Debugf("Error adding stream export to account [%s]: %v", a.traceLabel(), err.Error())
  3371  			}
  3372  		case jwt.Service:
  3373  			s.Debugf("Adding service export %q for %s", e.Subject, a.traceLabel())
  3374  			rt := Singleton
  3375  			switch e.ResponseType {
  3376  			case jwt.ResponseTypeStream:
  3377  				rt = Streamed
  3378  			case jwt.ResponseTypeChunked:
  3379  				rt = Chunked
  3380  			}
  3381  			if err := a.addServiceExportWithResponseAndAccountPos(
  3382  				string(e.Subject), rt, authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil {
  3383  				s.Debugf("Error adding service export to account [%s]: %v", a.traceLabel(), err)
  3384  				continue
  3385  			}
  3386  			sub := string(e.Subject)
  3387  			if e.Latency != nil {
  3388  				if err := a.TrackServiceExportWithSampling(sub, string(e.Latency.Results), int(e.Latency.Sampling)); err != nil {
  3389  					hdrNote := _EMPTY_
  3390  					if e.Latency.Sampling == jwt.Headers {
  3391  						hdrNote = " (using headers)"
  3392  					}
  3393  					s.Debugf("Error adding latency tracking%s for service export to account [%s]: %v", hdrNote, a.traceLabel(), err)
  3394  				}
  3395  			}
  3396  			if e.ResponseThreshold != 0 {
  3397  				// Response threshold was set in options.
  3398  				if err := a.SetServiceExportResponseThreshold(sub, e.ResponseThreshold); err != nil {
  3399  					s.Debugf("Error adding service export response threshold for [%s]: %v", a.traceLabel(), err)
  3400  				}
  3401  			}
  3402  			if err := a.SetServiceExportAllowTrace(sub, e.AllowTrace); err != nil {
  3403  				s.Debugf("Error adding allow_trace for %q: %v", sub, err)
  3404  			}
  3405  		}
  3406  
  3407  		var revocationChanged *bool
  3408  		var ea *exportAuth
  3409  
  3410  		a.mu.Lock()
  3411  		switch e.Type {
  3412  		case jwt.Stream:
  3413  			revocationChanged = &streamTokenExpirationChanged
  3414  			if se, ok := a.exports.streams[string(e.Subject)]; ok && se != nil {
  3415  				ea = &se.exportAuth
  3416  			}
  3417  		case jwt.Service:
  3418  			revocationChanged = &serviceTokenExpirationChanged
  3419  			if se, ok := a.exports.services[string(e.Subject)]; ok && se != nil {
  3420  				ea = &se.exportAuth
  3421  			}
  3422  		}
  3423  		if ea != nil {
  3424  			oldRevocations := ea.actsRevoked
  3425  			if len(e.Revocations) == 0 {
  3426  				// remove all, no need to evaluate existing imports
  3427  				ea.actsRevoked = nil
  3428  			} else if len(oldRevocations) == 0 {
  3429  				// add all, existing imports need to be re evaluated
  3430  				ea.actsRevoked = e.Revocations
  3431  				*revocationChanged = true
  3432  			} else {
  3433  				ea.actsRevoked = e.Revocations
  3434  				// diff, existing imports need to be conditionally re evaluated, depending on:
  3435  				// if a key was added, or it's timestamp increased
  3436  				for k, t := range e.Revocations {
  3437  					if tOld, ok := oldRevocations[k]; !ok || tOld < t {
  3438  						*revocationChanged = true
  3439  					}
  3440  				}
  3441  			}
  3442  		}
  3443  		a.mu.Unlock()
  3444  	}
  3445  	var incompleteImports []*jwt.Import
  3446  	for _, i := range ac.Imports {
  3447  		// check tmpAccounts with priority
  3448  		var acc *Account
  3449  		var err error
  3450  		if v, ok := s.tmpAccounts.Load(i.Account); ok {
  3451  			acc = v.(*Account)
  3452  		} else {
  3453  			acc, err = s.lookupAccount(i.Account)
  3454  		}
  3455  		if acc == nil || err != nil {
  3456  			s.Errorf("Can't locate account [%s] for import of [%v] %s (err=%v)", i.Account, i.Subject, i.Type, err)
  3457  			incompleteImports = append(incompleteImports, i)
  3458  			continue
  3459  		}
  3460  		from := string(i.Subject)
  3461  		to := i.GetTo()
  3462  		switch i.Type {
  3463  		case jwt.Stream:
  3464  			if i.LocalSubject != _EMPTY_ {
  3465  				// set local subject implies to is empty
  3466  				to = string(i.LocalSubject)
  3467  				s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to)
  3468  				err = a.AddMappedStreamImportWithClaim(acc, from, to, i)
  3469  			} else {
  3470  				s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to)
  3471  				err = a.AddStreamImportWithClaim(acc, from, to, i)
  3472  			}
  3473  			if err != nil {
  3474  				s.Debugf("Error adding stream import to account [%s]: %v", a.traceLabel(), err.Error())
  3475  				incompleteImports = append(incompleteImports, i)
  3476  			}
  3477  		case jwt.Service:
  3478  			if i.LocalSubject != _EMPTY_ {
  3479  				from = string(i.LocalSubject)
  3480  				to = string(i.Subject)
  3481  			}
  3482  			s.Debugf("Adding service import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to)
  3483  			if err := a.AddServiceImportWithClaim(acc, from, to, i); err != nil {
  3484  				s.Debugf("Error adding service import to account [%s]: %v", a.traceLabel(), err.Error())
  3485  				incompleteImports = append(incompleteImports, i)
  3486  			}
  3487  		}
  3488  	}
  3489  	// Now let's apply any needed changes from import/export changes.
  3490  	if !a.checkStreamImportsEqual(old) {
  3491  		awcsti := map[string]struct{}{a.Name: {}}
  3492  		for _, c := range a.getClients() {
  3493  			c.processSubsOnConfigReload(awcsti)
  3494  		}
  3495  	}
  3496  	// Now check if stream exports have changed.
  3497  	if !a.checkStreamExportsEqual(old) || signersChanged || streamTokenExpirationChanged {
  3498  		clients := map[*client]struct{}{}
  3499  		// We need to check all accounts that have an import claim from this account.
  3500  		awcsti := map[string]struct{}{}
  3501  		s.accounts.Range(func(k, v any) bool {
  3502  			acc := v.(*Account)
  3503  			// Move to the next if this account is actually account "a".
  3504  			if acc.Name == a.Name {
  3505  				return true
  3506  			}
  3507  			// TODO: checkStreamImportAuthorized() stack should not be trying
  3508  			// to lock "acc". If we find that to be needed, we will need to
  3509  			// rework this to ensure we don't lock acc.
  3510  			acc.mu.Lock()
  3511  			for _, im := range acc.imports.streams {
  3512  				if im != nil && im.acc.Name == a.Name {
  3513  					// Check for if we are still authorized for an import.
  3514  					im.invalid = !a.checkStreamImportAuthorized(acc, im.from, im.claim)
  3515  					awcsti[acc.Name] = struct{}{}
  3516  					for c := range acc.clients {
  3517  						clients[c] = struct{}{}
  3518  					}
  3519  				}
  3520  			}
  3521  			acc.mu.Unlock()
  3522  			return true
  3523  		})
  3524  		// Now walk clients.
  3525  		for c := range clients {
  3526  			c.processSubsOnConfigReload(awcsti)
  3527  		}
  3528  	}
  3529  	// Now check if service exports have changed.
  3530  	if !a.checkServiceExportsEqual(old) || signersChanged || serviceTokenExpirationChanged {
  3531  		s.accounts.Range(func(k, v any) bool {
  3532  			acc := v.(*Account)
  3533  			// Move to the next if this account is actually account "a".
  3534  			if acc.Name == a.Name {
  3535  				return true
  3536  			}
  3537  			// TODO: checkServiceImportAuthorized() stack should not be trying
  3538  			// to lock "acc". If we find that to be needed, we will need to
  3539  			// rework this to ensure we don't lock acc.
  3540  			acc.mu.Lock()
  3541  			for _, si := range acc.imports.services {
  3542  				if si != nil && si.acc.Name == a.Name {
  3543  					// Check for if we are still authorized for an import.
  3544  					si.invalid = !a.checkServiceImportAuthorized(acc, si.to, si.claim)
  3545  					// Make sure we should still be tracking latency and if we
  3546  					// are allowed to trace.
  3547  					if !si.response {
  3548  						if se := a.getServiceExport(si.to); se != nil {
  3549  							if si.latency != nil {
  3550  								si.latency = se.latency
  3551  							}
  3552  							// Update allow trace.
  3553  							si.atrc = se.atrc
  3554  						}
  3555  					}
  3556  				}
  3557  			}
  3558  			acc.mu.Unlock()
  3559  			return true
  3560  		})
  3561  	}
  3562  
  3563  	// Now make sure we shutdown the old service import subscriptions.
  3564  	var sids [][]byte
  3565  	a.mu.RLock()
  3566  	c := a.ic
  3567  	for _, si := range old.imports.services {
  3568  		if c != nil && si.sid != nil {
  3569  			sids = append(sids, si.sid)
  3570  		}
  3571  	}
  3572  	a.mu.RUnlock()
  3573  	for _, sid := range sids {
  3574  		c.processUnsub(sid)
  3575  	}
  3576  
  3577  	// Now do limits if they are present.
  3578  	a.mu.Lock()
  3579  	a.msubs = int32(ac.Limits.Subs)
  3580  	a.mpay = int32(ac.Limits.Payload)
  3581  	a.mconns = int32(ac.Limits.Conn)
  3582  	a.mleafs = int32(ac.Limits.LeafNodeConn)
  3583  	a.disallowBearer = ac.Limits.DisallowBearer
  3584  	// Check for any revocations
  3585  	if len(ac.Revocations) > 0 {
  3586  		// We will always replace whatever we had with most current, so no
  3587  		// need to look at what we have.
  3588  		a.usersRevoked = make(map[string]int64, len(ac.Revocations))
  3589  		for pk, t := range ac.Revocations {
  3590  			a.usersRevoked[pk] = t
  3591  		}
  3592  	} else {
  3593  		a.usersRevoked = nil
  3594  	}
  3595  	a.defaultPerms = buildPermissionsFromJwt(&ac.DefaultPermissions)
  3596  	a.incomplete = len(incompleteImports) != 0
  3597  	for _, i := range incompleteImports {
  3598  		s.incompleteAccExporterMap.Store(i.Account, struct{}{})
  3599  	}
  3600  	if a.srv == nil {
  3601  		a.srv = s
  3602  	}
  3603  
  3604  	if ac.Limits.IsJSEnabled() {
  3605  		toUnlimited := func(value int64) int64 {
  3606  			if value > 0 {
  3607  				return value
  3608  			}
  3609  			return -1
  3610  		}
  3611  		if ac.Limits.JetStreamLimits.DiskStorage != 0 || ac.Limits.JetStreamLimits.MemoryStorage != 0 {
  3612  			// JetStreamAccountLimits and jwt.JetStreamLimits use same value for unlimited
  3613  			a.jsLimits = map[string]JetStreamAccountLimits{
  3614  				_EMPTY_: {
  3615  					MaxMemory:            ac.Limits.JetStreamLimits.MemoryStorage,
  3616  					MaxStore:             ac.Limits.JetStreamLimits.DiskStorage,
  3617  					MaxStreams:           int(ac.Limits.JetStreamLimits.Streams),
  3618  					MaxConsumers:         int(ac.Limits.JetStreamLimits.Consumer),
  3619  					MemoryMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.MemoryMaxStreamBytes),
  3620  					StoreMaxStreamBytes:  toUnlimited(ac.Limits.JetStreamLimits.DiskMaxStreamBytes),
  3621  					MaxBytesRequired:     ac.Limits.JetStreamLimits.MaxBytesRequired,
  3622  					MaxAckPending:        int(toUnlimited(ac.Limits.JetStreamLimits.MaxAckPending)),
  3623  				},
  3624  			}
  3625  		} else {
  3626  			a.jsLimits = map[string]JetStreamAccountLimits{}
  3627  			for t, l := range ac.Limits.JetStreamTieredLimits {
  3628  				a.jsLimits[t] = JetStreamAccountLimits{
  3629  					MaxMemory:            l.MemoryStorage,
  3630  					MaxStore:             l.DiskStorage,
  3631  					MaxStreams:           int(l.Streams),
  3632  					MaxConsumers:         int(l.Consumer),
  3633  					MemoryMaxStreamBytes: toUnlimited(l.MemoryMaxStreamBytes),
  3634  					StoreMaxStreamBytes:  toUnlimited(l.DiskMaxStreamBytes),
  3635  					MaxBytesRequired:     l.MaxBytesRequired,
  3636  					MaxAckPending:        int(toUnlimited(l.MaxAckPending)),
  3637  				}
  3638  			}
  3639  		}
  3640  	} else if a.jsLimits != nil {
  3641  		// covers failed update followed by disable
  3642  		a.jsLimits = nil
  3643  	}
  3644  
  3645  	a.updated = time.Now()
  3646  	clients := a.getClientsLocked()
  3647  	a.mu.Unlock()
  3648  
  3649  	// Sort if we are over the limit.
  3650  	if a.MaxTotalConnectionsReached() {
  3651  		sort.Slice(clients, func(i, j int) bool {
  3652  			return clients[i].start.After(clients[j].start)
  3653  		})
  3654  	}
  3655  
  3656  	// If JetStream is enabled for this server we will call into configJetStream for the account
  3657  	// regardless of enabled or disabled. It handles both cases.
  3658  	if jsEnabled {
  3659  		if err := s.configJetStream(a); err != nil {
  3660  			s.Errorf("Error configuring jetstream for account [%s]: %v", a.traceLabel(), err.Error())
  3661  			a.mu.Lock()
  3662  			// Absent reload of js server cfg, this is going to be broken until js is disabled
  3663  			a.incomplete = true
  3664  			a.mu.Unlock()
  3665  		}
  3666  	} else if a.jsLimits != nil {
  3667  		// We do not have JS enabled for this server, but the account has it enabled so setup
  3668  		// our imports properly. This allows this server to proxy JS traffic correctly.
  3669  		s.checkJetStreamExports()
  3670  		a.enableAllJetStreamServiceImportsAndMappings()
  3671  	}
  3672  
  3673  	for i, c := range clients {
  3674  		a.mu.RLock()
  3675  		exceeded := a.mconns != jwt.NoLimit && i >= int(a.mconns)
  3676  		a.mu.RUnlock()
  3677  		if exceeded {
  3678  			c.maxAccountConnExceeded()
  3679  			continue
  3680  		}
  3681  		c.mu.Lock()
  3682  		c.applyAccountLimits()
  3683  		theJWT := c.opts.JWT
  3684  		c.mu.Unlock()
  3685  		// Check for being revoked here. We use ac one to avoid the account lock.
  3686  		if (ac.Revocations != nil || ac.Limits.DisallowBearer) && theJWT != _EMPTY_ {
  3687  			if juc, err := jwt.DecodeUserClaims(theJWT); err != nil {
  3688  				c.Debugf("User JWT not valid: %v", err)
  3689  				c.authViolation()
  3690  				continue
  3691  			} else if juc.BearerToken && ac.Limits.DisallowBearer {
  3692  				c.Debugf("Bearer User JWT not allowed")
  3693  				c.authViolation()
  3694  				continue
  3695  			} else if ok := ac.IsClaimRevoked(juc); ok {
  3696  				c.sendErrAndDebug("User Authentication Revoked")
  3697  				c.closeConnection(Revocation)
  3698  				continue
  3699  			}
  3700  		}
  3701  	}
  3702  
  3703  	// Check if the signing keys changed, might have to evict
  3704  	if signersChanged {
  3705  		for _, c := range clients {
  3706  			c.mu.Lock()
  3707  			if c.user == nil {
  3708  				c.mu.Unlock()
  3709  				continue
  3710  			}
  3711  			sk := c.user.SigningKey
  3712  			c.mu.Unlock()
  3713  			if sk == _EMPTY_ {
  3714  				continue
  3715  			}
  3716  			if _, ok := alteredScope[sk]; ok {
  3717  				c.closeConnection(AuthenticationViolation)
  3718  			} else if _, ok := a.hasIssuer(sk); !ok {
  3719  				c.closeConnection(AuthenticationViolation)
  3720  			}
  3721  		}
  3722  	}
  3723  
  3724  	if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok && refreshImportingAccounts {
  3725  		s.incompleteAccExporterMap.Delete(old.Name)
  3726  		s.accounts.Range(func(key, value any) bool {
  3727  			acc := value.(*Account)
  3728  			acc.mu.RLock()
  3729  			incomplete := acc.incomplete
  3730  			name := acc.Name
  3731  			label := acc.traceLabel()
  3732  			// Must use jwt in account or risk failing on fetch
  3733  			// This jwt may not be the same that caused exportingAcc to be in incompleteAccExporterMap
  3734  			claimJWT := acc.claimJWT
  3735  			acc.mu.RUnlock()
  3736  			if incomplete && name != old.Name {
  3737  				if accClaims, _, err := s.verifyAccountClaims(claimJWT); err == nil {
  3738  					// Since claimJWT has not changed, acc can become complete
  3739  					// but it won't alter incomplete for it's dependents accounts.
  3740  					s.updateAccountClaimsWithRefresh(acc, accClaims, false)
  3741  					// old.Name was deleted before ranging over accounts
  3742  					// If it exists again, UpdateAccountClaims set it for failed imports of acc.
  3743  					// So there was one import of acc that imported this account and failed again.
  3744  					// Since this account just got updated, the import itself may be in error. So trace that.
  3745  					if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok {
  3746  						s.incompleteAccExporterMap.Delete(old.Name)
  3747  						s.Errorf("Account %s has issues importing account %s", label, old.Name)
  3748  					}
  3749  				}
  3750  			}
  3751  			return true
  3752  		})
  3753  	}
  3754  }
  3755  
  3756  // Helper to build an internal account structure from a jwt.AccountClaims.
  3757  // Lock MUST NOT be held upon entry.
  3758  func (s *Server) buildInternalAccount(ac *jwt.AccountClaims) *Account {
  3759  	acc := NewAccount(ac.Subject)
  3760  	acc.Issuer = ac.Issuer
  3761  	// Override subject with logical name.
  3762  	acc.LogicalName = ac.Name
  3763  	// Set this here since we are placing in s.tmpAccounts below and may be
  3764  	// referenced by an route RS+, etc.
  3765  	s.setAccountSublist(acc)
  3766  
  3767  	// We don't want to register an account that is in the process of
  3768  	// being built, however, to solve circular import dependencies, we
  3769  	// need to store it here.
  3770  	s.tmpAccounts.Store(ac.Subject, acc)
  3771  	s.UpdateAccountClaims(acc, ac)
  3772  	return acc
  3773  }
  3774  
  3775  // Helper to build Permissions from jwt.Permissions
  3776  // or return nil if none were specified
  3777  func buildPermissionsFromJwt(uc *jwt.Permissions) *Permissions {
  3778  	if uc == nil {
  3779  		return nil
  3780  	}
  3781  	var p *Permissions
  3782  	if len(uc.Pub.Allow) > 0 || len(uc.Pub.Deny) > 0 {
  3783  		p = &Permissions{}
  3784  		p.Publish = &SubjectPermission{}
  3785  		p.Publish.Allow = uc.Pub.Allow
  3786  		p.Publish.Deny = uc.Pub.Deny
  3787  	}
  3788  	if len(uc.Sub.Allow) > 0 || len(uc.Sub.Deny) > 0 {
  3789  		if p == nil {
  3790  			p = &Permissions{}
  3791  		}
  3792  		p.Subscribe = &SubjectPermission{}
  3793  		p.Subscribe.Allow = uc.Sub.Allow
  3794  		p.Subscribe.Deny = uc.Sub.Deny
  3795  	}
  3796  	if uc.Resp != nil {
  3797  		if p == nil {
  3798  			p = &Permissions{}
  3799  		}
  3800  		p.Response = &ResponsePermission{
  3801  			MaxMsgs: uc.Resp.MaxMsgs,
  3802  			Expires: uc.Resp.Expires,
  3803  		}
  3804  		validateResponsePermissions(p)
  3805  	}
  3806  	return p
  3807  }
  3808  
  3809  // Helper to build internal NKeyUser.
  3810  func buildInternalNkeyUser(uc *jwt.UserClaims, acts map[string]struct{}, acc *Account) *NkeyUser {
  3811  	nu := &NkeyUser{Nkey: uc.Subject, Account: acc, AllowedConnectionTypes: acts}
  3812  	if uc.IssuerAccount != _EMPTY_ {
  3813  		nu.SigningKey = uc.Issuer
  3814  	}
  3815  
  3816  	// Now check for permissions.
  3817  	var p = buildPermissionsFromJwt(&uc.Permissions)
  3818  	if p == nil && acc.defaultPerms != nil {
  3819  		p = acc.defaultPerms.clone()
  3820  	}
  3821  	nu.Permissions = p
  3822  	return nu
  3823  }
  3824  
  3825  func fetchAccount(res AccountResolver, name string) (string, error) {
  3826  	if !nkeys.IsValidPublicAccountKey(name) {
  3827  		return _EMPTY_, fmt.Errorf("will only fetch valid account keys")
  3828  	}
  3829  	return res.Fetch(name)
  3830  }
  3831  
  3832  // AccountResolver interface. This is to fetch Account JWTs by public nkeys
  3833  type AccountResolver interface {
  3834  	Fetch(name string) (string, error)
  3835  	Store(name, jwt string) error
  3836  	IsReadOnly() bool
  3837  	Start(server *Server) error
  3838  	IsTrackingUpdate() bool
  3839  	Reload() error
  3840  	Close()
  3841  }
  3842  
  3843  // Default implementations of IsReadOnly/Start so only need to be written when changed
  3844  type resolverDefaultsOpsImpl struct{}
  3845  
  3846  func (*resolverDefaultsOpsImpl) IsReadOnly() bool {
  3847  	return true
  3848  }
  3849  
  3850  func (*resolverDefaultsOpsImpl) IsTrackingUpdate() bool {
  3851  	return false
  3852  }
  3853  
  3854  func (*resolverDefaultsOpsImpl) Start(*Server) error {
  3855  	return nil
  3856  }
  3857  
  3858  func (*resolverDefaultsOpsImpl) Reload() error {
  3859  	return nil
  3860  }
  3861  
  3862  func (*resolverDefaultsOpsImpl) Close() {
  3863  }
  3864  
  3865  func (*resolverDefaultsOpsImpl) Store(_, _ string) error {
  3866  	return fmt.Errorf("store operation not supported for URL Resolver")
  3867  }
  3868  
  3869  // MemAccResolver is a memory only resolver.
  3870  // Mostly for testing.
  3871  type MemAccResolver struct {
  3872  	sm sync.Map
  3873  	resolverDefaultsOpsImpl
  3874  }
  3875  
  3876  // Fetch will fetch the account jwt claims from the internal sync.Map.
  3877  func (m *MemAccResolver) Fetch(name string) (string, error) {
  3878  	if j, ok := m.sm.Load(name); ok {
  3879  		return j.(string), nil
  3880  	}
  3881  	return _EMPTY_, ErrMissingAccount
  3882  }
  3883  
  3884  // Store will store the account jwt claims in the internal sync.Map.
  3885  func (m *MemAccResolver) Store(name, jwt string) error {
  3886  	m.sm.Store(name, jwt)
  3887  	return nil
  3888  }
  3889  
  3890  func (m *MemAccResolver) IsReadOnly() bool {
  3891  	return false
  3892  }
  3893  
  3894  // URLAccResolver implements an http fetcher.
  3895  type URLAccResolver struct {
  3896  	url string
  3897  	c   *http.Client
  3898  	resolverDefaultsOpsImpl
  3899  }
  3900  
  3901  // NewURLAccResolver returns a new resolver for the given base URL.
  3902  func NewURLAccResolver(url string) (*URLAccResolver, error) {
  3903  	if !strings.HasSuffix(url, "/") {
  3904  		url += "/"
  3905  	}
  3906  	// FIXME(dlc) - Make timeout and others configurable.
  3907  	// We create our own transport to amortize TLS.
  3908  	tr := &http.Transport{
  3909  		MaxIdleConns:    10,
  3910  		IdleConnTimeout: 30 * time.Second,
  3911  	}
  3912  	ur := &URLAccResolver{
  3913  		url: url,
  3914  		c:   &http.Client{Timeout: DEFAULT_ACCOUNT_FETCH_TIMEOUT, Transport: tr},
  3915  	}
  3916  	return ur, nil
  3917  }
  3918  
  3919  // Fetch will fetch the account jwt claims from the base url, appending the
  3920  // account name onto the end.
  3921  func (ur *URLAccResolver) Fetch(name string) (string, error) {
  3922  	url := ur.url + name
  3923  	resp, err := ur.c.Get(url)
  3924  	if err != nil {
  3925  		return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), err)
  3926  	} else if resp == nil {
  3927  		return _EMPTY_, fmt.Errorf("could not fetch <%q>: no response", redactURLString(url))
  3928  	}
  3929  	defer resp.Body.Close()
  3930  	if resp.StatusCode != http.StatusOK {
  3931  		return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), resp.Status)
  3932  	}
  3933  	body, err := io.ReadAll(resp.Body)
  3934  	if err != nil {
  3935  		return _EMPTY_, err
  3936  	}
  3937  	return string(body), nil
  3938  }
  3939  
  3940  // Resolver based on nats for synchronization and backing directory for storage.
  3941  type DirAccResolver struct {
  3942  	*DirJWTStore
  3943  	*Server
  3944  	syncInterval time.Duration
  3945  	fetchTimeout time.Duration
  3946  }
  3947  
  3948  func (dr *DirAccResolver) IsTrackingUpdate() bool {
  3949  	return true
  3950  }
  3951  
  3952  func (dr *DirAccResolver) Reload() error {
  3953  	return dr.DirJWTStore.Reload()
  3954  }
  3955  
  3956  // ServerAPIClaimUpdateResponse is the response to $SYS.REQ.ACCOUNT.<id>.CLAIMS.UPDATE and $SYS.REQ.CLAIMS.UPDATE
  3957  type ServerAPIClaimUpdateResponse struct {
  3958  	Server *ServerInfo        `json:"server"`
  3959  	Data   *ClaimUpdateStatus `json:"data,omitempty"`
  3960  	Error  *ClaimUpdateError  `json:"error,omitempty"`
  3961  }
  3962  
  3963  type ClaimUpdateError struct {
  3964  	Account     string `json:"account,omitempty"`
  3965  	Code        int    `json:"code"`
  3966  	Description string `json:"description,omitempty"`
  3967  }
  3968  
  3969  type ClaimUpdateStatus struct {
  3970  	Account string `json:"account,omitempty"`
  3971  	Code    int    `json:"code,omitempty"`
  3972  	Message string `json:"message,omitempty"`
  3973  }
  3974  
  3975  func respondToUpdate(s *Server, respSubj string, acc string, message string, err error) {
  3976  	if err == nil {
  3977  		if acc == _EMPTY_ {
  3978  			s.Debugf("%s", message)
  3979  		} else {
  3980  			s.Debugf("%s - %s", message, acc)
  3981  		}
  3982  	} else {
  3983  		if acc == _EMPTY_ {
  3984  			s.Errorf("%s - %s", message, err)
  3985  		} else {
  3986  			s.Errorf("%s - %s - %s", message, acc, err)
  3987  		}
  3988  	}
  3989  	if respSubj == _EMPTY_ {
  3990  		return
  3991  	}
  3992  
  3993  	response := ServerAPIClaimUpdateResponse{
  3994  		Server: &ServerInfo{},
  3995  	}
  3996  
  3997  	if err == nil {
  3998  		response.Data = &ClaimUpdateStatus{
  3999  			Account: acc,
  4000  			Code:    http.StatusOK,
  4001  			Message: message,
  4002  		}
  4003  	} else {
  4004  		response.Error = &ClaimUpdateError{
  4005  			Account:     acc,
  4006  			Code:        http.StatusInternalServerError,
  4007  			Description: fmt.Sprintf("%s - %v", message, err),
  4008  		}
  4009  	}
  4010  
  4011  	s.sendInternalMsgLocked(respSubj, _EMPTY_, response.Server, response)
  4012  }
  4013  
  4014  func handleListRequest(store *DirJWTStore, s *Server, reply string) {
  4015  	if reply == _EMPTY_ {
  4016  		return
  4017  	}
  4018  	accIds := make([]string, 0, 1024)
  4019  	if err := store.PackWalk(1, func(partialPackMsg string) {
  4020  		if tk := strings.Split(partialPackMsg, "|"); len(tk) == 2 {
  4021  			accIds = append(accIds, tk[0])
  4022  		}
  4023  	}); err != nil {
  4024  		// let them timeout
  4025  		s.Errorf("list request error: %v", err)
  4026  	} else {
  4027  		s.Debugf("list request responded with %d account ids", len(accIds))
  4028  		server := &ServerInfo{}
  4029  		response := map[string]any{"server": server, "data": accIds}
  4030  		s.sendInternalMsgLocked(reply, _EMPTY_, server, response)
  4031  	}
  4032  }
  4033  
  4034  func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) {
  4035  	var accIds []any
  4036  	var subj, sysAccName string
  4037  	if sysAcc := s.SystemAccount(); sysAcc != nil {
  4038  		sysAccName = sysAcc.GetName()
  4039  	}
  4040  	// Only operator and operator signing key are allowed to delete
  4041  	gk, err := jwt.DecodeGeneric(string(msg))
  4042  	if err == nil {
  4043  		subj = gk.Subject
  4044  		if store.deleteType == NoDelete {
  4045  			err = fmt.Errorf("delete must be enabled in server config")
  4046  		} else if subj != gk.Issuer {
  4047  			err = fmt.Errorf("not self signed")
  4048  		} else if _, ok := store.operator[gk.Issuer]; !ok {
  4049  			err = fmt.Errorf("not trusted")
  4050  		} else if list, ok := gk.Data["accounts"]; !ok {
  4051  			err = fmt.Errorf("malformed request")
  4052  		} else if accIds, ok = list.([]any); !ok {
  4053  			err = fmt.Errorf("malformed request")
  4054  		} else {
  4055  			for _, entry := range accIds {
  4056  				if acc, ok := entry.(string); !ok ||
  4057  					acc == _EMPTY_ || !nkeys.IsValidPublicAccountKey(acc) {
  4058  					err = fmt.Errorf("malformed request")
  4059  					break
  4060  				} else if acc == sysAccName {
  4061  					err = fmt.Errorf("not allowed to delete system account")
  4062  					break
  4063  				}
  4064  			}
  4065  		}
  4066  	}
  4067  	if err != nil {
  4068  		respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("delete accounts request by %s failed", subj), err)
  4069  		return
  4070  	}
  4071  	errs := []string{}
  4072  	passCnt := 0
  4073  	for _, acc := range accIds {
  4074  		if err := store.delete(acc.(string)); err != nil {
  4075  			errs = append(errs, err.Error())
  4076  		} else {
  4077  			passCnt++
  4078  		}
  4079  	}
  4080  	if len(errs) == 0 {
  4081  		respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts", passCnt), nil)
  4082  	} else {
  4083  		respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts, failed for %d", passCnt, len(errs)),
  4084  			errors.New(strings.Join(errs, "\n")))
  4085  	}
  4086  }
  4087  
  4088  func getOperatorKeys(s *Server) (string, map[string]struct{}, bool, error) {
  4089  	var op string
  4090  	var strict bool
  4091  	keys := make(map[string]struct{})
  4092  	if opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 {
  4093  		op = opts.TrustedOperators[0].Subject
  4094  		strict = opts.TrustedOperators[0].StrictSigningKeyUsage
  4095  		if !strict {
  4096  			keys[opts.TrustedOperators[0].Subject] = struct{}{}
  4097  		}
  4098  		for _, key := range opts.TrustedOperators[0].SigningKeys {
  4099  			keys[key] = struct{}{}
  4100  		}
  4101  	}
  4102  	if len(keys) == 0 {
  4103  		return _EMPTY_, nil, false, fmt.Errorf("no operator key found")
  4104  	}
  4105  	return op, keys, strict, nil
  4106  }
  4107  
  4108  func claimValidate(claim *jwt.AccountClaims) error {
  4109  	vr := &jwt.ValidationResults{}
  4110  	claim.Validate(vr)
  4111  	if vr.IsBlocking(false) {
  4112  		return fmt.Errorf("validation errors: %v", vr.Errors())
  4113  	}
  4114  	return nil
  4115  }
  4116  
  4117  func removeCb(s *Server, pubKey string) {
  4118  	v, ok := s.accounts.Load(pubKey)
  4119  	if !ok {
  4120  		return
  4121  	}
  4122  	a := v.(*Account)
  4123  	s.Debugf("Disable account %s due to remove", pubKey)
  4124  	a.mu.Lock()
  4125  	// lock out new clients
  4126  	a.msubs = 0
  4127  	a.mpay = 0
  4128  	a.mconns = 0
  4129  	a.mleafs = 0
  4130  	a.updated = time.Now()
  4131  	jsa := a.js
  4132  	a.mu.Unlock()
  4133  	// set the account to be expired and disconnect clients
  4134  	a.expiredTimeout()
  4135  	// For JS, we need also to disable it.
  4136  	if js := s.getJetStream(); js != nil && jsa != nil {
  4137  		js.disableJetStream(jsa)
  4138  		// Remove JetStream state in memory, this will be reset
  4139  		// on the changed callback from the account in case it is
  4140  		// enabled again.
  4141  		a.js = nil
  4142  	}
  4143  	// We also need to remove all ServerImport subscriptions
  4144  	a.removeAllServiceImportSubs()
  4145  	a.mu.Lock()
  4146  	a.clearExpirationTimer()
  4147  	a.mu.Unlock()
  4148  }
  4149  
  4150  func (dr *DirAccResolver) Start(s *Server) error {
  4151  	op, opKeys, strict, err := getOperatorKeys(s)
  4152  	if err != nil {
  4153  		return err
  4154  	}
  4155  	dr.Lock()
  4156  	defer dr.Unlock()
  4157  	dr.Server = s
  4158  	dr.operator = opKeys
  4159  	dr.DirJWTStore.changed = func(pubKey string) {
  4160  		if v, ok := s.accounts.Load(pubKey); ok {
  4161  			if theJwt, err := dr.LoadAcc(pubKey); err != nil {
  4162  				s.Errorf("DirResolver - Update got error on load: %v", err)
  4163  			} else {
  4164  				acc := v.(*Account)
  4165  				if err = s.updateAccountWithClaimJWT(acc, theJwt); err != nil {
  4166  					s.Errorf("DirResolver - Update for account %q resulted in error %v", pubKey, err)
  4167  				} else {
  4168  					if _, jsa, err := acc.checkForJetStream(); err != nil {
  4169  						if !IsNatsErr(err, JSNotEnabledForAccountErr) {
  4170  							s.Warnf("DirResolver - Error checking for JetStream support for account %q: %v", pubKey, err)
  4171  						}
  4172  					} else if jsa == nil {
  4173  						if err = s.configJetStream(acc); err != nil {
  4174  							s.Errorf("DirResolver - Error configuring JetStream for account %q: %v", pubKey, err)
  4175  						}
  4176  					}
  4177  				}
  4178  			}
  4179  		}
  4180  	}
  4181  	dr.DirJWTStore.deleted = func(pubKey string) {
  4182  		removeCb(s, pubKey)
  4183  	}
  4184  	packRespIb := s.newRespInbox()
  4185  	for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {
  4186  		// subscribe to account jwt update requests
  4187  		if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) {
  4188  			var pubKey string
  4189  			tk := strings.Split(subj, tsep)
  4190  			if len(tk) == accUpdateTokensNew {
  4191  				pubKey = tk[accReqAccIndex]
  4192  			} else if len(tk) == accUpdateTokensOld {
  4193  				pubKey = tk[accUpdateAccIdxOld]
  4194  			} else {
  4195  				s.Debugf("DirResolver - jwt update skipped due to bad subject %q", subj)
  4196  				return
  4197  			}
  4198  			if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4199  				respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err)
  4200  			} else if err := claimValidate(claim); err != nil {
  4201  				respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err)
  4202  			} else if claim.Subject != pubKey {
  4203  				err := errors.New("subject does not match jwt content")
  4204  				respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err)
  4205  			} else if claim.Issuer == op && strict {
  4206  				err := errors.New("operator requires issuer to be a signing key")
  4207  				respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err)
  4208  			} else if err := dr.save(pubKey, string(msg)); err != nil {
  4209  				respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err)
  4210  			} else {
  4211  				respondToUpdate(s, resp, pubKey, "jwt updated", nil)
  4212  			}
  4213  		}); err != nil {
  4214  			return fmt.Errorf("error setting up update handling: %v", err)
  4215  		}
  4216  	}
  4217  	if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) {
  4218  		// As this is a raw message, we need to extract payload and only decode claims from it,
  4219  		// in case request is sent with headers.
  4220  		_, msg = c.msgParts(msg)
  4221  		if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4222  			respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err)
  4223  		} else if claim.Issuer == op && strict {
  4224  			err := errors.New("operator requires issuer to be a signing key")
  4225  			respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err)
  4226  		} else if err := claimValidate(claim); err != nil {
  4227  			respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err)
  4228  		} else if err := dr.save(claim.Subject, string(msg)); err != nil {
  4229  			respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err)
  4230  		} else {
  4231  			respondToUpdate(s, resp, claim.Subject, "jwt updated", nil)
  4232  		}
  4233  	}); err != nil {
  4234  		return fmt.Errorf("error setting up update handling: %v", err)
  4235  	}
  4236  	// respond to lookups with our version
  4237  	if _, err := s.sysSubscribe(fmt.Sprintf(accLookupReqSubj, "*"), func(_ *subscription, _ *client, _ *Account, subj, reply string, msg []byte) {
  4238  		if reply == _EMPTY_ {
  4239  			return
  4240  		}
  4241  		tk := strings.Split(subj, tsep)
  4242  		if len(tk) != accLookupReqTokens {
  4243  			return
  4244  		}
  4245  		accName := tk[accReqAccIndex]
  4246  		if theJWT, err := dr.DirJWTStore.LoadAcc(accName); err != nil {
  4247  			if errors.Is(err, fs.ErrNotExist) {
  4248  				s.Debugf("DirResolver - Could not find account %q", accName)
  4249  				// Reply with empty response to signal absence of JWT to others.
  4250  				s.sendInternalMsgLocked(reply, _EMPTY_, nil, nil)
  4251  			} else {
  4252  				s.Errorf("DirResolver - Error looking up account %q: %v", accName, err)
  4253  			}
  4254  		} else {
  4255  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(theJWT))
  4256  		}
  4257  	}); err != nil {
  4258  		return fmt.Errorf("error setting up lookup request handling: %v", err)
  4259  	}
  4260  	// respond to pack requests with one or more pack messages
  4261  	// an empty message signifies the end of the response responder.
  4262  	if _, err := s.sysSubscribeQ(accPackReqSubj, "responder", func(_ *subscription, _ *client, _ *Account, _, reply string, theirHash []byte) {
  4263  		if reply == _EMPTY_ {
  4264  			return
  4265  		}
  4266  		ourHash := dr.DirJWTStore.Hash()
  4267  		if bytes.Equal(theirHash, ourHash[:]) {
  4268  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{})
  4269  			s.Debugf("DirResolver - Pack request matches hash %x", ourHash[:])
  4270  		} else if err := dr.DirJWTStore.PackWalk(1, func(partialPackMsg string) {
  4271  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(partialPackMsg))
  4272  		}); err != nil {
  4273  			// let them timeout
  4274  			s.Errorf("DirResolver - Pack request error: %v", err)
  4275  		} else {
  4276  			s.Debugf("DirResolver - Pack request hash %x - finished responding with hash %x", theirHash, ourHash)
  4277  			s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{})
  4278  		}
  4279  	}); err != nil {
  4280  		return fmt.Errorf("error setting up pack request handling: %v", err)
  4281  	}
  4282  	// respond to list requests with one message containing all account ids
  4283  	if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) {
  4284  		handleListRequest(dr.DirJWTStore, s, reply)
  4285  	}); err != nil {
  4286  		return fmt.Errorf("error setting up list request handling: %v", err)
  4287  	}
  4288  	if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) {
  4289  		handleDeleteRequest(dr.DirJWTStore, s, msg, reply)
  4290  	}); err != nil {
  4291  		return fmt.Errorf("error setting up delete request handling: %v", err)
  4292  	}
  4293  	// embed pack responses into store
  4294  	if _, err := s.sysSubscribe(packRespIb, func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) {
  4295  		hash := dr.DirJWTStore.Hash()
  4296  		if len(msg) == 0 { // end of response stream
  4297  			s.Debugf("DirResolver - Merging finished and resulting in: %x", dr.DirJWTStore.Hash())
  4298  			return
  4299  		} else if err := dr.DirJWTStore.Merge(string(msg)); err != nil {
  4300  			s.Errorf("DirResolver - Merging resulted in error: %v", err)
  4301  		} else {
  4302  			s.Debugf("DirResolver - Merging succeeded and changed %x to %x", hash, dr.DirJWTStore.Hash())
  4303  		}
  4304  	}); err != nil {
  4305  		return fmt.Errorf("error setting up pack response handling: %v", err)
  4306  	}
  4307  	// periodically send out pack message
  4308  	quit := s.quitCh
  4309  	s.startGoRoutine(func() {
  4310  		defer s.grWG.Done()
  4311  		ticker := time.NewTicker(dr.syncInterval)
  4312  		for {
  4313  			select {
  4314  			case <-quit:
  4315  				ticker.Stop()
  4316  				return
  4317  			case <-ticker.C:
  4318  			}
  4319  			ourHash := dr.DirJWTStore.Hash()
  4320  			s.Debugf("DirResolver - Checking store state: %x", ourHash)
  4321  			s.sendInternalMsgLocked(accPackReqSubj, packRespIb, nil, ourHash[:])
  4322  		}
  4323  	})
  4324  	s.Noticef("Managing all jwt in exclusive directory %s", dr.directory)
  4325  	return nil
  4326  }
  4327  
  4328  func (dr *DirAccResolver) Fetch(name string) (string, error) {
  4329  	if theJWT, err := dr.LoadAcc(name); theJWT != _EMPTY_ {
  4330  		return theJWT, nil
  4331  	} else {
  4332  		dr.Lock()
  4333  		srv := dr.Server
  4334  		to := dr.fetchTimeout
  4335  		dr.Unlock()
  4336  		if srv == nil {
  4337  			return _EMPTY_, err
  4338  		}
  4339  		return srv.fetch(dr, name, to) // lookup from other server
  4340  	}
  4341  }
  4342  
  4343  func (dr *DirAccResolver) Store(name, jwt string) error {
  4344  	return dr.saveIfNewer(name, jwt)
  4345  }
  4346  
  4347  type DirResOption func(s *DirAccResolver) error
  4348  
  4349  // limits the amount of time spent waiting for an account fetch to complete
  4350  func FetchTimeout(to time.Duration) DirResOption {
  4351  	return func(r *DirAccResolver) error {
  4352  		if to <= time.Duration(0) {
  4353  			return fmt.Errorf("Fetch timeout %v is too smal", to)
  4354  		}
  4355  		r.fetchTimeout = to
  4356  		return nil
  4357  	}
  4358  }
  4359  
  4360  func (dr *DirAccResolver) apply(opts ...DirResOption) error {
  4361  	for _, o := range opts {
  4362  		if err := o(dr); err != nil {
  4363  			return err
  4364  		}
  4365  	}
  4366  	return nil
  4367  }
  4368  
  4369  func NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete deleteType, opts ...DirResOption) (*DirAccResolver, error) {
  4370  	if limit == 0 {
  4371  		limit = math.MaxInt64
  4372  	}
  4373  	if syncInterval <= 0 {
  4374  		syncInterval = time.Minute
  4375  	}
  4376  	store, err := NewExpiringDirJWTStore(path, false, true, delete, 0, limit, false, 0, nil)
  4377  	if err != nil {
  4378  		return nil, err
  4379  	}
  4380  
  4381  	res := &DirAccResolver{store, nil, syncInterval, DEFAULT_ACCOUNT_FETCH_TIMEOUT}
  4382  	if err := res.apply(opts...); err != nil {
  4383  		return nil, err
  4384  	}
  4385  	return res, nil
  4386  }
  4387  
  4388  // Caching resolver using nats for lookups and making use of a directory for storage
  4389  type CacheDirAccResolver struct {
  4390  	DirAccResolver
  4391  	ttl time.Duration
  4392  }
  4393  
  4394  func (s *Server) fetch(res AccountResolver, name string, timeout time.Duration) (string, error) {
  4395  	if s == nil {
  4396  		return _EMPTY_, ErrNoAccountResolver
  4397  	}
  4398  	respC := make(chan []byte, 1)
  4399  	accountLookupRequest := fmt.Sprintf(accLookupReqSubj, name)
  4400  	s.mu.Lock()
  4401  	if s.sys == nil || s.sys.replies == nil {
  4402  		s.mu.Unlock()
  4403  		return _EMPTY_, fmt.Errorf("eventing shut down")
  4404  	}
  4405  	// Resolver will wait for detected active servers to reply
  4406  	// before serving an error in case there weren't any found.
  4407  	expectedServers := len(s.sys.servers)
  4408  	replySubj := s.newRespInbox()
  4409  	replies := s.sys.replies
  4410  
  4411  	// Store our handler.
  4412  	replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) {
  4413  		var clone []byte
  4414  		isEmpty := len(msg) == 0
  4415  		if !isEmpty {
  4416  			clone = make([]byte, len(msg))
  4417  			copy(clone, msg)
  4418  		}
  4419  		s.mu.Lock()
  4420  		defer s.mu.Unlock()
  4421  		expectedServers--
  4422  		// Skip empty responses until getting all the available servers.
  4423  		if isEmpty && expectedServers > 0 {
  4424  			return
  4425  		}
  4426  		// Use the first valid response if there is still interest or
  4427  		// one of the empty responses to signal that it was not found.
  4428  		if _, ok := replies[replySubj]; ok {
  4429  			select {
  4430  			case respC <- clone:
  4431  			default:
  4432  			}
  4433  		}
  4434  	}
  4435  	s.sendInternalMsg(accountLookupRequest, replySubj, nil, []byte{})
  4436  	quit := s.quitCh
  4437  	s.mu.Unlock()
  4438  	var err error
  4439  	var theJWT string
  4440  	select {
  4441  	case <-quit:
  4442  		err = errors.New("fetching jwt failed due to shutdown")
  4443  	case <-time.After(timeout):
  4444  		err = errors.New("fetching jwt timed out")
  4445  	case m := <-respC:
  4446  		if len(m) == 0 {
  4447  			err = errors.New("account jwt not found")
  4448  		} else if err = res.Store(name, string(m)); err == nil {
  4449  			theJWT = string(m)
  4450  		}
  4451  	}
  4452  	s.mu.Lock()
  4453  	delete(replies, replySubj)
  4454  	s.mu.Unlock()
  4455  	close(respC)
  4456  	return theJWT, err
  4457  }
  4458  
  4459  func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, opts ...DirResOption) (*CacheDirAccResolver, error) {
  4460  	if limit <= 0 {
  4461  		limit = 1_000
  4462  	}
  4463  	store, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil)
  4464  	if err != nil {
  4465  		return nil, err
  4466  	}
  4467  	res := &CacheDirAccResolver{DirAccResolver{store, nil, 0, DEFAULT_ACCOUNT_FETCH_TIMEOUT}, ttl}
  4468  	if err := res.apply(opts...); err != nil {
  4469  		return nil, err
  4470  	}
  4471  	return res, nil
  4472  }
  4473  
  4474  func (dr *CacheDirAccResolver) Start(s *Server) error {
  4475  	op, opKeys, strict, err := getOperatorKeys(s)
  4476  	if err != nil {
  4477  		return err
  4478  	}
  4479  	dr.Lock()
  4480  	defer dr.Unlock()
  4481  	dr.Server = s
  4482  	dr.operator = opKeys
  4483  	dr.DirJWTStore.changed = func(pubKey string) {
  4484  		if v, ok := s.accounts.Load(pubKey); !ok {
  4485  		} else if theJwt, err := dr.LoadAcc(pubKey); err != nil {
  4486  			s.Errorf("DirResolver - Update got error on load: %v", err)
  4487  		} else if err := s.updateAccountWithClaimJWT(v.(*Account), theJwt); err != nil {
  4488  			s.Errorf("DirResolver - Update resulted in error %v", err)
  4489  		}
  4490  	}
  4491  	dr.DirJWTStore.deleted = func(pubKey string) {
  4492  		removeCb(s, pubKey)
  4493  	}
  4494  	for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {
  4495  		// subscribe to account jwt update requests
  4496  		if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) {
  4497  			var pubKey string
  4498  			tk := strings.Split(subj, tsep)
  4499  			if len(tk) == accUpdateTokensNew {
  4500  				pubKey = tk[accReqAccIndex]
  4501  			} else if len(tk) == accUpdateTokensOld {
  4502  				pubKey = tk[accUpdateAccIdxOld]
  4503  			} else {
  4504  				s.Debugf("DirResolver - jwt update cache skipped due to bad subject %q", subj)
  4505  				return
  4506  			}
  4507  			if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4508  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4509  			} else if claim.Subject != pubKey {
  4510  				err := errors.New("subject does not match jwt content")
  4511  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4512  			} else if claim.Issuer == op && strict {
  4513  				err := errors.New("operator requires issuer to be a signing key")
  4514  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4515  			} else if _, ok := s.accounts.Load(pubKey); !ok {
  4516  				respondToUpdate(s, resp, pubKey, "jwt update cache skipped", nil)
  4517  			} else if err := claimValidate(claim); err != nil {
  4518  				respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err)
  4519  			} else if err := dr.save(pubKey, string(msg)); err != nil {
  4520  				respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err)
  4521  			} else {
  4522  				respondToUpdate(s, resp, pubKey, "jwt updated cache", nil)
  4523  			}
  4524  		}); err != nil {
  4525  			return fmt.Errorf("error setting up update handling: %v", err)
  4526  		}
  4527  	}
  4528  	if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) {
  4529  		// As this is a raw message, we need to extract payload and only decode claims from it,
  4530  		// in case request is sent with headers.
  4531  		_, msg = c.msgParts(msg)
  4532  		if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {
  4533  			respondToUpdate(s, resp, "n/a", "jwt update cache resulted in error", err)
  4534  		} else if claim.Issuer == op && strict {
  4535  			err := errors.New("operator requires issuer to be a signing key")
  4536  			respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err)
  4537  		} else if _, ok := s.accounts.Load(claim.Subject); !ok {
  4538  			respondToUpdate(s, resp, claim.Subject, "jwt update cache skipped", nil)
  4539  		} else if err := claimValidate(claim); err != nil {
  4540  			respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err)
  4541  		} else if err := dr.save(claim.Subject, string(msg)); err != nil {
  4542  			respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err)
  4543  		} else {
  4544  			respondToUpdate(s, resp, claim.Subject, "jwt updated cache", nil)
  4545  		}
  4546  	}); err != nil {
  4547  		return fmt.Errorf("error setting up update handling: %v", err)
  4548  	}
  4549  	// respond to list requests with one message containing all account ids
  4550  	if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) {
  4551  		handleListRequest(dr.DirJWTStore, s, reply)
  4552  	}); err != nil {
  4553  		return fmt.Errorf("error setting up list request handling: %v", err)
  4554  	}
  4555  	if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) {
  4556  		handleDeleteRequest(dr.DirJWTStore, s, msg, reply)
  4557  	}); err != nil {
  4558  		return fmt.Errorf("error setting up list request handling: %v", err)
  4559  	}
  4560  	s.Noticef("Managing some jwt in exclusive directory %s", dr.directory)
  4561  	return nil
  4562  }
  4563  
  4564  func (dr *CacheDirAccResolver) Reload() error {
  4565  	return dr.DirAccResolver.Reload()
  4566  }