github.com/nats-io/nats-server/v2@v2.11.0-preview.2/server/accounts.go (about) 1 // Copyright 2018-2024 The NATS Authors 2 // Licensed under the Apache License, Version 2.0 (the "License"); 3 // you may not use this file except in compliance with the License. 4 // You may obtain a copy of the License at 5 // 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package server 15 16 import ( 17 "bytes" 18 "encoding/hex" 19 "errors" 20 "fmt" 21 "io" 22 "io/fs" 23 "math" 24 "math/rand" 25 "net/http" 26 "net/textproto" 27 "reflect" 28 "sort" 29 "strconv" 30 "strings" 31 "sync" 32 "sync/atomic" 33 "time" 34 35 "github.com/nats-io/jwt/v2" 36 "github.com/nats-io/nats-server/v2/internal/fastrand" 37 "github.com/nats-io/nkeys" 38 "github.com/nats-io/nuid" 39 ) 40 41 // For backwards compatibility with NATS < 2.0, users who are not explicitly defined into an 42 // account will be grouped in the default global account. 43 const globalAccountName = DEFAULT_GLOBAL_ACCOUNT 44 45 const defaultMaxSubLimitReportThreshold = int64(2 * time.Second) 46 47 var maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold 48 49 // Account are subject namespace definitions. By default no messages are shared between accounts. 50 // You can share via Exports and Imports of Streams and Services. 51 type Account struct { 52 stats 53 gwReplyMapping 54 Name string 55 LogicalName string 56 Nkey string 57 Issuer string 58 claimJWT string 59 updated time.Time 60 mu sync.RWMutex 61 sqmu sync.Mutex 62 sl *Sublist 63 ic *client 64 isid uint64 65 etmr *time.Timer 66 ctmr *time.Timer 67 strack map[string]sconns 68 nrclients int32 69 sysclients int32 70 nleafs int32 71 nrleafs int32 72 clients map[*client]struct{} 73 rm map[string]int32 74 lqws map[string]int32 75 usersRevoked map[string]int64 76 mappings []*mapping 77 hasMapped atomic.Bool 78 lmu sync.RWMutex 79 lleafs []*client 80 leafClusters map[string]uint64 81 imports importMap 82 exports exportMap 83 js *jsAccount 84 jsLimits map[string]JetStreamAccountLimits 85 limits 86 expired bool 87 incomplete bool 88 signingKeys map[string]jwt.Scope 89 extAuth *jwt.ExternalAuthorization 90 srv *Server // server this account is registered with (possibly nil) 91 lds string // loop detection subject for leaf nodes 92 siReply []byte // service reply prefix, will form wildcard subscription. 93 eventIds *nuid.NUID 94 eventIdsMu sync.Mutex 95 defaultPerms *Permissions 96 tags jwt.TagList 97 nameTag string 98 lastLimErr int64 99 routePoolIdx int 100 // If the trace destination is specified and a message with a traceParentHdr 101 // is received, and has the least significant bit of the last token set to 1, 102 // then if traceDestSampling is > 0 and < 100, a random value will be selected 103 // and if it falls between 0 and that value, message tracing will be triggered. 104 traceDest string 105 traceDestSampling int 106 } 107 108 const ( 109 accDedicatedRoute = -1 110 accTransitioningToDedicatedRoute = -2 111 ) 112 113 // Account based limits. 114 type limits struct { 115 mpay int32 116 msubs int32 117 mconns int32 118 mleafs int32 119 disallowBearer bool 120 } 121 122 // Used to track remote clients and leafnodes per remote server. 123 type sconns struct { 124 conns int32 125 leafs int32 126 } 127 128 // Import stream mapping struct 129 type streamImport struct { 130 acc *Account 131 from string 132 to string 133 tr *subjectTransform 134 rtr *subjectTransform 135 claim *jwt.Import 136 usePub bool 137 invalid bool 138 // This is `allow_trace` and when true and message tracing is happening, 139 // we will trace egresses past the account boundary, if `false`, we stop 140 // at the account boundary. 141 atrc bool 142 } 143 144 const ClientInfoHdr = "Nats-Request-Info" 145 146 // Import service mapping struct 147 type serviceImport struct { 148 acc *Account 149 claim *jwt.Import 150 se *serviceExport 151 sid []byte 152 from string 153 to string 154 tr *subjectTransform 155 ts int64 156 rt ServiceRespType 157 latency *serviceLatency 158 m1 *ServiceLatency 159 rc *client 160 usePub bool 161 response bool 162 invalid bool 163 share bool 164 tracking bool 165 didDeliver bool 166 atrc bool // allow trace (got from service export) 167 trackingHdr http.Header // header from request 168 } 169 170 // This is used to record when we create a mapping for implicit service 171 // imports. We use this to clean up entries that are not singletons when 172 // we detect that interest is no longer present. The key to the map will 173 // be the actual interest. We record the mapped subject and the account. 174 type serviceRespEntry struct { 175 acc *Account 176 msub string 177 } 178 179 // ServiceRespType represents the types of service request response types. 180 type ServiceRespType uint8 181 182 // Service response types. Defaults to a singleton. 183 const ( 184 Singleton ServiceRespType = iota 185 Streamed 186 Chunked 187 ) 188 189 // String helper. 190 func (rt ServiceRespType) String() string { 191 switch rt { 192 case Singleton: 193 return "Singleton" 194 case Streamed: 195 return "Streamed" 196 case Chunked: 197 return "Chunked" 198 } 199 return "Unknown ServiceResType" 200 } 201 202 // exportAuth holds configured approvals or boolean indicating an 203 // auth token is required for import. 204 type exportAuth struct { 205 tokenReq bool 206 accountPos uint 207 approved map[string]*Account 208 actsRevoked map[string]int64 209 } 210 211 // streamExport 212 type streamExport struct { 213 exportAuth 214 } 215 216 // serviceExport holds additional information for exported services. 217 type serviceExport struct { 218 exportAuth 219 acc *Account 220 respType ServiceRespType 221 latency *serviceLatency 222 rtmr *time.Timer 223 respThresh time.Duration 224 // This is `allow_trace` and when true and message tracing is happening, 225 // when processing a service import we will go through account boundary 226 // and trace egresses on that other account. If `false`, we stop at the 227 // account boundary. 228 atrc bool 229 } 230 231 // Used to track service latency. 232 type serviceLatency struct { 233 sampling int8 // percentage from 1-100 or 0 to indicate triggered by header 234 subject string 235 } 236 237 // exportMap tracks the exported streams and services. 238 type exportMap struct { 239 streams map[string]*streamExport 240 services map[string]*serviceExport 241 responses map[string]*serviceImport 242 } 243 244 // importMap tracks the imported streams and services. 245 // For services we will also track the response mappings as well. 246 type importMap struct { 247 streams []*streamImport 248 services map[string]*serviceImport 249 rrMap map[string][]*serviceRespEntry 250 } 251 252 // NewAccount creates a new unlimited account with the given name. 253 func NewAccount(name string) *Account { 254 a := &Account{ 255 Name: name, 256 LogicalName: name, 257 limits: limits{-1, -1, -1, -1, false}, 258 eventIds: nuid.New(), 259 } 260 return a 261 } 262 263 func (a *Account) String() string { 264 return a.Name 265 } 266 267 func (a *Account) setTraceDest(dest string) { 268 a.mu.Lock() 269 a.traceDest = dest 270 a.mu.Unlock() 271 } 272 273 func (a *Account) getTraceDestAndSampling() (string, int) { 274 a.mu.RLock() 275 dest := a.traceDest 276 sampling := a.traceDestSampling 277 a.mu.RUnlock() 278 return dest, sampling 279 } 280 281 // Used to create shallow copies of accounts for transfer 282 // from opts to real accounts in server struct. 283 // Account `na` write lock is expected to be held on entry 284 // while account `a` is the one from the Options struct 285 // being loaded/reloaded and do not need locking. 286 func (a *Account) shallowCopy(na *Account) { 287 na.Nkey = a.Nkey 288 na.Issuer = a.Issuer 289 na.traceDest, na.traceDestSampling = a.traceDest, a.traceDestSampling 290 291 if a.imports.streams != nil { 292 na.imports.streams = make([]*streamImport, 0, len(a.imports.streams)) 293 for _, v := range a.imports.streams { 294 si := *v 295 na.imports.streams = append(na.imports.streams, &si) 296 } 297 } 298 if a.imports.services != nil { 299 na.imports.services = make(map[string]*serviceImport) 300 for k, v := range a.imports.services { 301 si := *v 302 na.imports.services[k] = &si 303 } 304 } 305 if a.exports.streams != nil { 306 na.exports.streams = make(map[string]*streamExport) 307 for k, v := range a.exports.streams { 308 if v != nil { 309 se := *v 310 na.exports.streams[k] = &se 311 } else { 312 na.exports.streams[k] = nil 313 } 314 } 315 } 316 if a.exports.services != nil { 317 na.exports.services = make(map[string]*serviceExport) 318 for k, v := range a.exports.services { 319 if v != nil { 320 se := *v 321 na.exports.services[k] = &se 322 } else { 323 na.exports.services[k] = nil 324 } 325 } 326 } 327 na.mappings = a.mappings 328 na.hasMapped.Store(len(na.mappings) > 0) 329 330 // JetStream 331 na.jsLimits = a.jsLimits 332 // Server config account limits. 333 na.limits = a.limits 334 } 335 336 // nextEventID uses its own lock for better concurrency. 337 func (a *Account) nextEventID() string { 338 a.eventIdsMu.Lock() 339 id := a.eventIds.Next() 340 a.eventIdsMu.Unlock() 341 return id 342 } 343 344 // Returns a slice of clients stored in the account, or nil if none is present. 345 // Lock is held on entry. 346 func (a *Account) getClientsLocked() []*client { 347 if len(a.clients) == 0 { 348 return nil 349 } 350 clients := make([]*client, 0, len(a.clients)) 351 for c := range a.clients { 352 clients = append(clients, c) 353 } 354 return clients 355 } 356 357 // Returns a slice of clients stored in the account, or nil if none is present. 358 func (a *Account) getClients() []*client { 359 a.mu.RLock() 360 clients := a.getClientsLocked() 361 a.mu.RUnlock() 362 return clients 363 } 364 365 // Called to track a remote server and connections and leafnodes it 366 // has for this account. 367 func (a *Account) updateRemoteServer(m *AccountNumConns) []*client { 368 a.mu.Lock() 369 if a.strack == nil { 370 a.strack = make(map[string]sconns) 371 } 372 // This does not depend on receiving all updates since each one is idempotent. 373 // FIXME(dlc) - We should cleanup when these both go to zero. 374 prev := a.strack[m.Server.ID] 375 a.strack[m.Server.ID] = sconns{conns: int32(m.Conns), leafs: int32(m.LeafNodes)} 376 a.nrclients += int32(m.Conns) - prev.conns 377 a.nrleafs += int32(m.LeafNodes) - prev.leafs 378 379 mtce := a.mconns != jwt.NoLimit && (len(a.clients)-int(a.sysclients)+int(a.nrclients) > int(a.mconns)) 380 // If we are over here some have snuck in and we need to rebalance. 381 // All others will probably be doing the same thing but better to be 382 // conservative and bit harsh here. Clients will reconnect if we over compensate. 383 var clients []*client 384 if mtce { 385 clients := a.getClientsLocked() 386 sort.Slice(clients, func(i, j int) bool { 387 return clients[i].start.After(clients[j].start) 388 }) 389 over := (len(a.clients) - int(a.sysclients) + int(a.nrclients)) - int(a.mconns) 390 if over < len(clients) { 391 clients = clients[:over] 392 } 393 } 394 // Now check leafnodes. 395 mtlce := a.mleafs != jwt.NoLimit && (a.nleafs+a.nrleafs > a.mleafs) 396 if mtlce { 397 // Take ones from the end. 398 a.lmu.RLock() 399 leafs := a.lleafs 400 over := int(a.nleafs + a.nrleafs - a.mleafs) 401 if over < len(leafs) { 402 leafs = leafs[len(leafs)-over:] 403 } 404 clients = append(clients, leafs...) 405 a.lmu.RUnlock() 406 } 407 a.mu.Unlock() 408 409 // If we have exceeded our max clients this will be populated. 410 return clients 411 } 412 413 // Removes tracking for a remote server that has shutdown. 414 func (a *Account) removeRemoteServer(sid string) { 415 a.mu.Lock() 416 if a.strack != nil { 417 prev := a.strack[sid] 418 delete(a.strack, sid) 419 a.nrclients -= prev.conns 420 a.nrleafs -= prev.leafs 421 } 422 a.mu.Unlock() 423 } 424 425 // When querying for subject interest this is the number of 426 // expected responses. We need to actually check that the entry 427 // has active connections. 428 func (a *Account) expectedRemoteResponses() (expected int32) { 429 a.mu.RLock() 430 for _, sc := range a.strack { 431 if sc.conns > 0 || sc.leafs > 0 { 432 expected++ 433 } 434 } 435 a.mu.RUnlock() 436 return 437 } 438 439 // Clears eventing and tracking for this account. 440 func (a *Account) clearEventing() { 441 a.mu.Lock() 442 a.nrclients = 0 443 // Now clear state 444 clearTimer(&a.etmr) 445 clearTimer(&a.ctmr) 446 a.clients = nil 447 a.strack = nil 448 a.mu.Unlock() 449 } 450 451 // GetName will return the accounts name. 452 func (a *Account) GetName() string { 453 if a == nil { 454 return "n/a" 455 } 456 a.mu.RLock() 457 name := a.Name 458 a.mu.RUnlock() 459 return name 460 } 461 462 // NumConnections returns active number of clients for this account for 463 // all known servers. 464 func (a *Account) NumConnections() int { 465 a.mu.RLock() 466 nc := len(a.clients) - int(a.sysclients) + int(a.nrclients) 467 a.mu.RUnlock() 468 return nc 469 } 470 471 // NumRemoteConnections returns the number of client or leaf connections that 472 // are not on this server. 473 func (a *Account) NumRemoteConnections() int { 474 a.mu.RLock() 475 nc := int(a.nrclients + a.nrleafs) 476 a.mu.RUnlock() 477 return nc 478 } 479 480 // NumLocalConnections returns active number of clients for this account 481 // on this server. 482 func (a *Account) NumLocalConnections() int { 483 a.mu.RLock() 484 nlc := a.numLocalConnections() 485 a.mu.RUnlock() 486 return nlc 487 } 488 489 // Do not account for the system accounts. 490 func (a *Account) numLocalConnections() int { 491 return len(a.clients) - int(a.sysclients) - int(a.nleafs) 492 } 493 494 // This is for extended local interest. 495 // Lock should not be held. 496 func (a *Account) numLocalAndLeafConnections() int { 497 a.mu.RLock() 498 nlc := len(a.clients) - int(a.sysclients) 499 a.mu.RUnlock() 500 return nlc 501 } 502 503 func (a *Account) numLocalLeafNodes() int { 504 return int(a.nleafs) 505 } 506 507 // MaxTotalConnectionsReached returns if we have reached our limit for number of connections. 508 func (a *Account) MaxTotalConnectionsReached() bool { 509 var mtce bool 510 a.mu.RLock() 511 if a.mconns != jwt.NoLimit { 512 mtce = len(a.clients)-int(a.sysclients)+int(a.nrclients) >= int(a.mconns) 513 } 514 a.mu.RUnlock() 515 return mtce 516 } 517 518 // MaxActiveConnections return the set limit for the account system 519 // wide for total number of active connections. 520 func (a *Account) MaxActiveConnections() int { 521 a.mu.RLock() 522 mconns := int(a.mconns) 523 a.mu.RUnlock() 524 return mconns 525 } 526 527 // MaxTotalLeafNodesReached returns if we have reached our limit for number of leafnodes. 528 func (a *Account) MaxTotalLeafNodesReached() bool { 529 a.mu.RLock() 530 mtc := a.maxTotalLeafNodesReached() 531 a.mu.RUnlock() 532 return mtc 533 } 534 535 func (a *Account) maxTotalLeafNodesReached() bool { 536 if a.mleafs != jwt.NoLimit { 537 return a.nleafs+a.nrleafs >= a.mleafs 538 } 539 return false 540 } 541 542 // NumLeafNodes returns the active number of local and remote 543 // leaf node connections. 544 func (a *Account) NumLeafNodes() int { 545 a.mu.RLock() 546 nln := int(a.nleafs + a.nrleafs) 547 a.mu.RUnlock() 548 return nln 549 } 550 551 // NumRemoteLeafNodes returns the active number of remote 552 // leaf node connections. 553 func (a *Account) NumRemoteLeafNodes() int { 554 a.mu.RLock() 555 nrn := int(a.nrleafs) 556 a.mu.RUnlock() 557 return nrn 558 } 559 560 // MaxActiveLeafNodes return the set limit for the account system 561 // wide for total number of leavenode connections. 562 // NOTE: these are tracked separately. 563 func (a *Account) MaxActiveLeafNodes() int { 564 a.mu.RLock() 565 mleafs := int(a.mleafs) 566 a.mu.RUnlock() 567 return mleafs 568 } 569 570 // RoutedSubs returns how many subjects we would send across a route when first 571 // connected or expressing interest. Local client subs. 572 func (a *Account) RoutedSubs() int { 573 a.mu.RLock() 574 defer a.mu.RUnlock() 575 return len(a.rm) 576 } 577 578 // TotalSubs returns total number of Subscriptions for this account. 579 func (a *Account) TotalSubs() int { 580 a.mu.RLock() 581 defer a.mu.RUnlock() 582 if a.sl == nil { 583 return 0 584 } 585 return int(a.sl.Count()) 586 } 587 588 func (a *Account) shouldLogMaxSubErr() bool { 589 if a == nil { 590 return true 591 } 592 a.mu.RLock() 593 last := a.lastLimErr 594 a.mu.RUnlock() 595 if now := time.Now().UnixNano(); now-last >= maxSubLimitReportThreshold { 596 a.mu.Lock() 597 a.lastLimErr = now 598 a.mu.Unlock() 599 return true 600 } 601 return false 602 } 603 604 // MapDest is for mapping published subjects for clients. 605 type MapDest struct { 606 Subject string `json:"subject"` 607 Weight uint8 `json:"weight"` 608 Cluster string `json:"cluster,omitempty"` 609 } 610 611 func NewMapDest(subject string, weight uint8) *MapDest { 612 return &MapDest{subject, weight, _EMPTY_} 613 } 614 615 // destination is for internal representation for a weighted mapped destination. 616 type destination struct { 617 tr *subjectTransform 618 weight uint8 619 } 620 621 // mapping is an internal entry for mapping subjects. 622 type mapping struct { 623 src string 624 wc bool 625 dests []*destination 626 cdests map[string][]*destination 627 } 628 629 // AddMapping adds in a simple route mapping from src subject to dest subject 630 // for inbound client messages. 631 func (a *Account) AddMapping(src, dest string) error { 632 return a.AddWeightedMappings(src, NewMapDest(dest, 100)) 633 } 634 635 // AddWeightedMappings will add in a weighted mappings for the destinations. 636 func (a *Account) AddWeightedMappings(src string, dests ...*MapDest) error { 637 a.mu.Lock() 638 defer a.mu.Unlock() 639 640 if !IsValidSubject(src) { 641 return ErrBadSubject 642 } 643 644 m := &mapping{src: src, wc: subjectHasWildcard(src), dests: make([]*destination, 0, len(dests)+1)} 645 seen := make(map[string]struct{}) 646 647 var tw = make(map[string]uint8) 648 for _, d := range dests { 649 if _, ok := seen[d.Subject]; ok { 650 return fmt.Errorf("duplicate entry for %q", d.Subject) 651 } 652 seen[d.Subject] = struct{}{} 653 if d.Weight > 100 { 654 return fmt.Errorf("individual weights need to be <= 100") 655 } 656 tw[d.Cluster] += d.Weight 657 if tw[d.Cluster] > 100 { 658 return fmt.Errorf("total weight needs to be <= 100") 659 } 660 err := ValidateMappingDestination(d.Subject) 661 if err != nil { 662 return err 663 } 664 tr, err := NewSubjectTransform(src, d.Subject) 665 if err != nil { 666 return err 667 } 668 if d.Cluster == _EMPTY_ { 669 m.dests = append(m.dests, &destination{tr, d.Weight}) 670 } else { 671 // We have a cluster scoped filter. 672 if m.cdests == nil { 673 m.cdests = make(map[string][]*destination) 674 } 675 ad := m.cdests[d.Cluster] 676 ad = append(ad, &destination{tr, d.Weight}) 677 m.cdests[d.Cluster] = ad 678 } 679 } 680 681 processDestinations := func(dests []*destination) ([]*destination, error) { 682 var ltw uint8 683 for _, d := range dests { 684 ltw += d.weight 685 } 686 // Auto add in original at weight difference if all entries weight does not total to 100. 687 // Iff the src was not already added in explicitly, meaning they want loss. 688 _, haveSrc := seen[src] 689 if ltw != 100 && !haveSrc { 690 dest := src 691 if m.wc { 692 // We need to make the appropriate markers for the wildcards etc. 693 dest = transformTokenize(dest) 694 } 695 tr, err := NewSubjectTransform(src, dest) 696 if err != nil { 697 return nil, err 698 } 699 aw := 100 - ltw 700 if len(dests) == 0 { 701 aw = 100 702 } 703 dests = append(dests, &destination{tr, aw}) 704 } 705 sort.Slice(dests, func(i, j int) bool { return dests[i].weight < dests[j].weight }) 706 707 var lw uint8 708 for _, d := range dests { 709 d.weight += lw 710 lw = d.weight 711 } 712 return dests, nil 713 } 714 715 var err error 716 if m.dests, err = processDestinations(m.dests); err != nil { 717 return err 718 } 719 720 // Option cluster scoped destinations 721 for cluster, dests := range m.cdests { 722 if dests, err = processDestinations(dests); err != nil { 723 return err 724 } 725 m.cdests[cluster] = dests 726 } 727 728 // Replace an old one if it exists. 729 for i, em := range a.mappings { 730 if em.src == src { 731 a.mappings[i] = m 732 return nil 733 } 734 } 735 // If we did not replace add to the end. 736 a.mappings = append(a.mappings, m) 737 a.hasMapped.Store(len(a.mappings) > 0) 738 739 // If we have connected leafnodes make sure to update. 740 if a.nleafs > 0 { 741 // Need to release because lock ordering is client -> account 742 a.mu.Unlock() 743 // Now grab the leaf list lock. We can hold client lock under this one. 744 a.lmu.RLock() 745 for _, lc := range a.lleafs { 746 lc.forceAddToSmap(src) 747 } 748 a.lmu.RUnlock() 749 a.mu.Lock() 750 } 751 return nil 752 } 753 754 // RemoveMapping will remove an existing mapping. 755 func (a *Account) RemoveMapping(src string) bool { 756 a.mu.Lock() 757 defer a.mu.Unlock() 758 for i, m := range a.mappings { 759 if m.src == src { 760 // Swap last one into this spot. Its ok to change order. 761 a.mappings[i] = a.mappings[len(a.mappings)-1] 762 a.mappings[len(a.mappings)-1] = nil // gc 763 a.mappings = a.mappings[:len(a.mappings)-1] 764 a.hasMapped.Store(len(a.mappings) > 0) 765 // If we have connected leafnodes make sure to update. 766 if a.nleafs > 0 { 767 // Need to release because lock ordering is client -> account 768 a.mu.Unlock() 769 // Now grab the leaf list lock. We can hold client lock under this one. 770 a.lmu.RLock() 771 for _, lc := range a.lleafs { 772 lc.forceRemoveFromSmap(src) 773 } 774 a.lmu.RUnlock() 775 a.mu.Lock() 776 } 777 return true 778 } 779 } 780 return false 781 } 782 783 // Indicates we have mapping entries. 784 func (a *Account) hasMappings() bool { 785 if a == nil { 786 return false 787 } 788 return a.hasMapped.Load() 789 } 790 791 // This performs the logic to map to a new dest subject based on mappings. 792 // Should only be called from processInboundClientMsg or service import processing. 793 func (a *Account) selectMappedSubject(dest string) (string, bool) { 794 if !a.hasMappings() { 795 return dest, false 796 } 797 798 a.mu.Lock() 799 // In case we have to tokenize for subset matching. 800 tsa := [32]string{} 801 tts := tsa[:0] 802 803 var m *mapping 804 for _, rm := range a.mappings { 805 if !rm.wc && rm.src == dest { 806 m = rm 807 break 808 } else { 809 // tokenize and reuse for subset matching. 810 if len(tts) == 0 { 811 start := 0 812 subject := dest 813 for i := 0; i < len(subject); i++ { 814 if subject[i] == btsep { 815 tts = append(tts, subject[start:i]) 816 start = i + 1 817 } 818 } 819 tts = append(tts, subject[start:]) 820 } 821 if isSubsetMatch(tts, rm.src) { 822 m = rm 823 break 824 } 825 } 826 } 827 828 if m == nil { 829 a.mu.Unlock() 830 return dest, false 831 } 832 833 // The selected destination for the mapping. 834 var d *destination 835 var ndest string 836 837 dests := m.dests 838 if len(m.cdests) > 0 { 839 cn := a.srv.cachedClusterName() 840 dests = m.cdests[cn] 841 if dests == nil { 842 // Fallback to main if we do not match the cluster. 843 dests = m.dests 844 } 845 } 846 847 // Optimize for single entry case. 848 if len(dests) == 1 && dests[0].weight == 100 { 849 d = dests[0] 850 } else { 851 w := uint8(fastrand.Uint32n(100)) 852 for _, rm := range dests { 853 if w < rm.weight { 854 d = rm 855 break 856 } 857 } 858 } 859 860 if d != nil { 861 if len(d.tr.dtokmftokindexesargs) == 0 { 862 ndest = d.tr.dest 863 } else { 864 ndest = d.tr.TransformTokenizedSubject(tts) 865 } 866 } 867 868 a.mu.Unlock() 869 return ndest, true 870 } 871 872 // SubscriptionInterest returns true if this account has a matching subscription 873 // for the given `subject`. 874 func (a *Account) SubscriptionInterest(subject string) bool { 875 return a.Interest(subject) > 0 876 } 877 878 // Interest returns the number of subscriptions for a given subject that match. 879 func (a *Account) Interest(subject string) int { 880 var nms int 881 a.mu.RLock() 882 if a.sl != nil { 883 res := a.sl.Match(subject) 884 nms = len(res.psubs) + len(res.qsubs) 885 } 886 a.mu.RUnlock() 887 return nms 888 } 889 890 // addClient keeps our accounting of local active clients or leafnodes updated. 891 // Returns previous total. 892 func (a *Account) addClient(c *client) int { 893 a.mu.Lock() 894 n := len(a.clients) 895 if a.clients != nil { 896 a.clients[c] = struct{}{} 897 } 898 // If we did not add it, we are done 899 if n == len(a.clients) { 900 a.mu.Unlock() 901 return n 902 } 903 if c.kind != CLIENT && c.kind != LEAF { 904 a.sysclients++ 905 } else if c.kind == LEAF { 906 a.nleafs++ 907 } 908 a.mu.Unlock() 909 910 // If we added a new leaf use the list lock and add it to the list. 911 if c.kind == LEAF { 912 a.lmu.Lock() 913 a.lleafs = append(a.lleafs, c) 914 a.lmu.Unlock() 915 } 916 917 if c != nil && c.srv != nil { 918 c.srv.accConnsUpdate(a) 919 } 920 921 return n 922 } 923 924 // For registering clusters for remote leafnodes. 925 // We only register as the hub. 926 func (a *Account) registerLeafNodeCluster(cluster string) { 927 a.mu.Lock() 928 defer a.mu.Unlock() 929 if a.leafClusters == nil { 930 a.leafClusters = make(map[string]uint64) 931 } 932 a.leafClusters[cluster]++ 933 } 934 935 // Check to see if we already have this cluster registered. 936 func (a *Account) hasLeafNodeCluster(cluster string) bool { 937 a.mu.RLock() 938 defer a.mu.RUnlock() 939 return a.leafClusters[cluster] > 0 940 } 941 942 // Check to see if this cluster is isolated, meaning the only one. 943 // Read Lock should be held. 944 func (a *Account) isLeafNodeClusterIsolated(cluster string) bool { 945 if cluster == _EMPTY_ { 946 return false 947 } 948 if len(a.leafClusters) > 1 { 949 return false 950 } 951 return a.leafClusters[cluster] == uint64(a.nleafs) 952 } 953 954 // Helper function to remove leaf nodes. If number of leafnodes gets large 955 // this may need to be optimized out of linear search but believe number 956 // of active leafnodes per account scope to be small and therefore cache friendly. 957 // Lock should not be held on general account lock. 958 func (a *Account) removeLeafNode(c *client) { 959 // Make sure we hold the list lock as well. 960 a.lmu.Lock() 961 defer a.lmu.Unlock() 962 963 ll := len(a.lleafs) 964 for i, l := range a.lleafs { 965 if l == c { 966 a.lleafs[i] = a.lleafs[ll-1] 967 if ll == 1 { 968 a.lleafs = nil 969 } else { 970 a.lleafs = a.lleafs[:ll-1] 971 } 972 return 973 } 974 } 975 } 976 977 // removeClient keeps our accounting of local active clients updated. 978 func (a *Account) removeClient(c *client) int { 979 a.mu.Lock() 980 n := len(a.clients) 981 delete(a.clients, c) 982 // If we did not actually remove it, we are done. 983 if n == len(a.clients) { 984 a.mu.Unlock() 985 return n 986 } 987 if c.kind != CLIENT && c.kind != LEAF { 988 a.sysclients-- 989 } else if c.kind == LEAF { 990 a.nleafs-- 991 // Need to do cluster accounting here. 992 // Do cluster accounting if we are a hub. 993 if c.isHubLeafNode() { 994 cluster := c.remoteCluster() 995 if count := a.leafClusters[cluster]; count > 1 { 996 a.leafClusters[cluster]-- 997 } else if count == 1 { 998 delete(a.leafClusters, cluster) 999 } 1000 } 1001 } 1002 a.mu.Unlock() 1003 1004 if c.kind == LEAF { 1005 a.removeLeafNode(c) 1006 } 1007 1008 if c != nil && c.srv != nil { 1009 c.srv.accConnsUpdate(a) 1010 } 1011 1012 return n 1013 } 1014 1015 func setExportAuth(ea *exportAuth, subject string, accounts []*Account, accountPos uint) error { 1016 if accountPos > 0 { 1017 token := strings.Split(subject, tsep) 1018 if len(token) < int(accountPos) || token[accountPos-1] != "*" { 1019 return ErrInvalidSubject 1020 } 1021 } 1022 ea.accountPos = accountPos 1023 // empty means auth required but will be import token. 1024 if accounts == nil { 1025 return nil 1026 } 1027 if len(accounts) == 0 { 1028 ea.tokenReq = true 1029 return nil 1030 } 1031 if ea.approved == nil { 1032 ea.approved = make(map[string]*Account, len(accounts)) 1033 } 1034 for _, acc := range accounts { 1035 ea.approved[acc.Name] = acc 1036 } 1037 return nil 1038 } 1039 1040 // AddServiceExport will configure the account with the defined export. 1041 func (a *Account) AddServiceExport(subject string, accounts []*Account) error { 1042 return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, 0) 1043 } 1044 1045 // AddServiceExport will configure the account with the defined export. 1046 func (a *Account) addServiceExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error { 1047 return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, accountPos) 1048 } 1049 1050 // AddServiceExportWithResponse will configure the account with the defined export and response type. 1051 func (a *Account) AddServiceExportWithResponse(subject string, respType ServiceRespType, accounts []*Account) error { 1052 return a.addServiceExportWithResponseAndAccountPos(subject, respType, accounts, 0) 1053 } 1054 1055 // AddServiceExportWithresponse will configure the account with the defined export and response type. 1056 func (a *Account) addServiceExportWithResponseAndAccountPos( 1057 subject string, respType ServiceRespType, accounts []*Account, accountPos uint) error { 1058 if a == nil { 1059 return ErrMissingAccount 1060 } 1061 1062 a.mu.Lock() 1063 if a.exports.services == nil { 1064 a.exports.services = make(map[string]*serviceExport) 1065 } 1066 1067 se := a.exports.services[subject] 1068 // Always create a service export 1069 if se == nil { 1070 se = &serviceExport{} 1071 } 1072 1073 if respType != Singleton { 1074 se.respType = respType 1075 } 1076 1077 if accounts != nil || accountPos > 0 { 1078 if err := setExportAuth(&se.exportAuth, subject, accounts, accountPos); err != nil { 1079 a.mu.Unlock() 1080 return err 1081 } 1082 } 1083 lrt := a.lowestServiceExportResponseTime() 1084 se.acc = a 1085 se.respThresh = DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD 1086 a.exports.services[subject] = se 1087 1088 var clients []*client 1089 nlrt := a.lowestServiceExportResponseTime() 1090 if nlrt != lrt && len(a.clients) > 0 { 1091 clients = a.getClientsLocked() 1092 } 1093 // Need to release because lock ordering is client -> Account 1094 a.mu.Unlock() 1095 if len(clients) > 0 { 1096 updateAllClientsServiceExportResponseTime(clients, nlrt) 1097 } 1098 return nil 1099 } 1100 1101 // TrackServiceExport will enable latency tracking of the named service. 1102 // Results will be published in this account to the given results subject. 1103 func (a *Account) TrackServiceExport(service, results string) error { 1104 return a.TrackServiceExportWithSampling(service, results, DEFAULT_SERVICE_LATENCY_SAMPLING) 1105 } 1106 1107 // TrackServiceExportWithSampling will enable latency tracking of the named service for the given 1108 // sampling rate (1-100). Results will be published in this account to the given results subject. 1109 func (a *Account) TrackServiceExportWithSampling(service, results string, sampling int) error { 1110 if a == nil { 1111 return ErrMissingAccount 1112 } 1113 1114 if sampling != 0 { // 0 means triggered by header 1115 if sampling < 1 || sampling > 100 { 1116 return ErrBadSampling 1117 } 1118 } 1119 if !IsValidPublishSubject(results) { 1120 return ErrBadPublishSubject 1121 } 1122 // Don't loop back on outselves. 1123 if a.IsExportService(results) { 1124 return ErrBadPublishSubject 1125 } 1126 1127 if a.srv != nil && !a.srv.EventsEnabled() { 1128 return ErrNoSysAccount 1129 } 1130 1131 a.mu.Lock() 1132 if a.exports.services == nil { 1133 a.mu.Unlock() 1134 return ErrMissingService 1135 } 1136 ea, ok := a.exports.services[service] 1137 if !ok { 1138 a.mu.Unlock() 1139 return ErrMissingService 1140 } 1141 if ea == nil { 1142 ea = &serviceExport{} 1143 a.exports.services[service] = ea 1144 } else if ea.respType != Singleton { 1145 a.mu.Unlock() 1146 return ErrBadServiceType 1147 } 1148 ea.latency = &serviceLatency{ 1149 sampling: int8(sampling), 1150 subject: results, 1151 } 1152 s := a.srv 1153 a.mu.Unlock() 1154 1155 if s == nil { 1156 return nil 1157 } 1158 1159 // Now track down the imports and add in latency as needed to enable. 1160 s.accounts.Range(func(k, v any) bool { 1161 acc := v.(*Account) 1162 acc.mu.Lock() 1163 for _, im := range acc.imports.services { 1164 if im != nil && im.acc.Name == a.Name && subjectIsSubsetMatch(im.to, service) { 1165 im.latency = ea.latency 1166 } 1167 } 1168 acc.mu.Unlock() 1169 return true 1170 }) 1171 1172 return nil 1173 } 1174 1175 // UnTrackServiceExport will disable latency tracking of the named service. 1176 func (a *Account) UnTrackServiceExport(service string) { 1177 if a == nil || (a.srv != nil && !a.srv.EventsEnabled()) { 1178 return 1179 } 1180 1181 a.mu.Lock() 1182 if a.exports.services == nil { 1183 a.mu.Unlock() 1184 return 1185 } 1186 ea, ok := a.exports.services[service] 1187 if !ok || ea == nil || ea.latency == nil { 1188 a.mu.Unlock() 1189 return 1190 } 1191 // We have latency here. 1192 ea.latency = nil 1193 s := a.srv 1194 a.mu.Unlock() 1195 1196 if s == nil { 1197 return 1198 } 1199 1200 // Now track down the imports and clean them up. 1201 s.accounts.Range(func(k, v any) bool { 1202 acc := v.(*Account) 1203 acc.mu.Lock() 1204 for _, im := range acc.imports.services { 1205 if im != nil && im.acc.Name == a.Name { 1206 if subjectIsSubsetMatch(im.to, service) { 1207 im.latency, im.m1 = nil, nil 1208 } 1209 } 1210 } 1211 acc.mu.Unlock() 1212 return true 1213 }) 1214 } 1215 1216 // IsExportService will indicate if this service exists. Will check wildcard scenarios. 1217 func (a *Account) IsExportService(service string) bool { 1218 a.mu.RLock() 1219 defer a.mu.RUnlock() 1220 _, ok := a.exports.services[service] 1221 if ok { 1222 return true 1223 } 1224 tokens := strings.Split(service, tsep) 1225 for subj := range a.exports.services { 1226 if isSubsetMatch(tokens, subj) { 1227 return true 1228 } 1229 } 1230 return false 1231 } 1232 1233 // IsExportServiceTracking will indicate if given publish subject is an export service with tracking enabled. 1234 func (a *Account) IsExportServiceTracking(service string) bool { 1235 a.mu.RLock() 1236 ea, ok := a.exports.services[service] 1237 if ok && ea == nil { 1238 a.mu.RUnlock() 1239 return false 1240 } 1241 if ok && ea != nil && ea.latency != nil { 1242 a.mu.RUnlock() 1243 return true 1244 } 1245 // FIXME(dlc) - Might want to cache this is in the hot path checking for latency tracking. 1246 tokens := strings.Split(service, tsep) 1247 for subj, ea := range a.exports.services { 1248 if isSubsetMatch(tokens, subj) && ea != nil && ea.latency != nil { 1249 a.mu.RUnlock() 1250 return true 1251 } 1252 } 1253 a.mu.RUnlock() 1254 return false 1255 } 1256 1257 // ServiceLatency is the JSON message sent out in response to latency tracking for 1258 // an accounts exported services. Additional client info is available in requestor 1259 // and responder. Note that for a requestor, the only information shared by default 1260 // is the RTT used to calculate the total latency. The requestor's account can 1261 // designate to share the additional information in the service import. 1262 type ServiceLatency struct { 1263 TypedEvent 1264 Status int `json:"status"` 1265 Error string `json:"description,omitempty"` 1266 Requestor *ClientInfo `json:"requestor,omitempty"` 1267 Responder *ClientInfo `json:"responder,omitempty"` 1268 RequestHeader http.Header `json:"header,omitempty"` // only contains header(s) triggering the measurement 1269 RequestStart time.Time `json:"start"` 1270 ServiceLatency time.Duration `json:"service"` 1271 SystemLatency time.Duration `json:"system"` 1272 TotalLatency time.Duration `json:"total"` 1273 } 1274 1275 // ServiceLatencyType is the NATS Event Type for ServiceLatency 1276 const ServiceLatencyType = "io.nats.server.metric.v1.service_latency" 1277 1278 // NATSTotalTime is a helper function that totals the NATS latencies. 1279 func (m1 *ServiceLatency) NATSTotalTime() time.Duration { 1280 return m1.Requestor.RTT + m1.Responder.RTT + m1.SystemLatency 1281 } 1282 1283 // Merge function to merge m1 and m2 (requestor and responder) measurements 1284 // when there are two samples. This happens when the requestor and responder 1285 // are on different servers. 1286 // 1287 // m2 ServiceLatency is correct, so use that. 1288 // m1 TotalLatency is correct, so use that. 1289 // Will use those to back into NATS latency. 1290 func (m1 *ServiceLatency) merge(m2 *ServiceLatency) { 1291 rtt := time.Duration(0) 1292 if m2.Responder != nil { 1293 rtt = m2.Responder.RTT 1294 } 1295 m1.SystemLatency = m1.ServiceLatency - (m2.ServiceLatency + rtt) 1296 m1.ServiceLatency = m2.ServiceLatency 1297 m1.Responder = m2.Responder 1298 sanitizeLatencyMetric(m1) 1299 } 1300 1301 // sanitizeLatencyMetric adjusts latency metric values that could go 1302 // negative in some edge conditions since we estimate client RTT 1303 // for both requestor and responder. 1304 // These numbers are never meant to be negative, it just could be 1305 // how we back into the values based on estimated RTT. 1306 func sanitizeLatencyMetric(sl *ServiceLatency) { 1307 if sl.ServiceLatency < 0 { 1308 sl.ServiceLatency = 0 1309 } 1310 if sl.SystemLatency < 0 { 1311 sl.SystemLatency = 0 1312 } 1313 } 1314 1315 // Used for transporting remote latency measurements. 1316 type remoteLatency struct { 1317 Account string `json:"account"` 1318 ReqId string `json:"req_id"` 1319 M2 ServiceLatency `json:"m2"` 1320 respThresh time.Duration 1321 } 1322 1323 // sendLatencyResult will send a latency result and clear the si of the requestor(rc). 1324 func (a *Account) sendLatencyResult(si *serviceImport, sl *ServiceLatency) { 1325 sl.Type = ServiceLatencyType 1326 sl.ID = a.nextEventID() 1327 sl.Time = time.Now().UTC() 1328 a.mu.Lock() 1329 lsubj := si.latency.subject 1330 si.rc = nil 1331 a.mu.Unlock() 1332 1333 a.srv.sendInternalAccountMsg(a, lsubj, sl) 1334 } 1335 1336 // Used to send a bad request metric when we do not have a reply subject 1337 func (a *Account) sendBadRequestTrackingLatency(si *serviceImport, requestor *client, header http.Header) { 1338 sl := &ServiceLatency{ 1339 Status: 400, 1340 Error: "Bad Request", 1341 Requestor: requestor.getClientInfo(si.share), 1342 } 1343 sl.RequestHeader = header 1344 sl.RequestStart = time.Now().Add(-sl.Requestor.RTT).UTC() 1345 a.sendLatencyResult(si, sl) 1346 } 1347 1348 // Used to send a latency result when the requestor interest was lost before the 1349 // response could be delivered. 1350 func (a *Account) sendReplyInterestLostTrackLatency(si *serviceImport) { 1351 sl := &ServiceLatency{ 1352 Status: 408, 1353 Error: "Request Timeout", 1354 } 1355 a.mu.RLock() 1356 rc, share, ts := si.rc, si.share, si.ts 1357 sl.RequestHeader = si.trackingHdr 1358 a.mu.RUnlock() 1359 if rc != nil { 1360 sl.Requestor = rc.getClientInfo(share) 1361 } 1362 sl.RequestStart = time.Unix(0, ts-int64(sl.Requestor.RTT)).UTC() 1363 a.sendLatencyResult(si, sl) 1364 } 1365 1366 func (a *Account) sendBackendErrorTrackingLatency(si *serviceImport, reason rsiReason) { 1367 sl := &ServiceLatency{} 1368 a.mu.RLock() 1369 rc, share, ts := si.rc, si.share, si.ts 1370 sl.RequestHeader = si.trackingHdr 1371 a.mu.RUnlock() 1372 if rc != nil { 1373 sl.Requestor = rc.getClientInfo(share) 1374 } 1375 var reqRTT time.Duration 1376 if sl.Requestor != nil { 1377 reqRTT = sl.Requestor.RTT 1378 } 1379 sl.RequestStart = time.Unix(0, ts-int64(reqRTT)).UTC() 1380 if reason == rsiNoDelivery { 1381 sl.Status = 503 1382 sl.Error = "Service Unavailable" 1383 } else if reason == rsiTimeout { 1384 sl.Status = 504 1385 sl.Error = "Service Timeout" 1386 } 1387 a.sendLatencyResult(si, sl) 1388 } 1389 1390 // sendTrackingLatency will send out the appropriate tracking information for the 1391 // service request/response latency. This is called when the requestor's server has 1392 // received the response. 1393 // TODO(dlc) - holding locks for RTTs may be too much long term. Should revisit. 1394 func (a *Account) sendTrackingLatency(si *serviceImport, responder *client) bool { 1395 a.mu.RLock() 1396 rc := si.rc 1397 a.mu.RUnlock() 1398 if rc == nil { 1399 return true 1400 } 1401 1402 ts := time.Now() 1403 serviceRTT := time.Duration(ts.UnixNano() - si.ts) 1404 requestor := si.rc 1405 1406 sl := &ServiceLatency{ 1407 Status: 200, 1408 Requestor: requestor.getClientInfo(si.share), 1409 Responder: responder.getClientInfo(true), 1410 } 1411 var respRTT, reqRTT time.Duration 1412 if sl.Responder != nil { 1413 respRTT = sl.Responder.RTT 1414 } 1415 if sl.Requestor != nil { 1416 reqRTT = sl.Requestor.RTT 1417 } 1418 sl.RequestStart = time.Unix(0, si.ts-int64(reqRTT)).UTC() 1419 sl.ServiceLatency = serviceRTT - respRTT 1420 sl.TotalLatency = reqRTT + serviceRTT 1421 if respRTT > 0 { 1422 sl.SystemLatency = time.Since(ts) 1423 sl.TotalLatency += sl.SystemLatency 1424 } 1425 sl.RequestHeader = si.trackingHdr 1426 sanitizeLatencyMetric(sl) 1427 1428 sl.Type = ServiceLatencyType 1429 sl.ID = a.nextEventID() 1430 sl.Time = time.Now().UTC() 1431 1432 // If we are expecting a remote measurement, store our sl here. 1433 // We need to account for the race between this and us receiving the 1434 // remote measurement. 1435 // FIXME(dlc) - We need to clean these up but this should happen 1436 // already with the auto-expire logic. 1437 if responder != nil && responder.kind != CLIENT { 1438 si.acc.mu.Lock() 1439 if si.m1 != nil { 1440 m1, m2 := sl, si.m1 1441 m1.merge(m2) 1442 si.acc.mu.Unlock() 1443 a.srv.sendInternalAccountMsg(a, si.latency.subject, m1) 1444 a.mu.Lock() 1445 si.rc = nil 1446 a.mu.Unlock() 1447 return true 1448 } 1449 si.m1 = sl 1450 si.acc.mu.Unlock() 1451 return false 1452 } else { 1453 a.srv.sendInternalAccountMsg(a, si.latency.subject, sl) 1454 a.mu.Lock() 1455 si.rc = nil 1456 a.mu.Unlock() 1457 } 1458 return true 1459 } 1460 1461 // This will check to make sure our response lower threshold is set 1462 // properly in any clients doing rrTracking. 1463 func updateAllClientsServiceExportResponseTime(clients []*client, lrt time.Duration) { 1464 for _, c := range clients { 1465 c.mu.Lock() 1466 if c.rrTracking != nil && lrt != c.rrTracking.lrt { 1467 c.rrTracking.lrt = lrt 1468 if c.rrTracking.ptmr.Stop() { 1469 c.rrTracking.ptmr.Reset(lrt) 1470 } 1471 } 1472 c.mu.Unlock() 1473 } 1474 } 1475 1476 // Will select the lowest respThresh from all service exports. 1477 // Read lock should be held. 1478 func (a *Account) lowestServiceExportResponseTime() time.Duration { 1479 // Lowest we will allow is 5 minutes. Its an upper bound for this function. 1480 lrt := 5 * time.Minute 1481 for _, se := range a.exports.services { 1482 if se.respThresh < lrt { 1483 lrt = se.respThresh 1484 } 1485 } 1486 return lrt 1487 } 1488 1489 // AddServiceImportWithClaim will add in the service import via the jwt claim. 1490 func (a *Account) AddServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import) error { 1491 return a.addServiceImportWithClaim(destination, from, to, imClaim, false) 1492 } 1493 1494 // addServiceImportWithClaim will add in the service import via the jwt claim. 1495 // It will also skip the authorization check in cases where internal is true 1496 func (a *Account) addServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import, internal bool) error { 1497 if destination == nil { 1498 return ErrMissingAccount 1499 } 1500 // Empty means use from. 1501 if to == _EMPTY_ { 1502 to = from 1503 } 1504 if !IsValidSubject(from) || !IsValidSubject(to) { 1505 return ErrInvalidSubject 1506 } 1507 1508 // First check to see if the account has authorized us to route to the "to" subject. 1509 if !internal && !destination.checkServiceImportAuthorized(a, to, imClaim) { 1510 return ErrServiceImportAuthorization 1511 } 1512 1513 // Check if this introduces a cycle before proceeding. 1514 if err := a.serviceImportFormsCycle(destination, from); err != nil { 1515 return err 1516 } 1517 1518 _, err := a.addServiceImport(destination, from, to, imClaim) 1519 1520 return err 1521 } 1522 1523 const MaxAccountCycleSearchDepth = 1024 1524 1525 func (a *Account) serviceImportFormsCycle(dest *Account, from string) error { 1526 return dest.checkServiceImportsForCycles(from, map[string]bool{a.Name: true}) 1527 } 1528 1529 func (a *Account) checkServiceImportsForCycles(from string, visited map[string]bool) error { 1530 if len(visited) >= MaxAccountCycleSearchDepth { 1531 return ErrCycleSearchDepth 1532 } 1533 a.mu.RLock() 1534 for _, si := range a.imports.services { 1535 if SubjectsCollide(from, si.to) { 1536 a.mu.RUnlock() 1537 if visited[si.acc.Name] { 1538 return ErrImportFormsCycle 1539 } 1540 // Push ourselves and check si.acc 1541 visited[a.Name] = true 1542 if subjectIsSubsetMatch(si.from, from) { 1543 from = si.from 1544 } 1545 if err := si.acc.checkServiceImportsForCycles(from, visited); err != nil { 1546 return err 1547 } 1548 a.mu.RLock() 1549 } 1550 } 1551 a.mu.RUnlock() 1552 return nil 1553 } 1554 1555 func (a *Account) streamImportFormsCycle(dest *Account, to string) error { 1556 return dest.checkStreamImportsForCycles(to, map[string]bool{a.Name: true}) 1557 } 1558 1559 // Lock should be held. 1560 func (a *Account) hasServiceExportMatching(to string) bool { 1561 for subj := range a.exports.services { 1562 if subjectIsSubsetMatch(to, subj) { 1563 return true 1564 } 1565 } 1566 return false 1567 } 1568 1569 // Lock should be held. 1570 func (a *Account) hasStreamExportMatching(to string) bool { 1571 for subj := range a.exports.streams { 1572 if subjectIsSubsetMatch(to, subj) { 1573 return true 1574 } 1575 } 1576 return false 1577 } 1578 1579 func (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool) error { 1580 if len(visited) >= MaxAccountCycleSearchDepth { 1581 return ErrCycleSearchDepth 1582 } 1583 1584 a.mu.RLock() 1585 1586 if !a.hasStreamExportMatching(to) { 1587 a.mu.RUnlock() 1588 return nil 1589 } 1590 1591 for _, si := range a.imports.streams { 1592 if SubjectsCollide(to, si.to) { 1593 a.mu.RUnlock() 1594 if visited[si.acc.Name] { 1595 return ErrImportFormsCycle 1596 } 1597 // Push ourselves and check si.acc 1598 visited[a.Name] = true 1599 if subjectIsSubsetMatch(si.to, to) { 1600 to = si.to 1601 } 1602 if err := si.acc.checkStreamImportsForCycles(to, visited); err != nil { 1603 return err 1604 } 1605 a.mu.RLock() 1606 } 1607 } 1608 a.mu.RUnlock() 1609 return nil 1610 } 1611 1612 // SetServiceImportSharing will allow sharing of information about requests with the export account. 1613 // Used for service latency tracking at the moment. 1614 func (a *Account) SetServiceImportSharing(destination *Account, to string, allow bool) error { 1615 return a.setServiceImportSharing(destination, to, true, allow) 1616 } 1617 1618 // setServiceImportSharing will allow sharing of information about requests with the export account. 1619 func (a *Account) setServiceImportSharing(destination *Account, to string, check, allow bool) error { 1620 a.mu.Lock() 1621 defer a.mu.Unlock() 1622 if check && a.isClaimAccount() { 1623 return fmt.Errorf("claim based accounts can not be updated directly") 1624 } 1625 for _, si := range a.imports.services { 1626 if si.acc == destination && si.to == to { 1627 si.share = allow 1628 return nil 1629 } 1630 } 1631 return fmt.Errorf("service import not found") 1632 } 1633 1634 // AddServiceImport will add a route to an account to send published messages / requests 1635 // to the destination account. From is the local subject to map, To is the 1636 // subject that will appear on the destination account. Destination will need 1637 // to have an import rule to allow access via addService. 1638 func (a *Account) AddServiceImport(destination *Account, from, to string) error { 1639 return a.AddServiceImportWithClaim(destination, from, to, nil) 1640 } 1641 1642 // NumPendingReverseResponses returns the number of response mappings we have for all outstanding 1643 // requests for service imports. 1644 func (a *Account) NumPendingReverseResponses() int { 1645 a.mu.RLock() 1646 defer a.mu.RUnlock() 1647 return len(a.imports.rrMap) 1648 } 1649 1650 // NumPendingAllResponses return the number of all responses outstanding for service exports. 1651 func (a *Account) NumPendingAllResponses() int { 1652 return a.NumPendingResponses(_EMPTY_) 1653 } 1654 1655 // NumPendingResponses returns the number of responses outstanding for service exports 1656 // on this account. An empty filter string returns all responses regardless of which export. 1657 // If you specify the filter we will only return ones that are for that export. 1658 // NOTE this is only for what this server is tracking. 1659 func (a *Account) NumPendingResponses(filter string) int { 1660 a.mu.RLock() 1661 defer a.mu.RUnlock() 1662 if filter == _EMPTY_ { 1663 return len(a.exports.responses) 1664 } 1665 se := a.getServiceExport(filter) 1666 if se == nil { 1667 return 0 1668 } 1669 var nre int 1670 for _, si := range a.exports.responses { 1671 if si.se == se { 1672 nre++ 1673 } 1674 } 1675 return nre 1676 } 1677 1678 // NumServiceImports returns the number of service imports we have configured. 1679 func (a *Account) NumServiceImports() int { 1680 a.mu.RLock() 1681 defer a.mu.RUnlock() 1682 return len(a.imports.services) 1683 } 1684 1685 // Reason why we are removing this response serviceImport. 1686 type rsiReason int 1687 1688 const ( 1689 rsiOk = rsiReason(iota) 1690 rsiNoDelivery 1691 rsiTimeout 1692 ) 1693 1694 // removeRespServiceImport removes a response si mapping and the reverse entries for interest detection. 1695 func (a *Account) removeRespServiceImport(si *serviceImport, reason rsiReason) { 1696 if si == nil { 1697 return 1698 } 1699 1700 a.mu.Lock() 1701 c := a.ic 1702 delete(a.exports.responses, si.from) 1703 dest, to, tracking, rc, didDeliver := si.acc, si.to, si.tracking, si.rc, si.didDeliver 1704 a.mu.Unlock() 1705 1706 // If we have a sid make sure to unsub. 1707 if len(si.sid) > 0 && c != nil { 1708 c.processUnsub(si.sid) 1709 } 1710 1711 if tracking && rc != nil && !didDeliver { 1712 a.sendBackendErrorTrackingLatency(si, reason) 1713 } 1714 1715 dest.checkForReverseEntry(to, si, false) 1716 } 1717 1718 // removeServiceImport will remove the route by subject. 1719 func (a *Account) removeServiceImport(subject string) { 1720 a.mu.Lock() 1721 si, ok := a.imports.services[subject] 1722 delete(a.imports.services, subject) 1723 1724 var sid []byte 1725 c := a.ic 1726 1727 if ok && si != nil { 1728 if a.ic != nil && si.sid != nil { 1729 sid = si.sid 1730 } 1731 } 1732 a.mu.Unlock() 1733 1734 if sid != nil { 1735 c.processUnsub(sid) 1736 } 1737 } 1738 1739 // This tracks responses to service requests mappings. This is used for cleanup. 1740 func (a *Account) addReverseRespMapEntry(acc *Account, reply, from string) { 1741 a.mu.Lock() 1742 if a.imports.rrMap == nil { 1743 a.imports.rrMap = make(map[string][]*serviceRespEntry) 1744 } 1745 sre := &serviceRespEntry{acc, from} 1746 sra := a.imports.rrMap[reply] 1747 a.imports.rrMap[reply] = append(sra, sre) 1748 a.mu.Unlock() 1749 } 1750 1751 // checkForReverseEntries is for when we are trying to match reverse entries to a wildcard. 1752 // This will be called from checkForReverseEntry when the reply arg is a wildcard subject. 1753 // This will usually be called in a go routine since we need to walk all the entries. 1754 func (a *Account) checkForReverseEntries(reply string, checkInterest, recursed bool) { 1755 if subjectIsLiteral(reply) { 1756 a._checkForReverseEntry(reply, nil, checkInterest, recursed) 1757 return 1758 } 1759 1760 a.mu.RLock() 1761 if len(a.imports.rrMap) == 0 { 1762 a.mu.RUnlock() 1763 return 1764 } 1765 1766 var _rs [64]string 1767 rs := _rs[:0] 1768 if n := len(a.imports.rrMap); n > cap(rs) { 1769 rs = make([]string, 0, n) 1770 } 1771 1772 for k := range a.imports.rrMap { 1773 rs = append(rs, k) 1774 } 1775 a.mu.RUnlock() 1776 1777 tsa := [32]string{} 1778 tts := tokenizeSubjectIntoSlice(tsa[:0], reply) 1779 1780 rsa := [32]string{} 1781 for _, r := range rs { 1782 rts := tokenizeSubjectIntoSlice(rsa[:0], r) 1783 // isSubsetMatchTokenized is heavy so make sure we do this without the lock. 1784 if isSubsetMatchTokenized(rts, tts) { 1785 a._checkForReverseEntry(r, nil, checkInterest, recursed) 1786 } 1787 } 1788 } 1789 1790 // This checks for any response map entries. If you specify an si we will only match and 1791 // clean up for that one, otherwise we remove them all. 1792 func (a *Account) checkForReverseEntry(reply string, si *serviceImport, checkInterest bool) { 1793 a._checkForReverseEntry(reply, si, checkInterest, false) 1794 } 1795 1796 // Callers should use checkForReverseEntry instead. This function exists to help prevent 1797 // infinite recursion. 1798 func (a *Account) _checkForReverseEntry(reply string, si *serviceImport, checkInterest, recursed bool) { 1799 a.mu.RLock() 1800 if len(a.imports.rrMap) == 0 { 1801 a.mu.RUnlock() 1802 return 1803 } 1804 1805 if subjectHasWildcard(reply) { 1806 if recursed { 1807 // If we have reached this condition then it is because the reverse entries also 1808 // contain wildcards (that shouldn't happen but a client *could* provide an inbox 1809 // prefix that is illegal because it ends in a wildcard character), at which point 1810 // we will end up with infinite recursion between this func and checkForReverseEntries. 1811 // To avoid a stack overflow panic, we'll give up instead. 1812 a.mu.RUnlock() 1813 return 1814 } 1815 1816 doInline := len(a.imports.rrMap) <= 64 1817 a.mu.RUnlock() 1818 1819 if doInline { 1820 a.checkForReverseEntries(reply, checkInterest, true) 1821 } else { 1822 go a.checkForReverseEntries(reply, checkInterest, true) 1823 } 1824 return 1825 } 1826 1827 if sres := a.imports.rrMap[reply]; sres == nil { 1828 a.mu.RUnlock() 1829 return 1830 } 1831 1832 // If we are here we have an entry we should check. 1833 // If requested we will first check if there is any 1834 // interest for this subject for the entire account. 1835 // If there is we can not delete any entries yet. 1836 // Note that if we are here reply has to be a literal subject. 1837 if checkInterest { 1838 // If interest still exists we can not clean these up yet. 1839 if rr := a.sl.Match(reply); len(rr.psubs)+len(rr.qsubs) > 0 { 1840 a.mu.RUnlock() 1841 return 1842 } 1843 } 1844 a.mu.RUnlock() 1845 1846 // Delete the appropriate entries here based on optional si. 1847 a.mu.Lock() 1848 // We need a new lookup here because we have released the lock. 1849 sres := a.imports.rrMap[reply] 1850 if si == nil { 1851 delete(a.imports.rrMap, reply) 1852 } else if sres != nil { 1853 // Find the one we are looking for.. 1854 for i, sre := range sres { 1855 if sre.msub == si.from { 1856 sres = append(sres[:i], sres[i+1:]...) 1857 break 1858 } 1859 } 1860 if len(sres) > 0 { 1861 a.imports.rrMap[si.to] = sres 1862 } else { 1863 delete(a.imports.rrMap, si.to) 1864 } 1865 } 1866 a.mu.Unlock() 1867 1868 // If we are here we no longer have interest and we have 1869 // response entries that we should clean up. 1870 if si == nil { 1871 // sres is now known to have been removed from a.imports.rrMap, so we 1872 // can safely (data race wise) iterate through. 1873 for _, sre := range sres { 1874 acc := sre.acc 1875 var trackingCleanup bool 1876 var rsi *serviceImport 1877 acc.mu.Lock() 1878 c := acc.ic 1879 if rsi = acc.exports.responses[sre.msub]; rsi != nil && !rsi.didDeliver { 1880 delete(acc.exports.responses, rsi.from) 1881 trackingCleanup = rsi.tracking && rsi.rc != nil 1882 } 1883 acc.mu.Unlock() 1884 // If we are doing explicit subs for all responses (e.g. bound to leafnode) 1885 // we will have a non-empty sid here. 1886 if rsi != nil && len(rsi.sid) > 0 && c != nil { 1887 c.processUnsub(rsi.sid) 1888 } 1889 if trackingCleanup { 1890 acc.sendReplyInterestLostTrackLatency(rsi) 1891 } 1892 } 1893 } 1894 } 1895 1896 // Checks to see if a potential service import subject is already overshadowed. 1897 func (a *Account) serviceImportShadowed(from string) bool { 1898 a.mu.RLock() 1899 defer a.mu.RUnlock() 1900 if a.imports.services[from] != nil { 1901 return true 1902 } 1903 // We did not find a direct match, so check individually. 1904 for subj := range a.imports.services { 1905 if subjectIsSubsetMatch(from, subj) { 1906 return true 1907 } 1908 } 1909 return false 1910 } 1911 1912 // Internal check to see if a service import exists. 1913 func (a *Account) serviceImportExists(from string) bool { 1914 a.mu.RLock() 1915 dup := a.imports.services[from] 1916 a.mu.RUnlock() 1917 return dup != nil 1918 } 1919 1920 // Add a service import. 1921 // This does no checks and should only be called by the msg processing code. 1922 // Use AddServiceImport from above if responding to user input or config changes, etc. 1923 func (a *Account) addServiceImport(dest *Account, from, to string, claim *jwt.Import) (*serviceImport, error) { 1924 rt := Singleton 1925 var lat *serviceLatency 1926 1927 if dest == nil { 1928 return nil, ErrMissingAccount 1929 } 1930 1931 var atrc bool 1932 dest.mu.RLock() 1933 se := dest.getServiceExport(to) 1934 if se != nil { 1935 rt = se.respType 1936 lat = se.latency 1937 atrc = se.atrc 1938 } 1939 dest.mu.RUnlock() 1940 1941 a.mu.Lock() 1942 if a.imports.services == nil { 1943 a.imports.services = make(map[string]*serviceImport) 1944 } else if dup := a.imports.services[from]; dup != nil { 1945 a.mu.Unlock() 1946 return nil, fmt.Errorf("duplicate service import subject %q, previously used in import for account %q, subject %q", 1947 from, dup.acc.Name, dup.to) 1948 } 1949 1950 if to == _EMPTY_ { 1951 to = from 1952 } 1953 // Check to see if we have a wildcard 1954 var ( 1955 usePub bool 1956 tr *subjectTransform 1957 err error 1958 ) 1959 if subjectHasWildcard(to) { 1960 // If to and from match, then we use the published subject. 1961 if to == from { 1962 usePub = true 1963 } else { 1964 to, _ = transformUntokenize(to) 1965 // Create a transform. Do so in reverse such that $ symbols only exist in to 1966 if tr, err = NewSubjectTransformStrict(to, transformTokenize(from)); err != nil { 1967 a.mu.Unlock() 1968 return nil, fmt.Errorf("failed to create mapping transform for service import subject from %q to %q: %v", 1969 from, to, err) 1970 } else { 1971 // un-tokenize and reverse transform so we get the transform needed 1972 from, _ = transformUntokenize(from) 1973 tr = tr.reverse() 1974 } 1975 } 1976 } 1977 var share bool 1978 if claim != nil { 1979 share = claim.Share 1980 } 1981 si := &serviceImport{dest, claim, se, nil, from, to, tr, 0, rt, lat, nil, nil, usePub, false, false, share, false, false, atrc, nil} 1982 a.imports.services[from] = si 1983 a.mu.Unlock() 1984 1985 if err := a.addServiceImportSub(si); err != nil { 1986 a.removeServiceImport(si.from) 1987 return nil, err 1988 } 1989 return si, nil 1990 } 1991 1992 // Returns the internal client, will create one if not present. 1993 // Lock should be held. 1994 func (a *Account) internalClient() *client { 1995 if a.ic == nil && a.srv != nil { 1996 a.ic = a.srv.createInternalAccountClient() 1997 a.ic.acc = a 1998 } 1999 return a.ic 2000 } 2001 2002 // Internal account scoped subscriptions. 2003 func (a *Account) subscribeInternal(subject string, cb msgHandler) (*subscription, error) { 2004 return a.subscribeInternalEx(subject, cb, false) 2005 } 2006 2007 // Unsubscribe from an internal account subscription. 2008 func (a *Account) unsubscribeInternal(sub *subscription) { 2009 if ic := a.internalClient(); ic != nil { 2010 ic.processUnsub(sub.sid) 2011 } 2012 } 2013 2014 // Creates internal subscription for service import responses. 2015 func (a *Account) subscribeServiceImportResponse(subject string) (*subscription, error) { 2016 return a.subscribeInternalEx(subject, a.processServiceImportResponse, true) 2017 } 2018 2019 func (a *Account) subscribeInternalEx(subject string, cb msgHandler, ri bool) (*subscription, error) { 2020 a.mu.Lock() 2021 a.isid++ 2022 c, sid := a.internalClient(), strconv.FormatUint(a.isid, 10) 2023 a.mu.Unlock() 2024 2025 // This will happen in parsing when the account has not been properly setup. 2026 if c == nil { 2027 return nil, fmt.Errorf("no internal account client") 2028 } 2029 2030 return c.processSubEx([]byte(subject), nil, []byte(sid), cb, false, false, ri) 2031 } 2032 2033 // This will add an account subscription that matches the "from" from a service import entry. 2034 func (a *Account) addServiceImportSub(si *serviceImport) error { 2035 a.mu.Lock() 2036 c := a.internalClient() 2037 // This will happen in parsing when the account has not been properly setup. 2038 if c == nil { 2039 a.mu.Unlock() 2040 return nil 2041 } 2042 if si.sid != nil { 2043 a.mu.Unlock() 2044 return fmt.Errorf("duplicate call to create subscription for service import") 2045 } 2046 a.isid++ 2047 sid := strconv.FormatUint(a.isid, 10) 2048 si.sid = []byte(sid) 2049 subject := si.from 2050 a.mu.Unlock() 2051 2052 cb := func(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) { 2053 c.processServiceImport(si, acc, msg) 2054 } 2055 sub, err := c.processSubEx([]byte(subject), nil, []byte(sid), cb, true, true, false) 2056 if err != nil { 2057 return err 2058 } 2059 // Leafnodes introduce a new way to introduce messages into the system. Therefore forward import subscription 2060 // This is similar to what initLeafNodeSmapAndSendSubs does 2061 // TODO we need to consider performing this update as we get client subscriptions. 2062 // This behavior would result in subscription propagation only where actually used. 2063 a.updateLeafNodes(sub, 1) 2064 return nil 2065 } 2066 2067 // Remove all the subscriptions associated with service imports. 2068 func (a *Account) removeAllServiceImportSubs() { 2069 a.mu.RLock() 2070 var sids [][]byte 2071 for _, si := range a.imports.services { 2072 if si.sid != nil { 2073 sids = append(sids, si.sid) 2074 si.sid = nil 2075 } 2076 } 2077 c := a.ic 2078 a.ic = nil 2079 a.mu.RUnlock() 2080 2081 if c == nil { 2082 return 2083 } 2084 for _, sid := range sids { 2085 c.processUnsub(sid) 2086 } 2087 c.closeConnection(InternalClient) 2088 } 2089 2090 // Add in subscriptions for all registered service imports. 2091 func (a *Account) addAllServiceImportSubs() { 2092 var sis [32]*serviceImport 2093 serviceImports := sis[:0] 2094 a.mu.RLock() 2095 for _, si := range a.imports.services { 2096 serviceImports = append(serviceImports, si) 2097 } 2098 a.mu.RUnlock() 2099 for _, si := range serviceImports { 2100 a.addServiceImportSub(si) 2101 } 2102 } 2103 2104 var ( 2105 // header where all information is encoded in one value. 2106 trcUber = textproto.CanonicalMIMEHeaderKey("Uber-Trace-Id") 2107 trcCtx = textproto.CanonicalMIMEHeaderKey("Traceparent") 2108 trcB3 = textproto.CanonicalMIMEHeaderKey("B3") 2109 // openzipkin header to check 2110 trcB3Sm = textproto.CanonicalMIMEHeaderKey("X-B3-Sampled") 2111 trcB3Id = textproto.CanonicalMIMEHeaderKey("X-B3-TraceId") 2112 // additional header needed to include when present 2113 trcB3PSId = textproto.CanonicalMIMEHeaderKey("X-B3-ParentSpanId") 2114 trcB3SId = textproto.CanonicalMIMEHeaderKey("X-B3-SpanId") 2115 trcCtxSt = textproto.CanonicalMIMEHeaderKey("Tracestate") 2116 trcUberCtxPrefix = textproto.CanonicalMIMEHeaderKey("Uberctx-") 2117 ) 2118 2119 func newB3Header(h http.Header) http.Header { 2120 retHdr := http.Header{} 2121 if v, ok := h[trcB3Sm]; ok { 2122 retHdr[trcB3Sm] = v 2123 } 2124 if v, ok := h[trcB3Id]; ok { 2125 retHdr[trcB3Id] = v 2126 } 2127 if v, ok := h[trcB3PSId]; ok { 2128 retHdr[trcB3PSId] = v 2129 } 2130 if v, ok := h[trcB3SId]; ok { 2131 retHdr[trcB3SId] = v 2132 } 2133 return retHdr 2134 } 2135 2136 func newUberHeader(h http.Header, tId []string) http.Header { 2137 retHdr := http.Header{trcUber: tId} 2138 for k, v := range h { 2139 if strings.HasPrefix(k, trcUberCtxPrefix) { 2140 retHdr[k] = v 2141 } 2142 } 2143 return retHdr 2144 } 2145 2146 func newTraceCtxHeader(h http.Header, tId []string) http.Header { 2147 retHdr := http.Header{trcCtx: tId} 2148 if v, ok := h[trcCtxSt]; ok { 2149 retHdr[trcCtxSt] = v 2150 } 2151 return retHdr 2152 } 2153 2154 // Helper to determine when to sample. When header has a value, sampling is driven by header 2155 func shouldSample(l *serviceLatency, c *client) (bool, http.Header) { 2156 if l == nil { 2157 return false, nil 2158 } 2159 if l.sampling < 0 { 2160 return false, nil 2161 } 2162 if l.sampling >= 100 { 2163 return true, nil 2164 } 2165 if l.sampling > 0 && rand.Int31n(100) <= int32(l.sampling) { 2166 return true, nil 2167 } 2168 h := c.parseState.getHeader() 2169 if len(h) == 0 { 2170 return false, nil 2171 } 2172 if tId := h[trcUber]; len(tId) != 0 { 2173 // sample 479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1 2174 tk := strings.Split(tId[0], ":") 2175 if len(tk) == 4 && len(tk[3]) > 0 && len(tk[3]) <= 2 { 2176 dst := [2]byte{} 2177 src := [2]byte{'0', tk[3][0]} 2178 if len(tk[3]) == 2 { 2179 src[1] = tk[3][1] 2180 } 2181 if _, err := hex.Decode(dst[:], src[:]); err == nil && dst[0]&1 == 1 { 2182 return true, newUberHeader(h, tId) 2183 } 2184 } 2185 return false, nil 2186 } else if sampled := h[trcB3Sm]; len(sampled) != 0 && sampled[0] == "1" { 2187 return true, newB3Header(h) // allowed 2188 } else if len(sampled) != 0 && sampled[0] == "0" { 2189 return false, nil // denied 2190 } else if _, ok := h[trcB3Id]; ok { 2191 // sample 80f198ee56343ba864fe8b2a57d3eff7 2192 // presence (with X-B3-Sampled not being 0) means sampling left to recipient 2193 return true, newB3Header(h) 2194 } else if b3 := h[trcB3]; len(b3) != 0 { 2195 // sample 80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90 2196 // sample 0 2197 tk := strings.Split(b3[0], "-") 2198 if len(tk) > 2 && tk[2] == "0" { 2199 return false, nil // denied 2200 } else if len(tk) == 1 && tk[0] == "0" { 2201 return false, nil // denied 2202 } 2203 return true, http.Header{trcB3: b3} // sampling allowed or left to recipient of header 2204 } else if tId := h[trcCtx]; len(tId) != 0 { 2205 var sample bool 2206 // sample 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01 2207 tk := strings.Split(tId[0], "-") 2208 if len(tk) == 4 && len([]byte(tk[3])) == 2 { 2209 if hexVal, err := strconv.ParseInt(tk[3], 16, 8); err == nil { 2210 sample = hexVal&0x1 == 0x1 2211 } 2212 } 2213 if sample { 2214 return true, newTraceCtxHeader(h, tId) 2215 } else { 2216 return false, nil 2217 } 2218 } 2219 return false, nil 2220 } 2221 2222 // Used to mimic client like replies. 2223 const ( 2224 replyPrefix = "_R_." 2225 replyPrefixLen = len(replyPrefix) 2226 baseServerLen = 10 2227 replyLen = 6 2228 minReplyLen = 15 2229 digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 2230 base = 62 2231 ) 2232 2233 // This is where all service export responses are handled. 2234 func (a *Account) processServiceImportResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { 2235 a.mu.RLock() 2236 if a.expired || len(a.exports.responses) == 0 { 2237 a.mu.RUnlock() 2238 return 2239 } 2240 si := a.exports.responses[subject] 2241 2242 if si == nil || si.invalid { 2243 a.mu.RUnlock() 2244 return 2245 } 2246 a.mu.RUnlock() 2247 2248 // Send for normal processing. 2249 c.processServiceImport(si, a, msg) 2250 } 2251 2252 // Will create the response prefix for fast generation of responses. 2253 // A wildcard subscription may be used handle interest graph propagation 2254 // for all service replies, unless we are bound to a leafnode. 2255 // Lock should be held. 2256 func (a *Account) createRespWildcard() { 2257 var b = [baseServerLen]byte{'_', 'R', '_', '.'} 2258 rn := fastrand.Uint64() 2259 for i, l := replyPrefixLen, rn; i < len(b); i++ { 2260 b[i] = digits[l%base] 2261 l /= base 2262 } 2263 a.siReply = append(b[:], '.') 2264 } 2265 2266 // Test whether this is a tracked reply. 2267 func isTrackedReply(reply []byte) bool { 2268 lreply := len(reply) - 1 2269 return lreply > 3 && reply[lreply-1] == '.' && reply[lreply] == 'T' 2270 } 2271 2272 // Generate a new service reply from the wildcard prefix. 2273 // FIXME(dlc) - probably do not have to use rand here. about 25ns per. 2274 func (a *Account) newServiceReply(tracking bool) []byte { 2275 a.mu.Lock() 2276 s := a.srv 2277 rn := fastrand.Uint64() 2278 2279 // Check if we need to create the reply here. 2280 var createdSiReply bool 2281 if a.siReply == nil { 2282 a.createRespWildcard() 2283 createdSiReply = true 2284 } 2285 replyPre := a.siReply 2286 a.mu.Unlock() 2287 2288 // If we created the siReply and we are not bound to a leafnode 2289 // we need to do the wildcard subscription. 2290 if createdSiReply { 2291 a.subscribeServiceImportResponse(string(append(replyPre, '>'))) 2292 } 2293 2294 var b [replyLen]byte 2295 for i, l := 0, rn; i < len(b); i++ { 2296 b[i] = digits[l%base] 2297 l /= base 2298 } 2299 // Make sure to copy. 2300 reply := make([]byte, 0, len(replyPre)+len(b)) 2301 reply = append(reply, replyPre...) 2302 reply = append(reply, b[:]...) 2303 2304 if tracking && s.sys != nil { 2305 // Add in our tracking identifier. This allows the metrics to get back to only 2306 // this server without needless SUBS/UNSUBS. 2307 reply = append(reply, '.') 2308 reply = append(reply, s.sys.shash...) 2309 reply = append(reply, '.', 'T') 2310 } 2311 2312 return reply 2313 } 2314 2315 // Checks if a serviceImport was created to map responses. 2316 func (si *serviceImport) isRespServiceImport() bool { 2317 return si != nil && si.response 2318 } 2319 2320 // Sets the response threshold timer for a service export. 2321 // Account lock should be held 2322 func (se *serviceExport) setResponseThresholdTimer() { 2323 if se.rtmr != nil { 2324 return // Already set 2325 } 2326 se.rtmr = time.AfterFunc(se.respThresh, se.checkExpiredResponses) 2327 } 2328 2329 // Account lock should be held 2330 func (se *serviceExport) clearResponseThresholdTimer() bool { 2331 if se.rtmr == nil { 2332 return true 2333 } 2334 stopped := se.rtmr.Stop() 2335 se.rtmr = nil 2336 return stopped 2337 } 2338 2339 // checkExpiredResponses will check for any pending responses that need to 2340 // be cleaned up. 2341 func (se *serviceExport) checkExpiredResponses() { 2342 acc := se.acc 2343 if acc == nil { 2344 se.clearResponseThresholdTimer() 2345 return 2346 } 2347 2348 var expired []*serviceImport 2349 mints := time.Now().UnixNano() - int64(se.respThresh) 2350 2351 // TODO(dlc) - Should we release lock while doing this? Or only do these in batches? 2352 // Should we break this up for responses only from this service export? 2353 // Responses live on acc directly for fast inbound processsing for the _R_ wildcard. 2354 // We could do another indirection at this level but just to get to the service export? 2355 var totalResponses int 2356 acc.mu.RLock() 2357 for _, si := range acc.exports.responses { 2358 if si.se == se { 2359 totalResponses++ 2360 if si.ts <= mints { 2361 expired = append(expired, si) 2362 } 2363 } 2364 } 2365 acc.mu.RUnlock() 2366 2367 for _, si := range expired { 2368 acc.removeRespServiceImport(si, rsiTimeout) 2369 } 2370 2371 // Pull out expired to determine if we have any left for timer. 2372 totalResponses -= len(expired) 2373 2374 // Redo timer as needed. 2375 acc.mu.Lock() 2376 if totalResponses > 0 && se.rtmr != nil { 2377 se.rtmr.Stop() 2378 se.rtmr.Reset(se.respThresh) 2379 } else { 2380 se.clearResponseThresholdTimer() 2381 } 2382 acc.mu.Unlock() 2383 } 2384 2385 // ServiceExportResponseThreshold returns the current threshold. 2386 func (a *Account) ServiceExportResponseThreshold(export string) (time.Duration, error) { 2387 a.mu.Lock() 2388 defer a.mu.Unlock() 2389 se := a.getServiceExport(export) 2390 if se == nil { 2391 return 0, fmt.Errorf("no export defined for %q", export) 2392 } 2393 return se.respThresh, nil 2394 } 2395 2396 // SetServiceExportResponseThreshold sets the maximum time the system will a response to be delivered 2397 // from a service export responder. 2398 func (a *Account) SetServiceExportResponseThreshold(export string, maxTime time.Duration) error { 2399 a.mu.Lock() 2400 if a.isClaimAccount() { 2401 a.mu.Unlock() 2402 return fmt.Errorf("claim based accounts can not be updated directly") 2403 } 2404 lrt := a.lowestServiceExportResponseTime() 2405 se := a.getServiceExport(export) 2406 if se == nil { 2407 a.mu.Unlock() 2408 return fmt.Errorf("no export defined for %q", export) 2409 } 2410 se.respThresh = maxTime 2411 2412 var clients []*client 2413 nlrt := a.lowestServiceExportResponseTime() 2414 if nlrt != lrt && len(a.clients) > 0 { 2415 clients = a.getClientsLocked() 2416 } 2417 // Need to release because lock ordering is client -> Account 2418 a.mu.Unlock() 2419 if len(clients) > 0 { 2420 updateAllClientsServiceExportResponseTime(clients, nlrt) 2421 } 2422 return nil 2423 } 2424 2425 func (a *Account) SetServiceExportAllowTrace(export string, allowTrace bool) error { 2426 a.mu.Lock() 2427 se := a.getServiceExport(export) 2428 if se == nil { 2429 a.mu.Unlock() 2430 return fmt.Errorf("no export defined for %q", export) 2431 } 2432 se.atrc = allowTrace 2433 a.mu.Unlock() 2434 return nil 2435 } 2436 2437 // This is for internal service import responses. 2438 func (a *Account) addRespServiceImport(dest *Account, to string, osi *serviceImport, tracking bool, header http.Header) *serviceImport { 2439 nrr := string(osi.acc.newServiceReply(tracking)) 2440 2441 a.mu.Lock() 2442 rt := osi.rt 2443 2444 // dest is the requestor's account. a is the service responder with the export. 2445 // Marked as internal here, that is how we distinguish. 2446 si := &serviceImport{dest, nil, osi.se, nil, nrr, to, nil, 0, rt, nil, nil, nil, false, true, false, osi.share, false, false, false, nil} 2447 2448 if a.exports.responses == nil { 2449 a.exports.responses = make(map[string]*serviceImport) 2450 } 2451 a.exports.responses[nrr] = si 2452 2453 // Always grab time and make sure response threshold timer is running. 2454 si.ts = time.Now().UnixNano() 2455 if osi.se != nil { 2456 osi.se.setResponseThresholdTimer() 2457 } 2458 2459 if rt == Singleton && tracking { 2460 si.latency = osi.latency 2461 si.tracking = true 2462 si.trackingHdr = header 2463 } 2464 a.mu.Unlock() 2465 2466 // We do add in the reverse map such that we can detect loss of interest and do proper 2467 // cleanup of this si as interest goes away. 2468 dest.addReverseRespMapEntry(a, to, nrr) 2469 2470 return si 2471 } 2472 2473 // AddStreamImportWithClaim will add in the stream import from a specific account with optional token. 2474 func (a *Account) AddStreamImportWithClaim(account *Account, from, prefix string, imClaim *jwt.Import) error { 2475 return a.addStreamImportWithClaim(account, from, prefix, false, imClaim) 2476 } 2477 2478 func (a *Account) addStreamImportWithClaim(account *Account, from, prefix string, allowTrace bool, imClaim *jwt.Import) error { 2479 if account == nil { 2480 return ErrMissingAccount 2481 } 2482 2483 // First check to see if the account has authorized export of the subject. 2484 if !account.checkStreamImportAuthorized(a, from, imClaim) { 2485 return ErrStreamImportAuthorization 2486 } 2487 2488 // Check prefix if it exists and make sure its a literal. 2489 // Append token separator if not already present. 2490 if prefix != _EMPTY_ { 2491 // Make sure there are no wildcards here, this prefix needs to be a literal 2492 // since it will be prepended to a publish subject. 2493 if !subjectIsLiteral(prefix) { 2494 return ErrStreamImportBadPrefix 2495 } 2496 if prefix[len(prefix)-1] != btsep { 2497 prefix = prefix + string(btsep) 2498 } 2499 } 2500 2501 return a.addMappedStreamImportWithClaim(account, from, prefix+from, allowTrace, imClaim) 2502 } 2503 2504 // AddMappedStreamImport helper for AddMappedStreamImportWithClaim 2505 func (a *Account) AddMappedStreamImport(account *Account, from, to string) error { 2506 return a.AddMappedStreamImportWithClaim(account, from, to, nil) 2507 } 2508 2509 // AddMappedStreamImportWithClaim will add in the stream import from a specific account with optional token. 2510 func (a *Account) AddMappedStreamImportWithClaim(account *Account, from, to string, imClaim *jwt.Import) error { 2511 return a.addMappedStreamImportWithClaim(account, from, to, false, imClaim) 2512 } 2513 2514 func (a *Account) addMappedStreamImportWithClaim(account *Account, from, to string, allowTrace bool, imClaim *jwt.Import) error { 2515 if account == nil { 2516 return ErrMissingAccount 2517 } 2518 2519 // First check to see if the account has authorized export of the subject. 2520 if !account.checkStreamImportAuthorized(a, from, imClaim) { 2521 return ErrStreamImportAuthorization 2522 } 2523 2524 if to == _EMPTY_ { 2525 to = from 2526 } 2527 2528 // Check if this forms a cycle. 2529 if err := a.streamImportFormsCycle(account, to); err != nil { 2530 return err 2531 } 2532 2533 var ( 2534 usePub bool 2535 tr *subjectTransform 2536 err error 2537 ) 2538 if subjectHasWildcard(from) { 2539 if to == from { 2540 usePub = true 2541 } else { 2542 // Create a transform 2543 if tr, err = NewSubjectTransformStrict(from, transformTokenize(to)); err != nil { 2544 return fmt.Errorf("failed to create mapping transform for stream import subject from %q to %q: %v", 2545 from, to, err) 2546 } 2547 to, _ = transformUntokenize(to) 2548 } 2549 } 2550 2551 a.mu.Lock() 2552 if a.isStreamImportDuplicate(account, from) { 2553 a.mu.Unlock() 2554 return ErrStreamImportDuplicate 2555 } 2556 if imClaim != nil { 2557 allowTrace = imClaim.AllowTrace 2558 } 2559 a.imports.streams = append(a.imports.streams, &streamImport{account, from, to, tr, nil, imClaim, usePub, false, allowTrace}) 2560 a.mu.Unlock() 2561 return nil 2562 } 2563 2564 // isStreamImportDuplicate checks for duplicate. 2565 // Lock should be held. 2566 func (a *Account) isStreamImportDuplicate(acc *Account, from string) bool { 2567 for _, si := range a.imports.streams { 2568 if si.acc == acc && si.from == from { 2569 return true 2570 } 2571 } 2572 return false 2573 } 2574 2575 // AddStreamImport will add in the stream import from a specific account. 2576 func (a *Account) AddStreamImport(account *Account, from, prefix string) error { 2577 return a.addStreamImportWithClaim(account, from, prefix, false, nil) 2578 } 2579 2580 // IsPublicExport is a placeholder to denote a public export. 2581 var IsPublicExport = []*Account(nil) 2582 2583 // AddStreamExport will add an export to the account. If accounts is nil 2584 // it will signify a public export, meaning anyone can import. 2585 func (a *Account) AddStreamExport(subject string, accounts []*Account) error { 2586 return a.addStreamExportWithAccountPos(subject, accounts, 0) 2587 } 2588 2589 // AddStreamExport will add an export to the account. If accounts is nil 2590 // it will signify a public export, meaning anyone can import. 2591 // if accountPos is > 0, all imports will be granted where the following holds: 2592 // strings.Split(subject, tsep)[accountPos] == account id will be granted. 2593 func (a *Account) addStreamExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error { 2594 if a == nil { 2595 return ErrMissingAccount 2596 } 2597 2598 a.mu.Lock() 2599 defer a.mu.Unlock() 2600 2601 if a.exports.streams == nil { 2602 a.exports.streams = make(map[string]*streamExport) 2603 } 2604 ea := a.exports.streams[subject] 2605 if accounts != nil || accountPos > 0 { 2606 if ea == nil { 2607 ea = &streamExport{} 2608 } 2609 if err := setExportAuth(&ea.exportAuth, subject, accounts, accountPos); err != nil { 2610 return err 2611 } 2612 } 2613 a.exports.streams[subject] = ea 2614 return nil 2615 } 2616 2617 // Check if another account is authorized to import from us. 2618 func (a *Account) checkStreamImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool { 2619 // Find the subject in the exports list. 2620 a.mu.RLock() 2621 auth := a.checkStreamImportAuthorizedNoLock(account, subject, imClaim) 2622 a.mu.RUnlock() 2623 return auth 2624 } 2625 2626 func (a *Account) checkStreamImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool { 2627 if a.exports.streams == nil || !IsValidSubject(subject) { 2628 return false 2629 } 2630 return a.checkStreamExportApproved(account, subject, imClaim) 2631 } 2632 2633 func (a *Account) checkAuth(ea *exportAuth, account *Account, imClaim *jwt.Import, tokens []string) bool { 2634 // if ea is nil or ea.approved is nil, that denotes a public export 2635 if ea == nil || (len(ea.approved) == 0 && !ea.tokenReq && ea.accountPos == 0) { 2636 return true 2637 } 2638 // Check if the export is protected and enforces presence of importing account identity 2639 if ea.accountPos > 0 { 2640 return ea.accountPos <= uint(len(tokens)) && tokens[ea.accountPos-1] == account.Name 2641 } 2642 // Check if token required 2643 if ea.tokenReq { 2644 return a.checkActivation(account, imClaim, ea, true) 2645 } 2646 if ea.approved == nil { 2647 return false 2648 } 2649 // If we have a matching account we are authorized 2650 _, ok := ea.approved[account.Name] 2651 return ok 2652 } 2653 2654 func (a *Account) checkStreamExportApproved(account *Account, subject string, imClaim *jwt.Import) bool { 2655 // Check direct match of subject first 2656 ea, ok := a.exports.streams[subject] 2657 if ok { 2658 // if ea is nil or eq.approved is nil, that denotes a public export 2659 if ea == nil { 2660 return true 2661 } 2662 return a.checkAuth(&ea.exportAuth, account, imClaim, nil) 2663 } 2664 2665 // ok if we are here we did not match directly so we need to test each one. 2666 // The import subject arg has to take precedence, meaning the export 2667 // has to be a true subset of the import claim. We already checked for 2668 // exact matches above. 2669 tokens := strings.Split(subject, tsep) 2670 for subj, ea := range a.exports.streams { 2671 if isSubsetMatch(tokens, subj) { 2672 if ea == nil { 2673 return true 2674 } 2675 return a.checkAuth(&ea.exportAuth, account, imClaim, tokens) 2676 } 2677 } 2678 return false 2679 } 2680 2681 func (a *Account) checkServiceExportApproved(account *Account, subject string, imClaim *jwt.Import) bool { 2682 // Check direct match of subject first 2683 se, ok := a.exports.services[subject] 2684 if ok { 2685 // if se is nil or eq.approved is nil, that denotes a public export 2686 if se == nil { 2687 return true 2688 } 2689 return a.checkAuth(&se.exportAuth, account, imClaim, nil) 2690 } 2691 // ok if we are here we did not match directly so we need to test each one. 2692 // The import subject arg has to take precedence, meaning the export 2693 // has to be a true subset of the import claim. We already checked for 2694 // exact matches above. 2695 tokens := strings.Split(subject, tsep) 2696 for subj, se := range a.exports.services { 2697 if isSubsetMatch(tokens, subj) { 2698 if se == nil { 2699 return true 2700 } 2701 return a.checkAuth(&se.exportAuth, account, imClaim, tokens) 2702 } 2703 } 2704 return false 2705 } 2706 2707 // Helper function to get a serviceExport. 2708 // Lock should be held on entry. 2709 func (a *Account) getServiceExport(subj string) *serviceExport { 2710 se, ok := a.exports.services[subj] 2711 // The export probably has a wildcard, so lookup that up. 2712 if !ok { 2713 se = a.getWildcardServiceExport(subj) 2714 } 2715 return se 2716 } 2717 2718 // This helper is used when trying to match a serviceExport record that is 2719 // represented by a wildcard. 2720 // Lock should be held on entry. 2721 func (a *Account) getWildcardServiceExport(from string) *serviceExport { 2722 tokens := strings.Split(from, tsep) 2723 for subj, se := range a.exports.services { 2724 if isSubsetMatch(tokens, subj) { 2725 return se 2726 } 2727 } 2728 return nil 2729 } 2730 2731 // These are import stream specific versions for when an activation expires. 2732 func (a *Account) streamActivationExpired(exportAcc *Account, subject string) { 2733 a.mu.RLock() 2734 if a.expired || a.imports.streams == nil { 2735 a.mu.RUnlock() 2736 return 2737 } 2738 var si *streamImport 2739 for _, si = range a.imports.streams { 2740 if si.acc == exportAcc && si.from == subject { 2741 break 2742 } 2743 } 2744 2745 if si == nil || si.invalid { 2746 a.mu.RUnlock() 2747 return 2748 } 2749 a.mu.RUnlock() 2750 2751 if si.acc.checkActivation(a, si.claim, nil, false) { 2752 // The token has been updated most likely and we are good to go. 2753 return 2754 } 2755 2756 a.mu.Lock() 2757 si.invalid = true 2758 clients := a.getClientsLocked() 2759 awcsti := map[string]struct{}{a.Name: {}} 2760 a.mu.Unlock() 2761 for _, c := range clients { 2762 c.processSubsOnConfigReload(awcsti) 2763 } 2764 } 2765 2766 // These are import service specific versions for when an activation expires. 2767 func (a *Account) serviceActivationExpired(subject string) { 2768 a.mu.RLock() 2769 if a.expired || a.imports.services == nil { 2770 a.mu.RUnlock() 2771 return 2772 } 2773 si := a.imports.services[subject] 2774 if si == nil || si.invalid { 2775 a.mu.RUnlock() 2776 return 2777 } 2778 a.mu.RUnlock() 2779 2780 if si.acc.checkActivation(a, si.claim, nil, false) { 2781 // The token has been updated most likely and we are good to go. 2782 return 2783 } 2784 2785 a.mu.Lock() 2786 si.invalid = true 2787 a.mu.Unlock() 2788 } 2789 2790 // Fires for expired activation tokens. We could track this with timers etc. 2791 // Instead we just re-analyze where we are and if we need to act. 2792 func (a *Account) activationExpired(exportAcc *Account, subject string, kind jwt.ExportType) { 2793 switch kind { 2794 case jwt.Stream: 2795 a.streamActivationExpired(exportAcc, subject) 2796 case jwt.Service: 2797 a.serviceActivationExpired(subject) 2798 } 2799 } 2800 2801 func isRevoked(revocations map[string]int64, subject string, issuedAt int64) bool { 2802 if len(revocations) == 0 { 2803 return false 2804 } 2805 if t, ok := revocations[subject]; !ok || t < issuedAt { 2806 if t, ok := revocations[jwt.All]; !ok || t < issuedAt { 2807 return false 2808 } 2809 } 2810 return true 2811 } 2812 2813 // checkActivation will check the activation token for validity. 2814 // ea may only be nil in cases where revocation may not be checked, say triggered by expiration timer. 2815 func (a *Account) checkActivation(importAcc *Account, claim *jwt.Import, ea *exportAuth, expTimer bool) bool { 2816 if claim == nil || claim.Token == _EMPTY_ { 2817 return false 2818 } 2819 // Create a quick clone so we can inline Token JWT. 2820 clone := *claim 2821 2822 vr := jwt.CreateValidationResults() 2823 clone.Validate(importAcc.Name, vr) 2824 if vr.IsBlocking(true) { 2825 return false 2826 } 2827 act, err := jwt.DecodeActivationClaims(clone.Token) 2828 if err != nil { 2829 return false 2830 } 2831 if !a.isIssuerClaimTrusted(act) { 2832 return false 2833 } 2834 vr = jwt.CreateValidationResults() 2835 act.Validate(vr) 2836 if vr.IsBlocking(true) { 2837 return false 2838 } 2839 if act.Expires != 0 { 2840 tn := time.Now().Unix() 2841 if act.Expires <= tn { 2842 return false 2843 } 2844 if expTimer { 2845 expiresAt := time.Duration(act.Expires - tn) 2846 time.AfterFunc(expiresAt*time.Second, func() { 2847 importAcc.activationExpired(a, string(act.ImportSubject), claim.Type) 2848 }) 2849 } 2850 } 2851 if ea == nil { 2852 return true 2853 } 2854 // Check for token revocation.. 2855 return !isRevoked(ea.actsRevoked, act.Subject, act.IssuedAt) 2856 } 2857 2858 // Returns true if the activation claim is trusted. That is the issuer matches 2859 // the account or is an entry in the signing keys. 2860 func (a *Account) isIssuerClaimTrusted(claims *jwt.ActivationClaims) bool { 2861 // if no issuer account, issuer is the account 2862 if claims.IssuerAccount == _EMPTY_ { 2863 return true 2864 } 2865 // If the IssuerAccount is not us, then this is considered an error. 2866 if a.Name != claims.IssuerAccount { 2867 if a.srv != nil { 2868 a.srv.Errorf("Invalid issuer account %q in activation claim (subject: %q - type: %q) for account %q", 2869 claims.IssuerAccount, claims.Activation.ImportSubject, claims.Activation.ImportType, a.Name) 2870 } 2871 return false 2872 } 2873 _, ok := a.hasIssuerNoLock(claims.Issuer) 2874 return ok 2875 } 2876 2877 // Returns true if `a` and `b` stream imports are the same. Note that the 2878 // check is done with the account's name, not the pointer. This is used 2879 // during config reload where we are comparing current and new config 2880 // in which pointers are different. 2881 // No lock is acquired in this function, so it is assumed that the 2882 // import maps are not changed while this executes. 2883 func (a *Account) checkStreamImportsEqual(b *Account) bool { 2884 if len(a.imports.streams) != len(b.imports.streams) { 2885 return false 2886 } 2887 // Load the b imports into a map index by what we are looking for. 2888 bm := make(map[string]*streamImport, len(b.imports.streams)) 2889 for _, bim := range b.imports.streams { 2890 bm[bim.acc.Name+bim.from+bim.to] = bim 2891 } 2892 for _, aim := range a.imports.streams { 2893 if bim, ok := bm[aim.acc.Name+aim.from+aim.to]; !ok { 2894 return false 2895 } else if aim.atrc != bim.atrc { 2896 return false 2897 } 2898 } 2899 return true 2900 } 2901 2902 // Returns true if `a` and `b` stream exports are the same. 2903 // Acquires `a` read lock, but `b` is assumed to not be accessed 2904 // by anyone but the caller (`b` is not registered anywhere). 2905 func (a *Account) checkStreamExportsEqual(b *Account) bool { 2906 a.mu.RLock() 2907 defer a.mu.RUnlock() 2908 if len(a.exports.streams) != len(b.exports.streams) { 2909 return false 2910 } 2911 for subj, aea := range a.exports.streams { 2912 bea, ok := b.exports.streams[subj] 2913 if !ok { 2914 return false 2915 } 2916 if !isStreamExportEqual(aea, bea) { 2917 return false 2918 } 2919 } 2920 return true 2921 } 2922 2923 func isStreamExportEqual(a, b *streamExport) bool { 2924 if a == nil && b == nil { 2925 return true 2926 } 2927 if (a == nil && b != nil) || (a != nil && b == nil) { 2928 return false 2929 } 2930 return isExportAuthEqual(&a.exportAuth, &b.exportAuth) 2931 } 2932 2933 // Returns true if `a` and `b` service exports are the same. 2934 // Acquires `a` read lock, but `b` is assumed to not be accessed 2935 // by anyone but the caller (`b` is not registered anywhere). 2936 func (a *Account) checkServiceExportsEqual(b *Account) bool { 2937 a.mu.RLock() 2938 defer a.mu.RUnlock() 2939 if len(a.exports.services) != len(b.exports.services) { 2940 return false 2941 } 2942 for subj, aea := range a.exports.services { 2943 bea, ok := b.exports.services[subj] 2944 if !ok { 2945 return false 2946 } 2947 if !isServiceExportEqual(aea, bea) { 2948 return false 2949 } 2950 } 2951 return true 2952 } 2953 2954 func isServiceExportEqual(a, b *serviceExport) bool { 2955 if a == nil && b == nil { 2956 return true 2957 } 2958 if (a == nil && b != nil) || (a != nil && b == nil) { 2959 return false 2960 } 2961 if !isExportAuthEqual(&a.exportAuth, &b.exportAuth) { 2962 return false 2963 } 2964 if a.acc.Name != b.acc.Name { 2965 return false 2966 } 2967 if a.respType != b.respType { 2968 return false 2969 } 2970 if a.latency != nil || b.latency != nil { 2971 if (a.latency != nil && b.latency == nil) || (a.latency == nil && b.latency != nil) { 2972 return false 2973 } 2974 if a.latency.sampling != b.latency.sampling { 2975 return false 2976 } 2977 if a.latency.subject != b.latency.subject { 2978 return false 2979 } 2980 } 2981 if a.atrc != b.atrc { 2982 return false 2983 } 2984 return true 2985 } 2986 2987 // Returns true if `a` and `b` exportAuth structures are equal. 2988 // Both `a` and `b` are guaranteed to be non-nil. 2989 // Locking is handled by the caller. 2990 func isExportAuthEqual(a, b *exportAuth) bool { 2991 if a.tokenReq != b.tokenReq { 2992 return false 2993 } 2994 if a.accountPos != b.accountPos { 2995 return false 2996 } 2997 if len(a.approved) != len(b.approved) { 2998 return false 2999 } 3000 for ak, av := range a.approved { 3001 if bv, ok := b.approved[ak]; !ok || av.Name != bv.Name { 3002 return false 3003 } 3004 } 3005 if len(a.actsRevoked) != len(b.actsRevoked) { 3006 return false 3007 } 3008 for ak, av := range a.actsRevoked { 3009 if bv, ok := b.actsRevoked[ak]; !ok || av != bv { 3010 return false 3011 } 3012 } 3013 return true 3014 } 3015 3016 // Check if another account is authorized to route requests to this service. 3017 func (a *Account) checkServiceImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool { 3018 a.mu.RLock() 3019 authorized := a.checkServiceImportAuthorizedNoLock(account, subject, imClaim) 3020 a.mu.RUnlock() 3021 return authorized 3022 } 3023 3024 // Check if another account is authorized to route requests to this service. 3025 func (a *Account) checkServiceImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool { 3026 // Find the subject in the services list. 3027 if a.exports.services == nil { 3028 return false 3029 } 3030 return a.checkServiceExportApproved(account, subject, imClaim) 3031 } 3032 3033 // IsExpired returns expiration status. 3034 func (a *Account) IsExpired() bool { 3035 a.mu.RLock() 3036 exp := a.expired 3037 a.mu.RUnlock() 3038 return exp 3039 } 3040 3041 // Called when an account has expired. 3042 func (a *Account) expiredTimeout() { 3043 // Mark expired first. 3044 a.mu.Lock() 3045 a.expired = true 3046 a.mu.Unlock() 3047 3048 // Collect the clients and expire them. 3049 cs := a.getClients() 3050 for _, c := range cs { 3051 c.accountAuthExpired() 3052 } 3053 } 3054 3055 // Sets the expiration timer for an account JWT that has it set. 3056 func (a *Account) setExpirationTimer(d time.Duration) { 3057 a.etmr = time.AfterFunc(d, a.expiredTimeout) 3058 } 3059 3060 // Lock should be held 3061 func (a *Account) clearExpirationTimer() bool { 3062 if a.etmr == nil { 3063 return true 3064 } 3065 stopped := a.etmr.Stop() 3066 a.etmr = nil 3067 return stopped 3068 } 3069 3070 // checkUserRevoked will check if a user has been revoked. 3071 func (a *Account) checkUserRevoked(nkey string, issuedAt int64) bool { 3072 a.mu.RLock() 3073 defer a.mu.RUnlock() 3074 return isRevoked(a.usersRevoked, nkey, issuedAt) 3075 } 3076 3077 // failBearer will return if bearer token are allowed (false) or disallowed (true) 3078 func (a *Account) failBearer() bool { 3079 a.mu.RLock() 3080 defer a.mu.RUnlock() 3081 return a.disallowBearer 3082 } 3083 3084 // Check expiration and set the proper state as needed. 3085 func (a *Account) checkExpiration(claims *jwt.ClaimsData) { 3086 a.mu.Lock() 3087 defer a.mu.Unlock() 3088 3089 a.clearExpirationTimer() 3090 if claims.Expires == 0 { 3091 a.expired = false 3092 return 3093 } 3094 tn := time.Now().Unix() 3095 if claims.Expires <= tn { 3096 a.expired = true 3097 return 3098 } 3099 expiresAt := time.Duration(claims.Expires - tn) 3100 a.setExpirationTimer(expiresAt * time.Second) 3101 a.expired = false 3102 } 3103 3104 // hasIssuer returns true if the issuer matches the account 3105 // If the issuer is a scoped signing key, the scope will be returned as well 3106 // issuer or it is a signing key for the account. 3107 func (a *Account) hasIssuer(issuer string) (jwt.Scope, bool) { 3108 a.mu.RLock() 3109 scope, ok := a.hasIssuerNoLock(issuer) 3110 a.mu.RUnlock() 3111 return scope, ok 3112 } 3113 3114 // hasIssuerNoLock is the unlocked version of hasIssuer 3115 func (a *Account) hasIssuerNoLock(issuer string) (jwt.Scope, bool) { 3116 scope, ok := a.signingKeys[issuer] 3117 return scope, ok 3118 } 3119 3120 // Returns the loop detection subject used for leafnodes 3121 func (a *Account) getLDSubject() string { 3122 a.mu.RLock() 3123 lds := a.lds 3124 a.mu.RUnlock() 3125 return lds 3126 } 3127 3128 // Placeholder for signaling token auth required. 3129 var tokenAuthReq = []*Account{} 3130 3131 func authAccounts(tokenReq bool) []*Account { 3132 if tokenReq { 3133 return tokenAuthReq 3134 } 3135 return nil 3136 } 3137 3138 // SetAccountResolver will assign the account resolver. 3139 func (s *Server) SetAccountResolver(ar AccountResolver) { 3140 s.mu.Lock() 3141 s.accResolver = ar 3142 s.mu.Unlock() 3143 } 3144 3145 // AccountResolver returns the registered account resolver. 3146 func (s *Server) AccountResolver() AccountResolver { 3147 s.mu.Lock() 3148 ar := s.accResolver 3149 s.mu.Unlock() 3150 return ar 3151 } 3152 3153 // isClaimAccount returns if this account is backed by a JWT claim. 3154 // Lock should be held. 3155 func (a *Account) isClaimAccount() bool { 3156 return a.claimJWT != _EMPTY_ 3157 } 3158 3159 // UpdateAccountClaims will update an existing account with new claims. 3160 // This will replace any exports or imports previously defined. 3161 // Lock MUST NOT be held upon entry. 3162 func (s *Server) UpdateAccountClaims(a *Account, ac *jwt.AccountClaims) { 3163 s.updateAccountClaimsWithRefresh(a, ac, true) 3164 } 3165 3166 func (a *Account) traceLabel() string { 3167 if a == nil { 3168 return _EMPTY_ 3169 } 3170 if a.nameTag != _EMPTY_ { 3171 return fmt.Sprintf("%s/%s", a.Name, a.nameTag) 3172 } 3173 return a.Name 3174 } 3175 3176 // Check if an account has external auth set. 3177 // Operator/Account Resolver only. 3178 func (a *Account) hasExternalAuth() bool { 3179 if a == nil { 3180 return false 3181 } 3182 a.mu.RLock() 3183 defer a.mu.RUnlock() 3184 return a.extAuth != nil 3185 } 3186 3187 // Deterimine if this is an external auth user. 3188 func (a *Account) isExternalAuthUser(userID string) bool { 3189 if a == nil { 3190 return false 3191 } 3192 a.mu.RLock() 3193 defer a.mu.RUnlock() 3194 if a.extAuth != nil { 3195 for _, u := range a.extAuth.AuthUsers { 3196 if userID == u { 3197 return true 3198 } 3199 } 3200 } 3201 return false 3202 } 3203 3204 // Return the external authorization xkey if external authorization is enabled and the xkey is set. 3205 // Operator/Account Resolver only. 3206 func (a *Account) externalAuthXKey() string { 3207 if a == nil { 3208 return _EMPTY_ 3209 } 3210 a.mu.RLock() 3211 defer a.mu.RUnlock() 3212 if a.extAuth != nil && a.extAuth.XKey != _EMPTY_ { 3213 return a.extAuth.XKey 3214 } 3215 return _EMPTY_ 3216 } 3217 3218 // Check if an account switch for external authorization is allowed. 3219 func (a *Account) isAllowedAcount(acc string) bool { 3220 if a == nil { 3221 return false 3222 } 3223 a.mu.RLock() 3224 defer a.mu.RUnlock() 3225 if a.extAuth != nil { 3226 // if we have a single allowed account, and we have a wildcard 3227 // we accept it 3228 if len(a.extAuth.AllowedAccounts) == 1 && 3229 a.extAuth.AllowedAccounts[0] == jwt.AnyAccount { 3230 return true 3231 } 3232 // otherwise must match exactly 3233 for _, a := range a.extAuth.AllowedAccounts { 3234 if a == acc { 3235 return true 3236 } 3237 } 3238 } 3239 return false 3240 } 3241 3242 // updateAccountClaimsWithRefresh will update an existing account with new claims. 3243 // If refreshImportingAccounts is true it will also update incomplete dependent accounts 3244 // This will replace any exports or imports previously defined. 3245 // Lock MUST NOT be held upon entry. 3246 func (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaims, refreshImportingAccounts bool) { 3247 if a == nil { 3248 return 3249 } 3250 s.Debugf("Updating account claims: %s/%s", a.Name, ac.Name) 3251 a.checkExpiration(ac.Claims()) 3252 3253 a.mu.Lock() 3254 // Clone to update, only select certain fields. 3255 old := &Account{Name: a.Name, exports: a.exports, limits: a.limits, signingKeys: a.signingKeys} 3256 3257 // overwrite claim meta data 3258 a.nameTag = ac.Name 3259 a.tags = ac.Tags 3260 3261 var td string 3262 var tds int 3263 if ac.Trace != nil { 3264 // Update trace destination and sampling 3265 td, tds = string(ac.Trace.Destination), ac.Trace.Sampling 3266 if !IsValidPublishSubject(td) { 3267 td, tds = _EMPTY_, 0 3268 } else if tds <= 0 || tds > 100 { 3269 tds = 100 3270 } 3271 } 3272 a.traceDest, a.traceDestSampling = td, tds 3273 3274 // Check for external authorization. 3275 if ac.HasExternalAuthorization() { 3276 a.extAuth = &jwt.ExternalAuthorization{} 3277 a.extAuth.AuthUsers.Add(ac.Authorization.AuthUsers...) 3278 a.extAuth.AllowedAccounts.Add(ac.Authorization.AllowedAccounts...) 3279 a.extAuth.XKey = ac.Authorization.XKey 3280 } 3281 3282 // Reset exports and imports here. 3283 3284 // Exports is creating a whole new map. 3285 a.exports = exportMap{} 3286 3287 // Imports are checked unlocked in processInbound, so we can't change out the struct here. Need to process inline. 3288 if a.imports.streams != nil { 3289 old.imports.streams = a.imports.streams 3290 a.imports.streams = nil 3291 } 3292 if a.imports.services != nil { 3293 old.imports.services = make(map[string]*serviceImport, len(a.imports.services)) 3294 } 3295 for k, v := range a.imports.services { 3296 old.imports.services[k] = v 3297 delete(a.imports.services, k) 3298 } 3299 3300 alteredScope := map[string]struct{}{} 3301 3302 // update account signing keys 3303 a.signingKeys = nil 3304 _, strict := s.strictSigningKeyUsage[a.Issuer] 3305 if len(ac.SigningKeys) > 0 || !strict { 3306 a.signingKeys = make(map[string]jwt.Scope) 3307 } 3308 signersChanged := false 3309 for k, scope := range ac.SigningKeys { 3310 a.signingKeys[k] = scope 3311 } 3312 if !strict { 3313 a.signingKeys[a.Name] = nil 3314 } 3315 if len(a.signingKeys) != len(old.signingKeys) { 3316 signersChanged = true 3317 } 3318 for k, scope := range a.signingKeys { 3319 if oldScope, ok := old.signingKeys[k]; !ok { 3320 signersChanged = true 3321 } else if !reflect.DeepEqual(scope, oldScope) { 3322 signersChanged = true 3323 alteredScope[k] = struct{}{} 3324 } 3325 } 3326 // collect mappings that need to be removed 3327 removeList := []string{} 3328 for _, m := range a.mappings { 3329 if _, ok := ac.Mappings[jwt.Subject(m.src)]; !ok { 3330 removeList = append(removeList, m.src) 3331 } 3332 } 3333 a.mu.Unlock() 3334 3335 for sub, wm := range ac.Mappings { 3336 mappings := make([]*MapDest, len(wm)) 3337 for i, m := range wm { 3338 mappings[i] = &MapDest{ 3339 Subject: string(m.Subject), 3340 Weight: m.GetWeight(), 3341 Cluster: m.Cluster, 3342 } 3343 } 3344 // This will overwrite existing entries 3345 a.AddWeightedMappings(string(sub), mappings...) 3346 } 3347 // remove mappings 3348 for _, rmMapping := range removeList { 3349 a.RemoveMapping(rmMapping) 3350 } 3351 3352 // Re-register system exports/imports. 3353 if a == s.SystemAccount() { 3354 s.addSystemAccountExports(a) 3355 } else { 3356 s.registerSystemImports(a) 3357 } 3358 3359 jsEnabled := s.JetStreamEnabled() 3360 3361 streamTokenExpirationChanged := false 3362 serviceTokenExpirationChanged := false 3363 3364 for _, e := range ac.Exports { 3365 switch e.Type { 3366 case jwt.Stream: 3367 s.Debugf("Adding stream export %q for %s", e.Subject, a.traceLabel()) 3368 if err := a.addStreamExportWithAccountPos( 3369 string(e.Subject), authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil { 3370 s.Debugf("Error adding stream export to account [%s]: %v", a.traceLabel(), err.Error()) 3371 } 3372 case jwt.Service: 3373 s.Debugf("Adding service export %q for %s", e.Subject, a.traceLabel()) 3374 rt := Singleton 3375 switch e.ResponseType { 3376 case jwt.ResponseTypeStream: 3377 rt = Streamed 3378 case jwt.ResponseTypeChunked: 3379 rt = Chunked 3380 } 3381 if err := a.addServiceExportWithResponseAndAccountPos( 3382 string(e.Subject), rt, authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil { 3383 s.Debugf("Error adding service export to account [%s]: %v", a.traceLabel(), err) 3384 continue 3385 } 3386 sub := string(e.Subject) 3387 if e.Latency != nil { 3388 if err := a.TrackServiceExportWithSampling(sub, string(e.Latency.Results), int(e.Latency.Sampling)); err != nil { 3389 hdrNote := _EMPTY_ 3390 if e.Latency.Sampling == jwt.Headers { 3391 hdrNote = " (using headers)" 3392 } 3393 s.Debugf("Error adding latency tracking%s for service export to account [%s]: %v", hdrNote, a.traceLabel(), err) 3394 } 3395 } 3396 if e.ResponseThreshold != 0 { 3397 // Response threshold was set in options. 3398 if err := a.SetServiceExportResponseThreshold(sub, e.ResponseThreshold); err != nil { 3399 s.Debugf("Error adding service export response threshold for [%s]: %v", a.traceLabel(), err) 3400 } 3401 } 3402 if err := a.SetServiceExportAllowTrace(sub, e.AllowTrace); err != nil { 3403 s.Debugf("Error adding allow_trace for %q: %v", sub, err) 3404 } 3405 } 3406 3407 var revocationChanged *bool 3408 var ea *exportAuth 3409 3410 a.mu.Lock() 3411 switch e.Type { 3412 case jwt.Stream: 3413 revocationChanged = &streamTokenExpirationChanged 3414 if se, ok := a.exports.streams[string(e.Subject)]; ok && se != nil { 3415 ea = &se.exportAuth 3416 } 3417 case jwt.Service: 3418 revocationChanged = &serviceTokenExpirationChanged 3419 if se, ok := a.exports.services[string(e.Subject)]; ok && se != nil { 3420 ea = &se.exportAuth 3421 } 3422 } 3423 if ea != nil { 3424 oldRevocations := ea.actsRevoked 3425 if len(e.Revocations) == 0 { 3426 // remove all, no need to evaluate existing imports 3427 ea.actsRevoked = nil 3428 } else if len(oldRevocations) == 0 { 3429 // add all, existing imports need to be re evaluated 3430 ea.actsRevoked = e.Revocations 3431 *revocationChanged = true 3432 } else { 3433 ea.actsRevoked = e.Revocations 3434 // diff, existing imports need to be conditionally re evaluated, depending on: 3435 // if a key was added, or it's timestamp increased 3436 for k, t := range e.Revocations { 3437 if tOld, ok := oldRevocations[k]; !ok || tOld < t { 3438 *revocationChanged = true 3439 } 3440 } 3441 } 3442 } 3443 a.mu.Unlock() 3444 } 3445 var incompleteImports []*jwt.Import 3446 for _, i := range ac.Imports { 3447 // check tmpAccounts with priority 3448 var acc *Account 3449 var err error 3450 if v, ok := s.tmpAccounts.Load(i.Account); ok { 3451 acc = v.(*Account) 3452 } else { 3453 acc, err = s.lookupAccount(i.Account) 3454 } 3455 if acc == nil || err != nil { 3456 s.Errorf("Can't locate account [%s] for import of [%v] %s (err=%v)", i.Account, i.Subject, i.Type, err) 3457 incompleteImports = append(incompleteImports, i) 3458 continue 3459 } 3460 from := string(i.Subject) 3461 to := i.GetTo() 3462 switch i.Type { 3463 case jwt.Stream: 3464 if i.LocalSubject != _EMPTY_ { 3465 // set local subject implies to is empty 3466 to = string(i.LocalSubject) 3467 s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to) 3468 err = a.AddMappedStreamImportWithClaim(acc, from, to, i) 3469 } else { 3470 s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to) 3471 err = a.AddStreamImportWithClaim(acc, from, to, i) 3472 } 3473 if err != nil { 3474 s.Debugf("Error adding stream import to account [%s]: %v", a.traceLabel(), err.Error()) 3475 incompleteImports = append(incompleteImports, i) 3476 } 3477 case jwt.Service: 3478 if i.LocalSubject != _EMPTY_ { 3479 from = string(i.LocalSubject) 3480 to = string(i.Subject) 3481 } 3482 s.Debugf("Adding service import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to) 3483 if err := a.AddServiceImportWithClaim(acc, from, to, i); err != nil { 3484 s.Debugf("Error adding service import to account [%s]: %v", a.traceLabel(), err.Error()) 3485 incompleteImports = append(incompleteImports, i) 3486 } 3487 } 3488 } 3489 // Now let's apply any needed changes from import/export changes. 3490 if !a.checkStreamImportsEqual(old) { 3491 awcsti := map[string]struct{}{a.Name: {}} 3492 for _, c := range a.getClients() { 3493 c.processSubsOnConfigReload(awcsti) 3494 } 3495 } 3496 // Now check if stream exports have changed. 3497 if !a.checkStreamExportsEqual(old) || signersChanged || streamTokenExpirationChanged { 3498 clients := map[*client]struct{}{} 3499 // We need to check all accounts that have an import claim from this account. 3500 awcsti := map[string]struct{}{} 3501 s.accounts.Range(func(k, v any) bool { 3502 acc := v.(*Account) 3503 // Move to the next if this account is actually account "a". 3504 if acc.Name == a.Name { 3505 return true 3506 } 3507 // TODO: checkStreamImportAuthorized() stack should not be trying 3508 // to lock "acc". If we find that to be needed, we will need to 3509 // rework this to ensure we don't lock acc. 3510 acc.mu.Lock() 3511 for _, im := range acc.imports.streams { 3512 if im != nil && im.acc.Name == a.Name { 3513 // Check for if we are still authorized for an import. 3514 im.invalid = !a.checkStreamImportAuthorized(acc, im.from, im.claim) 3515 awcsti[acc.Name] = struct{}{} 3516 for c := range acc.clients { 3517 clients[c] = struct{}{} 3518 } 3519 } 3520 } 3521 acc.mu.Unlock() 3522 return true 3523 }) 3524 // Now walk clients. 3525 for c := range clients { 3526 c.processSubsOnConfigReload(awcsti) 3527 } 3528 } 3529 // Now check if service exports have changed. 3530 if !a.checkServiceExportsEqual(old) || signersChanged || serviceTokenExpirationChanged { 3531 s.accounts.Range(func(k, v any) bool { 3532 acc := v.(*Account) 3533 // Move to the next if this account is actually account "a". 3534 if acc.Name == a.Name { 3535 return true 3536 } 3537 // TODO: checkServiceImportAuthorized() stack should not be trying 3538 // to lock "acc". If we find that to be needed, we will need to 3539 // rework this to ensure we don't lock acc. 3540 acc.mu.Lock() 3541 for _, si := range acc.imports.services { 3542 if si != nil && si.acc.Name == a.Name { 3543 // Check for if we are still authorized for an import. 3544 si.invalid = !a.checkServiceImportAuthorized(acc, si.to, si.claim) 3545 // Make sure we should still be tracking latency and if we 3546 // are allowed to trace. 3547 if !si.response { 3548 if se := a.getServiceExport(si.to); se != nil { 3549 if si.latency != nil { 3550 si.latency = se.latency 3551 } 3552 // Update allow trace. 3553 si.atrc = se.atrc 3554 } 3555 } 3556 } 3557 } 3558 acc.mu.Unlock() 3559 return true 3560 }) 3561 } 3562 3563 // Now make sure we shutdown the old service import subscriptions. 3564 var sids [][]byte 3565 a.mu.RLock() 3566 c := a.ic 3567 for _, si := range old.imports.services { 3568 if c != nil && si.sid != nil { 3569 sids = append(sids, si.sid) 3570 } 3571 } 3572 a.mu.RUnlock() 3573 for _, sid := range sids { 3574 c.processUnsub(sid) 3575 } 3576 3577 // Now do limits if they are present. 3578 a.mu.Lock() 3579 a.msubs = int32(ac.Limits.Subs) 3580 a.mpay = int32(ac.Limits.Payload) 3581 a.mconns = int32(ac.Limits.Conn) 3582 a.mleafs = int32(ac.Limits.LeafNodeConn) 3583 a.disallowBearer = ac.Limits.DisallowBearer 3584 // Check for any revocations 3585 if len(ac.Revocations) > 0 { 3586 // We will always replace whatever we had with most current, so no 3587 // need to look at what we have. 3588 a.usersRevoked = make(map[string]int64, len(ac.Revocations)) 3589 for pk, t := range ac.Revocations { 3590 a.usersRevoked[pk] = t 3591 } 3592 } else { 3593 a.usersRevoked = nil 3594 } 3595 a.defaultPerms = buildPermissionsFromJwt(&ac.DefaultPermissions) 3596 a.incomplete = len(incompleteImports) != 0 3597 for _, i := range incompleteImports { 3598 s.incompleteAccExporterMap.Store(i.Account, struct{}{}) 3599 } 3600 if a.srv == nil { 3601 a.srv = s 3602 } 3603 3604 if ac.Limits.IsJSEnabled() { 3605 toUnlimited := func(value int64) int64 { 3606 if value > 0 { 3607 return value 3608 } 3609 return -1 3610 } 3611 if ac.Limits.JetStreamLimits.DiskStorage != 0 || ac.Limits.JetStreamLimits.MemoryStorage != 0 { 3612 // JetStreamAccountLimits and jwt.JetStreamLimits use same value for unlimited 3613 a.jsLimits = map[string]JetStreamAccountLimits{ 3614 _EMPTY_: { 3615 MaxMemory: ac.Limits.JetStreamLimits.MemoryStorage, 3616 MaxStore: ac.Limits.JetStreamLimits.DiskStorage, 3617 MaxStreams: int(ac.Limits.JetStreamLimits.Streams), 3618 MaxConsumers: int(ac.Limits.JetStreamLimits.Consumer), 3619 MemoryMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.MemoryMaxStreamBytes), 3620 StoreMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.DiskMaxStreamBytes), 3621 MaxBytesRequired: ac.Limits.JetStreamLimits.MaxBytesRequired, 3622 MaxAckPending: int(toUnlimited(ac.Limits.JetStreamLimits.MaxAckPending)), 3623 }, 3624 } 3625 } else { 3626 a.jsLimits = map[string]JetStreamAccountLimits{} 3627 for t, l := range ac.Limits.JetStreamTieredLimits { 3628 a.jsLimits[t] = JetStreamAccountLimits{ 3629 MaxMemory: l.MemoryStorage, 3630 MaxStore: l.DiskStorage, 3631 MaxStreams: int(l.Streams), 3632 MaxConsumers: int(l.Consumer), 3633 MemoryMaxStreamBytes: toUnlimited(l.MemoryMaxStreamBytes), 3634 StoreMaxStreamBytes: toUnlimited(l.DiskMaxStreamBytes), 3635 MaxBytesRequired: l.MaxBytesRequired, 3636 MaxAckPending: int(toUnlimited(l.MaxAckPending)), 3637 } 3638 } 3639 } 3640 } else if a.jsLimits != nil { 3641 // covers failed update followed by disable 3642 a.jsLimits = nil 3643 } 3644 3645 a.updated = time.Now() 3646 clients := a.getClientsLocked() 3647 a.mu.Unlock() 3648 3649 // Sort if we are over the limit. 3650 if a.MaxTotalConnectionsReached() { 3651 sort.Slice(clients, func(i, j int) bool { 3652 return clients[i].start.After(clients[j].start) 3653 }) 3654 } 3655 3656 // If JetStream is enabled for this server we will call into configJetStream for the account 3657 // regardless of enabled or disabled. It handles both cases. 3658 if jsEnabled { 3659 if err := s.configJetStream(a); err != nil { 3660 s.Errorf("Error configuring jetstream for account [%s]: %v", a.traceLabel(), err.Error()) 3661 a.mu.Lock() 3662 // Absent reload of js server cfg, this is going to be broken until js is disabled 3663 a.incomplete = true 3664 a.mu.Unlock() 3665 } 3666 } else if a.jsLimits != nil { 3667 // We do not have JS enabled for this server, but the account has it enabled so setup 3668 // our imports properly. This allows this server to proxy JS traffic correctly. 3669 s.checkJetStreamExports() 3670 a.enableAllJetStreamServiceImportsAndMappings() 3671 } 3672 3673 for i, c := range clients { 3674 a.mu.RLock() 3675 exceeded := a.mconns != jwt.NoLimit && i >= int(a.mconns) 3676 a.mu.RUnlock() 3677 if exceeded { 3678 c.maxAccountConnExceeded() 3679 continue 3680 } 3681 c.mu.Lock() 3682 c.applyAccountLimits() 3683 theJWT := c.opts.JWT 3684 c.mu.Unlock() 3685 // Check for being revoked here. We use ac one to avoid the account lock. 3686 if (ac.Revocations != nil || ac.Limits.DisallowBearer) && theJWT != _EMPTY_ { 3687 if juc, err := jwt.DecodeUserClaims(theJWT); err != nil { 3688 c.Debugf("User JWT not valid: %v", err) 3689 c.authViolation() 3690 continue 3691 } else if juc.BearerToken && ac.Limits.DisallowBearer { 3692 c.Debugf("Bearer User JWT not allowed") 3693 c.authViolation() 3694 continue 3695 } else if ok := ac.IsClaimRevoked(juc); ok { 3696 c.sendErrAndDebug("User Authentication Revoked") 3697 c.closeConnection(Revocation) 3698 continue 3699 } 3700 } 3701 } 3702 3703 // Check if the signing keys changed, might have to evict 3704 if signersChanged { 3705 for _, c := range clients { 3706 c.mu.Lock() 3707 if c.user == nil { 3708 c.mu.Unlock() 3709 continue 3710 } 3711 sk := c.user.SigningKey 3712 c.mu.Unlock() 3713 if sk == _EMPTY_ { 3714 continue 3715 } 3716 if _, ok := alteredScope[sk]; ok { 3717 c.closeConnection(AuthenticationViolation) 3718 } else if _, ok := a.hasIssuer(sk); !ok { 3719 c.closeConnection(AuthenticationViolation) 3720 } 3721 } 3722 } 3723 3724 if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok && refreshImportingAccounts { 3725 s.incompleteAccExporterMap.Delete(old.Name) 3726 s.accounts.Range(func(key, value any) bool { 3727 acc := value.(*Account) 3728 acc.mu.RLock() 3729 incomplete := acc.incomplete 3730 name := acc.Name 3731 label := acc.traceLabel() 3732 // Must use jwt in account or risk failing on fetch 3733 // This jwt may not be the same that caused exportingAcc to be in incompleteAccExporterMap 3734 claimJWT := acc.claimJWT 3735 acc.mu.RUnlock() 3736 if incomplete && name != old.Name { 3737 if accClaims, _, err := s.verifyAccountClaims(claimJWT); err == nil { 3738 // Since claimJWT has not changed, acc can become complete 3739 // but it won't alter incomplete for it's dependents accounts. 3740 s.updateAccountClaimsWithRefresh(acc, accClaims, false) 3741 // old.Name was deleted before ranging over accounts 3742 // If it exists again, UpdateAccountClaims set it for failed imports of acc. 3743 // So there was one import of acc that imported this account and failed again. 3744 // Since this account just got updated, the import itself may be in error. So trace that. 3745 if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok { 3746 s.incompleteAccExporterMap.Delete(old.Name) 3747 s.Errorf("Account %s has issues importing account %s", label, old.Name) 3748 } 3749 } 3750 } 3751 return true 3752 }) 3753 } 3754 } 3755 3756 // Helper to build an internal account structure from a jwt.AccountClaims. 3757 // Lock MUST NOT be held upon entry. 3758 func (s *Server) buildInternalAccount(ac *jwt.AccountClaims) *Account { 3759 acc := NewAccount(ac.Subject) 3760 acc.Issuer = ac.Issuer 3761 // Override subject with logical name. 3762 acc.LogicalName = ac.Name 3763 // Set this here since we are placing in s.tmpAccounts below and may be 3764 // referenced by an route RS+, etc. 3765 s.setAccountSublist(acc) 3766 3767 // We don't want to register an account that is in the process of 3768 // being built, however, to solve circular import dependencies, we 3769 // need to store it here. 3770 s.tmpAccounts.Store(ac.Subject, acc) 3771 s.UpdateAccountClaims(acc, ac) 3772 return acc 3773 } 3774 3775 // Helper to build Permissions from jwt.Permissions 3776 // or return nil if none were specified 3777 func buildPermissionsFromJwt(uc *jwt.Permissions) *Permissions { 3778 if uc == nil { 3779 return nil 3780 } 3781 var p *Permissions 3782 if len(uc.Pub.Allow) > 0 || len(uc.Pub.Deny) > 0 { 3783 p = &Permissions{} 3784 p.Publish = &SubjectPermission{} 3785 p.Publish.Allow = uc.Pub.Allow 3786 p.Publish.Deny = uc.Pub.Deny 3787 } 3788 if len(uc.Sub.Allow) > 0 || len(uc.Sub.Deny) > 0 { 3789 if p == nil { 3790 p = &Permissions{} 3791 } 3792 p.Subscribe = &SubjectPermission{} 3793 p.Subscribe.Allow = uc.Sub.Allow 3794 p.Subscribe.Deny = uc.Sub.Deny 3795 } 3796 if uc.Resp != nil { 3797 if p == nil { 3798 p = &Permissions{} 3799 } 3800 p.Response = &ResponsePermission{ 3801 MaxMsgs: uc.Resp.MaxMsgs, 3802 Expires: uc.Resp.Expires, 3803 } 3804 validateResponsePermissions(p) 3805 } 3806 return p 3807 } 3808 3809 // Helper to build internal NKeyUser. 3810 func buildInternalNkeyUser(uc *jwt.UserClaims, acts map[string]struct{}, acc *Account) *NkeyUser { 3811 nu := &NkeyUser{Nkey: uc.Subject, Account: acc, AllowedConnectionTypes: acts} 3812 if uc.IssuerAccount != _EMPTY_ { 3813 nu.SigningKey = uc.Issuer 3814 } 3815 3816 // Now check for permissions. 3817 var p = buildPermissionsFromJwt(&uc.Permissions) 3818 if p == nil && acc.defaultPerms != nil { 3819 p = acc.defaultPerms.clone() 3820 } 3821 nu.Permissions = p 3822 return nu 3823 } 3824 3825 func fetchAccount(res AccountResolver, name string) (string, error) { 3826 if !nkeys.IsValidPublicAccountKey(name) { 3827 return _EMPTY_, fmt.Errorf("will only fetch valid account keys") 3828 } 3829 return res.Fetch(name) 3830 } 3831 3832 // AccountResolver interface. This is to fetch Account JWTs by public nkeys 3833 type AccountResolver interface { 3834 Fetch(name string) (string, error) 3835 Store(name, jwt string) error 3836 IsReadOnly() bool 3837 Start(server *Server) error 3838 IsTrackingUpdate() bool 3839 Reload() error 3840 Close() 3841 } 3842 3843 // Default implementations of IsReadOnly/Start so only need to be written when changed 3844 type resolverDefaultsOpsImpl struct{} 3845 3846 func (*resolverDefaultsOpsImpl) IsReadOnly() bool { 3847 return true 3848 } 3849 3850 func (*resolverDefaultsOpsImpl) IsTrackingUpdate() bool { 3851 return false 3852 } 3853 3854 func (*resolverDefaultsOpsImpl) Start(*Server) error { 3855 return nil 3856 } 3857 3858 func (*resolverDefaultsOpsImpl) Reload() error { 3859 return nil 3860 } 3861 3862 func (*resolverDefaultsOpsImpl) Close() { 3863 } 3864 3865 func (*resolverDefaultsOpsImpl) Store(_, _ string) error { 3866 return fmt.Errorf("store operation not supported for URL Resolver") 3867 } 3868 3869 // MemAccResolver is a memory only resolver. 3870 // Mostly for testing. 3871 type MemAccResolver struct { 3872 sm sync.Map 3873 resolverDefaultsOpsImpl 3874 } 3875 3876 // Fetch will fetch the account jwt claims from the internal sync.Map. 3877 func (m *MemAccResolver) Fetch(name string) (string, error) { 3878 if j, ok := m.sm.Load(name); ok { 3879 return j.(string), nil 3880 } 3881 return _EMPTY_, ErrMissingAccount 3882 } 3883 3884 // Store will store the account jwt claims in the internal sync.Map. 3885 func (m *MemAccResolver) Store(name, jwt string) error { 3886 m.sm.Store(name, jwt) 3887 return nil 3888 } 3889 3890 func (m *MemAccResolver) IsReadOnly() bool { 3891 return false 3892 } 3893 3894 // URLAccResolver implements an http fetcher. 3895 type URLAccResolver struct { 3896 url string 3897 c *http.Client 3898 resolverDefaultsOpsImpl 3899 } 3900 3901 // NewURLAccResolver returns a new resolver for the given base URL. 3902 func NewURLAccResolver(url string) (*URLAccResolver, error) { 3903 if !strings.HasSuffix(url, "/") { 3904 url += "/" 3905 } 3906 // FIXME(dlc) - Make timeout and others configurable. 3907 // We create our own transport to amortize TLS. 3908 tr := &http.Transport{ 3909 MaxIdleConns: 10, 3910 IdleConnTimeout: 30 * time.Second, 3911 } 3912 ur := &URLAccResolver{ 3913 url: url, 3914 c: &http.Client{Timeout: DEFAULT_ACCOUNT_FETCH_TIMEOUT, Transport: tr}, 3915 } 3916 return ur, nil 3917 } 3918 3919 // Fetch will fetch the account jwt claims from the base url, appending the 3920 // account name onto the end. 3921 func (ur *URLAccResolver) Fetch(name string) (string, error) { 3922 url := ur.url + name 3923 resp, err := ur.c.Get(url) 3924 if err != nil { 3925 return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), err) 3926 } else if resp == nil { 3927 return _EMPTY_, fmt.Errorf("could not fetch <%q>: no response", redactURLString(url)) 3928 } 3929 defer resp.Body.Close() 3930 if resp.StatusCode != http.StatusOK { 3931 return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), resp.Status) 3932 } 3933 body, err := io.ReadAll(resp.Body) 3934 if err != nil { 3935 return _EMPTY_, err 3936 } 3937 return string(body), nil 3938 } 3939 3940 // Resolver based on nats for synchronization and backing directory for storage. 3941 type DirAccResolver struct { 3942 *DirJWTStore 3943 *Server 3944 syncInterval time.Duration 3945 fetchTimeout time.Duration 3946 } 3947 3948 func (dr *DirAccResolver) IsTrackingUpdate() bool { 3949 return true 3950 } 3951 3952 func (dr *DirAccResolver) Reload() error { 3953 return dr.DirJWTStore.Reload() 3954 } 3955 3956 // ServerAPIClaimUpdateResponse is the response to $SYS.REQ.ACCOUNT.<id>.CLAIMS.UPDATE and $SYS.REQ.CLAIMS.UPDATE 3957 type ServerAPIClaimUpdateResponse struct { 3958 Server *ServerInfo `json:"server"` 3959 Data *ClaimUpdateStatus `json:"data,omitempty"` 3960 Error *ClaimUpdateError `json:"error,omitempty"` 3961 } 3962 3963 type ClaimUpdateError struct { 3964 Account string `json:"account,omitempty"` 3965 Code int `json:"code"` 3966 Description string `json:"description,omitempty"` 3967 } 3968 3969 type ClaimUpdateStatus struct { 3970 Account string `json:"account,omitempty"` 3971 Code int `json:"code,omitempty"` 3972 Message string `json:"message,omitempty"` 3973 } 3974 3975 func respondToUpdate(s *Server, respSubj string, acc string, message string, err error) { 3976 if err == nil { 3977 if acc == _EMPTY_ { 3978 s.Debugf("%s", message) 3979 } else { 3980 s.Debugf("%s - %s", message, acc) 3981 } 3982 } else { 3983 if acc == _EMPTY_ { 3984 s.Errorf("%s - %s", message, err) 3985 } else { 3986 s.Errorf("%s - %s - %s", message, acc, err) 3987 } 3988 } 3989 if respSubj == _EMPTY_ { 3990 return 3991 } 3992 3993 response := ServerAPIClaimUpdateResponse{ 3994 Server: &ServerInfo{}, 3995 } 3996 3997 if err == nil { 3998 response.Data = &ClaimUpdateStatus{ 3999 Account: acc, 4000 Code: http.StatusOK, 4001 Message: message, 4002 } 4003 } else { 4004 response.Error = &ClaimUpdateError{ 4005 Account: acc, 4006 Code: http.StatusInternalServerError, 4007 Description: fmt.Sprintf("%s - %v", message, err), 4008 } 4009 } 4010 4011 s.sendInternalMsgLocked(respSubj, _EMPTY_, response.Server, response) 4012 } 4013 4014 func handleListRequest(store *DirJWTStore, s *Server, reply string) { 4015 if reply == _EMPTY_ { 4016 return 4017 } 4018 accIds := make([]string, 0, 1024) 4019 if err := store.PackWalk(1, func(partialPackMsg string) { 4020 if tk := strings.Split(partialPackMsg, "|"); len(tk) == 2 { 4021 accIds = append(accIds, tk[0]) 4022 } 4023 }); err != nil { 4024 // let them timeout 4025 s.Errorf("list request error: %v", err) 4026 } else { 4027 s.Debugf("list request responded with %d account ids", len(accIds)) 4028 server := &ServerInfo{} 4029 response := map[string]any{"server": server, "data": accIds} 4030 s.sendInternalMsgLocked(reply, _EMPTY_, server, response) 4031 } 4032 } 4033 4034 func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) { 4035 var accIds []any 4036 var subj, sysAccName string 4037 if sysAcc := s.SystemAccount(); sysAcc != nil { 4038 sysAccName = sysAcc.GetName() 4039 } 4040 // Only operator and operator signing key are allowed to delete 4041 gk, err := jwt.DecodeGeneric(string(msg)) 4042 if err == nil { 4043 subj = gk.Subject 4044 if store.deleteType == NoDelete { 4045 err = fmt.Errorf("delete must be enabled in server config") 4046 } else if subj != gk.Issuer { 4047 err = fmt.Errorf("not self signed") 4048 } else if _, ok := store.operator[gk.Issuer]; !ok { 4049 err = fmt.Errorf("not trusted") 4050 } else if list, ok := gk.Data["accounts"]; !ok { 4051 err = fmt.Errorf("malformed request") 4052 } else if accIds, ok = list.([]any); !ok { 4053 err = fmt.Errorf("malformed request") 4054 } else { 4055 for _, entry := range accIds { 4056 if acc, ok := entry.(string); !ok || 4057 acc == _EMPTY_ || !nkeys.IsValidPublicAccountKey(acc) { 4058 err = fmt.Errorf("malformed request") 4059 break 4060 } else if acc == sysAccName { 4061 err = fmt.Errorf("not allowed to delete system account") 4062 break 4063 } 4064 } 4065 } 4066 } 4067 if err != nil { 4068 respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("delete accounts request by %s failed", subj), err) 4069 return 4070 } 4071 errs := []string{} 4072 passCnt := 0 4073 for _, acc := range accIds { 4074 if err := store.delete(acc.(string)); err != nil { 4075 errs = append(errs, err.Error()) 4076 } else { 4077 passCnt++ 4078 } 4079 } 4080 if len(errs) == 0 { 4081 respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts", passCnt), nil) 4082 } else { 4083 respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts, failed for %d", passCnt, len(errs)), 4084 errors.New(strings.Join(errs, "\n"))) 4085 } 4086 } 4087 4088 func getOperatorKeys(s *Server) (string, map[string]struct{}, bool, error) { 4089 var op string 4090 var strict bool 4091 keys := make(map[string]struct{}) 4092 if opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 { 4093 op = opts.TrustedOperators[0].Subject 4094 strict = opts.TrustedOperators[0].StrictSigningKeyUsage 4095 if !strict { 4096 keys[opts.TrustedOperators[0].Subject] = struct{}{} 4097 } 4098 for _, key := range opts.TrustedOperators[0].SigningKeys { 4099 keys[key] = struct{}{} 4100 } 4101 } 4102 if len(keys) == 0 { 4103 return _EMPTY_, nil, false, fmt.Errorf("no operator key found") 4104 } 4105 return op, keys, strict, nil 4106 } 4107 4108 func claimValidate(claim *jwt.AccountClaims) error { 4109 vr := &jwt.ValidationResults{} 4110 claim.Validate(vr) 4111 if vr.IsBlocking(false) { 4112 return fmt.Errorf("validation errors: %v", vr.Errors()) 4113 } 4114 return nil 4115 } 4116 4117 func removeCb(s *Server, pubKey string) { 4118 v, ok := s.accounts.Load(pubKey) 4119 if !ok { 4120 return 4121 } 4122 a := v.(*Account) 4123 s.Debugf("Disable account %s due to remove", pubKey) 4124 a.mu.Lock() 4125 // lock out new clients 4126 a.msubs = 0 4127 a.mpay = 0 4128 a.mconns = 0 4129 a.mleafs = 0 4130 a.updated = time.Now() 4131 jsa := a.js 4132 a.mu.Unlock() 4133 // set the account to be expired and disconnect clients 4134 a.expiredTimeout() 4135 // For JS, we need also to disable it. 4136 if js := s.getJetStream(); js != nil && jsa != nil { 4137 js.disableJetStream(jsa) 4138 // Remove JetStream state in memory, this will be reset 4139 // on the changed callback from the account in case it is 4140 // enabled again. 4141 a.js = nil 4142 } 4143 // We also need to remove all ServerImport subscriptions 4144 a.removeAllServiceImportSubs() 4145 a.mu.Lock() 4146 a.clearExpirationTimer() 4147 a.mu.Unlock() 4148 } 4149 4150 func (dr *DirAccResolver) Start(s *Server) error { 4151 op, opKeys, strict, err := getOperatorKeys(s) 4152 if err != nil { 4153 return err 4154 } 4155 dr.Lock() 4156 defer dr.Unlock() 4157 dr.Server = s 4158 dr.operator = opKeys 4159 dr.DirJWTStore.changed = func(pubKey string) { 4160 if v, ok := s.accounts.Load(pubKey); ok { 4161 if theJwt, err := dr.LoadAcc(pubKey); err != nil { 4162 s.Errorf("DirResolver - Update got error on load: %v", err) 4163 } else { 4164 acc := v.(*Account) 4165 if err = s.updateAccountWithClaimJWT(acc, theJwt); err != nil { 4166 s.Errorf("DirResolver - Update for account %q resulted in error %v", pubKey, err) 4167 } else { 4168 if _, jsa, err := acc.checkForJetStream(); err != nil { 4169 if !IsNatsErr(err, JSNotEnabledForAccountErr) { 4170 s.Warnf("DirResolver - Error checking for JetStream support for account %q: %v", pubKey, err) 4171 } 4172 } else if jsa == nil { 4173 if err = s.configJetStream(acc); err != nil { 4174 s.Errorf("DirResolver - Error configuring JetStream for account %q: %v", pubKey, err) 4175 } 4176 } 4177 } 4178 } 4179 } 4180 } 4181 dr.DirJWTStore.deleted = func(pubKey string) { 4182 removeCb(s, pubKey) 4183 } 4184 packRespIb := s.newRespInbox() 4185 for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { 4186 // subscribe to account jwt update requests 4187 if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) { 4188 var pubKey string 4189 tk := strings.Split(subj, tsep) 4190 if len(tk) == accUpdateTokensNew { 4191 pubKey = tk[accReqAccIndex] 4192 } else if len(tk) == accUpdateTokensOld { 4193 pubKey = tk[accUpdateAccIdxOld] 4194 } else { 4195 s.Debugf("DirResolver - jwt update skipped due to bad subject %q", subj) 4196 return 4197 } 4198 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4199 respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err) 4200 } else if err := claimValidate(claim); err != nil { 4201 respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err) 4202 } else if claim.Subject != pubKey { 4203 err := errors.New("subject does not match jwt content") 4204 respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) 4205 } else if claim.Issuer == op && strict { 4206 err := errors.New("operator requires issuer to be a signing key") 4207 respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) 4208 } else if err := dr.save(pubKey, string(msg)); err != nil { 4209 respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) 4210 } else { 4211 respondToUpdate(s, resp, pubKey, "jwt updated", nil) 4212 } 4213 }); err != nil { 4214 return fmt.Errorf("error setting up update handling: %v", err) 4215 } 4216 } 4217 if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) { 4218 // As this is a raw message, we need to extract payload and only decode claims from it, 4219 // in case request is sent with headers. 4220 _, msg = c.msgParts(msg) 4221 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4222 respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err) 4223 } else if claim.Issuer == op && strict { 4224 err := errors.New("operator requires issuer to be a signing key") 4225 respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err) 4226 } else if err := claimValidate(claim); err != nil { 4227 respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err) 4228 } else if err := dr.save(claim.Subject, string(msg)); err != nil { 4229 respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err) 4230 } else { 4231 respondToUpdate(s, resp, claim.Subject, "jwt updated", nil) 4232 } 4233 }); err != nil { 4234 return fmt.Errorf("error setting up update handling: %v", err) 4235 } 4236 // respond to lookups with our version 4237 if _, err := s.sysSubscribe(fmt.Sprintf(accLookupReqSubj, "*"), func(_ *subscription, _ *client, _ *Account, subj, reply string, msg []byte) { 4238 if reply == _EMPTY_ { 4239 return 4240 } 4241 tk := strings.Split(subj, tsep) 4242 if len(tk) != accLookupReqTokens { 4243 return 4244 } 4245 accName := tk[accReqAccIndex] 4246 if theJWT, err := dr.DirJWTStore.LoadAcc(accName); err != nil { 4247 if errors.Is(err, fs.ErrNotExist) { 4248 s.Debugf("DirResolver - Could not find account %q", accName) 4249 // Reply with empty response to signal absence of JWT to others. 4250 s.sendInternalMsgLocked(reply, _EMPTY_, nil, nil) 4251 } else { 4252 s.Errorf("DirResolver - Error looking up account %q: %v", accName, err) 4253 } 4254 } else { 4255 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(theJWT)) 4256 } 4257 }); err != nil { 4258 return fmt.Errorf("error setting up lookup request handling: %v", err) 4259 } 4260 // respond to pack requests with one or more pack messages 4261 // an empty message signifies the end of the response responder. 4262 if _, err := s.sysSubscribeQ(accPackReqSubj, "responder", func(_ *subscription, _ *client, _ *Account, _, reply string, theirHash []byte) { 4263 if reply == _EMPTY_ { 4264 return 4265 } 4266 ourHash := dr.DirJWTStore.Hash() 4267 if bytes.Equal(theirHash, ourHash[:]) { 4268 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{}) 4269 s.Debugf("DirResolver - Pack request matches hash %x", ourHash[:]) 4270 } else if err := dr.DirJWTStore.PackWalk(1, func(partialPackMsg string) { 4271 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(partialPackMsg)) 4272 }); err != nil { 4273 // let them timeout 4274 s.Errorf("DirResolver - Pack request error: %v", err) 4275 } else { 4276 s.Debugf("DirResolver - Pack request hash %x - finished responding with hash %x", theirHash, ourHash) 4277 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{}) 4278 } 4279 }); err != nil { 4280 return fmt.Errorf("error setting up pack request handling: %v", err) 4281 } 4282 // respond to list requests with one message containing all account ids 4283 if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) { 4284 handleListRequest(dr.DirJWTStore, s, reply) 4285 }); err != nil { 4286 return fmt.Errorf("error setting up list request handling: %v", err) 4287 } 4288 if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { 4289 handleDeleteRequest(dr.DirJWTStore, s, msg, reply) 4290 }); err != nil { 4291 return fmt.Errorf("error setting up delete request handling: %v", err) 4292 } 4293 // embed pack responses into store 4294 if _, err := s.sysSubscribe(packRespIb, func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) { 4295 hash := dr.DirJWTStore.Hash() 4296 if len(msg) == 0 { // end of response stream 4297 s.Debugf("DirResolver - Merging finished and resulting in: %x", dr.DirJWTStore.Hash()) 4298 return 4299 } else if err := dr.DirJWTStore.Merge(string(msg)); err != nil { 4300 s.Errorf("DirResolver - Merging resulted in error: %v", err) 4301 } else { 4302 s.Debugf("DirResolver - Merging succeeded and changed %x to %x", hash, dr.DirJWTStore.Hash()) 4303 } 4304 }); err != nil { 4305 return fmt.Errorf("error setting up pack response handling: %v", err) 4306 } 4307 // periodically send out pack message 4308 quit := s.quitCh 4309 s.startGoRoutine(func() { 4310 defer s.grWG.Done() 4311 ticker := time.NewTicker(dr.syncInterval) 4312 for { 4313 select { 4314 case <-quit: 4315 ticker.Stop() 4316 return 4317 case <-ticker.C: 4318 } 4319 ourHash := dr.DirJWTStore.Hash() 4320 s.Debugf("DirResolver - Checking store state: %x", ourHash) 4321 s.sendInternalMsgLocked(accPackReqSubj, packRespIb, nil, ourHash[:]) 4322 } 4323 }) 4324 s.Noticef("Managing all jwt in exclusive directory %s", dr.directory) 4325 return nil 4326 } 4327 4328 func (dr *DirAccResolver) Fetch(name string) (string, error) { 4329 if theJWT, err := dr.LoadAcc(name); theJWT != _EMPTY_ { 4330 return theJWT, nil 4331 } else { 4332 dr.Lock() 4333 srv := dr.Server 4334 to := dr.fetchTimeout 4335 dr.Unlock() 4336 if srv == nil { 4337 return _EMPTY_, err 4338 } 4339 return srv.fetch(dr, name, to) // lookup from other server 4340 } 4341 } 4342 4343 func (dr *DirAccResolver) Store(name, jwt string) error { 4344 return dr.saveIfNewer(name, jwt) 4345 } 4346 4347 type DirResOption func(s *DirAccResolver) error 4348 4349 // limits the amount of time spent waiting for an account fetch to complete 4350 func FetchTimeout(to time.Duration) DirResOption { 4351 return func(r *DirAccResolver) error { 4352 if to <= time.Duration(0) { 4353 return fmt.Errorf("Fetch timeout %v is too smal", to) 4354 } 4355 r.fetchTimeout = to 4356 return nil 4357 } 4358 } 4359 4360 func (dr *DirAccResolver) apply(opts ...DirResOption) error { 4361 for _, o := range opts { 4362 if err := o(dr); err != nil { 4363 return err 4364 } 4365 } 4366 return nil 4367 } 4368 4369 func NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete deleteType, opts ...DirResOption) (*DirAccResolver, error) { 4370 if limit == 0 { 4371 limit = math.MaxInt64 4372 } 4373 if syncInterval <= 0 { 4374 syncInterval = time.Minute 4375 } 4376 store, err := NewExpiringDirJWTStore(path, false, true, delete, 0, limit, false, 0, nil) 4377 if err != nil { 4378 return nil, err 4379 } 4380 4381 res := &DirAccResolver{store, nil, syncInterval, DEFAULT_ACCOUNT_FETCH_TIMEOUT} 4382 if err := res.apply(opts...); err != nil { 4383 return nil, err 4384 } 4385 return res, nil 4386 } 4387 4388 // Caching resolver using nats for lookups and making use of a directory for storage 4389 type CacheDirAccResolver struct { 4390 DirAccResolver 4391 ttl time.Duration 4392 } 4393 4394 func (s *Server) fetch(res AccountResolver, name string, timeout time.Duration) (string, error) { 4395 if s == nil { 4396 return _EMPTY_, ErrNoAccountResolver 4397 } 4398 respC := make(chan []byte, 1) 4399 accountLookupRequest := fmt.Sprintf(accLookupReqSubj, name) 4400 s.mu.Lock() 4401 if s.sys == nil || s.sys.replies == nil { 4402 s.mu.Unlock() 4403 return _EMPTY_, fmt.Errorf("eventing shut down") 4404 } 4405 // Resolver will wait for detected active servers to reply 4406 // before serving an error in case there weren't any found. 4407 expectedServers := len(s.sys.servers) 4408 replySubj := s.newRespInbox() 4409 replies := s.sys.replies 4410 4411 // Store our handler. 4412 replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { 4413 var clone []byte 4414 isEmpty := len(msg) == 0 4415 if !isEmpty { 4416 clone = make([]byte, len(msg)) 4417 copy(clone, msg) 4418 } 4419 s.mu.Lock() 4420 defer s.mu.Unlock() 4421 expectedServers-- 4422 // Skip empty responses until getting all the available servers. 4423 if isEmpty && expectedServers > 0 { 4424 return 4425 } 4426 // Use the first valid response if there is still interest or 4427 // one of the empty responses to signal that it was not found. 4428 if _, ok := replies[replySubj]; ok { 4429 select { 4430 case respC <- clone: 4431 default: 4432 } 4433 } 4434 } 4435 s.sendInternalMsg(accountLookupRequest, replySubj, nil, []byte{}) 4436 quit := s.quitCh 4437 s.mu.Unlock() 4438 var err error 4439 var theJWT string 4440 select { 4441 case <-quit: 4442 err = errors.New("fetching jwt failed due to shutdown") 4443 case <-time.After(timeout): 4444 err = errors.New("fetching jwt timed out") 4445 case m := <-respC: 4446 if len(m) == 0 { 4447 err = errors.New("account jwt not found") 4448 } else if err = res.Store(name, string(m)); err == nil { 4449 theJWT = string(m) 4450 } 4451 } 4452 s.mu.Lock() 4453 delete(replies, replySubj) 4454 s.mu.Unlock() 4455 close(respC) 4456 return theJWT, err 4457 } 4458 4459 func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, opts ...DirResOption) (*CacheDirAccResolver, error) { 4460 if limit <= 0 { 4461 limit = 1_000 4462 } 4463 store, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil) 4464 if err != nil { 4465 return nil, err 4466 } 4467 res := &CacheDirAccResolver{DirAccResolver{store, nil, 0, DEFAULT_ACCOUNT_FETCH_TIMEOUT}, ttl} 4468 if err := res.apply(opts...); err != nil { 4469 return nil, err 4470 } 4471 return res, nil 4472 } 4473 4474 func (dr *CacheDirAccResolver) Start(s *Server) error { 4475 op, opKeys, strict, err := getOperatorKeys(s) 4476 if err != nil { 4477 return err 4478 } 4479 dr.Lock() 4480 defer dr.Unlock() 4481 dr.Server = s 4482 dr.operator = opKeys 4483 dr.DirJWTStore.changed = func(pubKey string) { 4484 if v, ok := s.accounts.Load(pubKey); !ok { 4485 } else if theJwt, err := dr.LoadAcc(pubKey); err != nil { 4486 s.Errorf("DirResolver - Update got error on load: %v", err) 4487 } else if err := s.updateAccountWithClaimJWT(v.(*Account), theJwt); err != nil { 4488 s.Errorf("DirResolver - Update resulted in error %v", err) 4489 } 4490 } 4491 dr.DirJWTStore.deleted = func(pubKey string) { 4492 removeCb(s, pubKey) 4493 } 4494 for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { 4495 // subscribe to account jwt update requests 4496 if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) { 4497 var pubKey string 4498 tk := strings.Split(subj, tsep) 4499 if len(tk) == accUpdateTokensNew { 4500 pubKey = tk[accReqAccIndex] 4501 } else if len(tk) == accUpdateTokensOld { 4502 pubKey = tk[accUpdateAccIdxOld] 4503 } else { 4504 s.Debugf("DirResolver - jwt update cache skipped due to bad subject %q", subj) 4505 return 4506 } 4507 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4508 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4509 } else if claim.Subject != pubKey { 4510 err := errors.New("subject does not match jwt content") 4511 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4512 } else if claim.Issuer == op && strict { 4513 err := errors.New("operator requires issuer to be a signing key") 4514 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4515 } else if _, ok := s.accounts.Load(pubKey); !ok { 4516 respondToUpdate(s, resp, pubKey, "jwt update cache skipped", nil) 4517 } else if err := claimValidate(claim); err != nil { 4518 respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err) 4519 } else if err := dr.save(pubKey, string(msg)); err != nil { 4520 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4521 } else { 4522 respondToUpdate(s, resp, pubKey, "jwt updated cache", nil) 4523 } 4524 }); err != nil { 4525 return fmt.Errorf("error setting up update handling: %v", err) 4526 } 4527 } 4528 if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) { 4529 // As this is a raw message, we need to extract payload and only decode claims from it, 4530 // in case request is sent with headers. 4531 _, msg = c.msgParts(msg) 4532 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4533 respondToUpdate(s, resp, "n/a", "jwt update cache resulted in error", err) 4534 } else if claim.Issuer == op && strict { 4535 err := errors.New("operator requires issuer to be a signing key") 4536 respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err) 4537 } else if _, ok := s.accounts.Load(claim.Subject); !ok { 4538 respondToUpdate(s, resp, claim.Subject, "jwt update cache skipped", nil) 4539 } else if err := claimValidate(claim); err != nil { 4540 respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err) 4541 } else if err := dr.save(claim.Subject, string(msg)); err != nil { 4542 respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err) 4543 } else { 4544 respondToUpdate(s, resp, claim.Subject, "jwt updated cache", nil) 4545 } 4546 }); err != nil { 4547 return fmt.Errorf("error setting up update handling: %v", err) 4548 } 4549 // respond to list requests with one message containing all account ids 4550 if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) { 4551 handleListRequest(dr.DirJWTStore, s, reply) 4552 }); err != nil { 4553 return fmt.Errorf("error setting up list request handling: %v", err) 4554 } 4555 if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { 4556 handleDeleteRequest(dr.DirJWTStore, s, msg, reply) 4557 }); err != nil { 4558 return fmt.Errorf("error setting up list request handling: %v", err) 4559 } 4560 s.Noticef("Managing some jwt in exclusive directory %s", dr.directory) 4561 return nil 4562 } 4563 4564 func (dr *CacheDirAccResolver) Reload() error { 4565 return dr.DirAccResolver.Reload() 4566 }