get.pme.sh/pnats@v0.0.0-20240304004023-26bb5a137ed0/server/accounts.go (about) 1 // Copyright 2018-2024 The NATS Authors 2 // Licensed under the Apache License, Version 2.0 (the "License"); 3 // you may not use this file except in compliance with the License. 4 // You may obtain a copy of the License at 5 // 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package server 15 16 import ( 17 "bytes" 18 "encoding/hex" 19 "errors" 20 "fmt" 21 "io" 22 "io/fs" 23 "math" 24 "math/rand" 25 "net/http" 26 "net/textproto" 27 "reflect" 28 "sort" 29 "strconv" 30 "strings" 31 "sync" 32 "sync/atomic" 33 "time" 34 35 "get.pme.sh/pnats/internal/fastrand" 36 "github.com/nats-io/jwt/v2" 37 "github.com/nats-io/nkeys" 38 "github.com/nats-io/nuid" 39 ) 40 41 // For backwards compatibility with NATS < 2.0, users who are not explicitly defined into an 42 // account will be grouped in the default global account. 43 const globalAccountName = DEFAULT_GLOBAL_ACCOUNT 44 45 const defaultMaxSubLimitReportThreshold = int64(2 * time.Second) 46 47 var maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold 48 49 // Account are subject namespace definitions. By default no messages are shared between accounts. 50 // You can share via Exports and Imports of Streams and Services. 51 type Account struct { 52 stats 53 gwReplyMapping 54 Name string 55 Nkey string 56 Issuer string 57 claimJWT string 58 updated time.Time 59 mu sync.RWMutex 60 sqmu sync.Mutex 61 sl *Sublist 62 ic *client 63 isid uint64 64 etmr *time.Timer 65 ctmr *time.Timer 66 strack map[string]sconns 67 nrclients int32 68 sysclients int32 69 nleafs int32 70 nrleafs int32 71 clients map[*client]struct{} 72 rm map[string]int32 73 lqws map[string]int32 74 usersRevoked map[string]int64 75 mappings []*mapping 76 hasMapped atomic.Bool 77 lmu sync.RWMutex 78 lleafs []*client 79 leafClusters map[string]uint64 80 imports importMap 81 exports exportMap 82 js *jsAccount 83 jsLimits map[string]JetStreamAccountLimits 84 limits 85 expired bool 86 incomplete bool 87 signingKeys map[string]jwt.Scope 88 extAuth *jwt.ExternalAuthorization 89 srv *Server // server this account is registered with (possibly nil) 90 lds string // loop detection subject for leaf nodes 91 siReply []byte // service reply prefix, will form wildcard subscription. 92 eventIds *nuid.NUID 93 eventIdsMu sync.Mutex 94 defaultPerms *Permissions 95 tags jwt.TagList 96 nameTag string 97 lastLimErr int64 98 routePoolIdx int 99 // If the trace destination is specified and a message with a traceParentHdr 100 // is received, and has the least significant bit of the last token set to 1, 101 // then if traceDestSampling is > 0 and < 100, a random value will be selected 102 // and if it falls between 0 and that value, message tracing will be triggered. 103 traceDest string 104 traceDestSampling int 105 } 106 107 const ( 108 accDedicatedRoute = -1 109 accTransitioningToDedicatedRoute = -2 110 ) 111 112 // Account based limits. 113 type limits struct { 114 mpay int32 115 msubs int32 116 mconns int32 117 mleafs int32 118 disallowBearer bool 119 } 120 121 // Used to track remote clients and leafnodes per remote server. 122 type sconns struct { 123 conns int32 124 leafs int32 125 } 126 127 // Import stream mapping struct 128 type streamImport struct { 129 acc *Account 130 from string 131 to string 132 tr *subjectTransform 133 rtr *subjectTransform 134 claim *jwt.Import 135 usePub bool 136 invalid bool 137 // This is `allow_trace` and when true and message tracing is happening, 138 // we will trace egresses past the account boundary, if `false`, we stop 139 // at the account boundary. 140 atrc bool 141 } 142 143 const ClientInfoHdr = "Nats-Request-Info" 144 145 // Import service mapping struct 146 type serviceImport struct { 147 acc *Account 148 claim *jwt.Import 149 se *serviceExport 150 sid []byte 151 from string 152 to string 153 tr *subjectTransform 154 ts int64 155 rt ServiceRespType 156 latency *serviceLatency 157 m1 *ServiceLatency 158 rc *client 159 usePub bool 160 response bool 161 invalid bool 162 share bool 163 tracking bool 164 didDeliver bool 165 atrc bool // allow trace (got from service export) 166 trackingHdr http.Header // header from request 167 } 168 169 // This is used to record when we create a mapping for implicit service 170 // imports. We use this to clean up entries that are not singletons when 171 // we detect that interest is no longer present. The key to the map will 172 // be the actual interest. We record the mapped subject and the account. 173 type serviceRespEntry struct { 174 acc *Account 175 msub string 176 } 177 178 // ServiceRespType represents the types of service request response types. 179 type ServiceRespType uint8 180 181 // Service response types. Defaults to a singleton. 182 const ( 183 Singleton ServiceRespType = iota 184 Streamed 185 Chunked 186 ) 187 188 // String helper. 189 func (rt ServiceRespType) String() string { 190 switch rt { 191 case Singleton: 192 return "Singleton" 193 case Streamed: 194 return "Streamed" 195 case Chunked: 196 return "Chunked" 197 } 198 return "Unknown ServiceResType" 199 } 200 201 // exportAuth holds configured approvals or boolean indicating an 202 // auth token is required for import. 203 type exportAuth struct { 204 tokenReq bool 205 accountPos uint 206 approved map[string]*Account 207 actsRevoked map[string]int64 208 } 209 210 // streamExport 211 type streamExport struct { 212 exportAuth 213 } 214 215 // serviceExport holds additional information for exported services. 216 type serviceExport struct { 217 exportAuth 218 acc *Account 219 respType ServiceRespType 220 latency *serviceLatency 221 rtmr *time.Timer 222 respThresh time.Duration 223 // This is `allow_trace` and when true and message tracing is happening, 224 // when processing a service import we will go through account boundary 225 // and trace egresses on that other account. If `false`, we stop at the 226 // account boundary. 227 atrc bool 228 } 229 230 // Used to track service latency. 231 type serviceLatency struct { 232 sampling int8 // percentage from 1-100 or 0 to indicate triggered by header 233 subject string 234 } 235 236 // exportMap tracks the exported streams and services. 237 type exportMap struct { 238 streams map[string]*streamExport 239 services map[string]*serviceExport 240 responses map[string]*serviceImport 241 } 242 243 // importMap tracks the imported streams and services. 244 // For services we will also track the response mappings as well. 245 type importMap struct { 246 streams []*streamImport 247 services map[string]*serviceImport 248 rrMap map[string][]*serviceRespEntry 249 } 250 251 // NewAccount creates a new unlimited account with the given name. 252 func NewAccount(name string) *Account { 253 a := &Account{ 254 Name: name, 255 limits: limits{-1, -1, -1, -1, false}, 256 eventIds: nuid.New(), 257 } 258 return a 259 } 260 261 func (a *Account) String() string { 262 return a.Name 263 } 264 265 func (a *Account) setTraceDest(dest string) { 266 a.mu.Lock() 267 a.traceDest = dest 268 a.mu.Unlock() 269 } 270 271 func (a *Account) getTraceDestAndSampling() (string, int) { 272 a.mu.RLock() 273 dest := a.traceDest 274 sampling := a.traceDestSampling 275 a.mu.RUnlock() 276 return dest, sampling 277 } 278 279 // Used to create shallow copies of accounts for transfer 280 // from opts to real accounts in server struct. 281 // Account `na` write lock is expected to be held on entry 282 // while account `a` is the one from the Options struct 283 // being loaded/reloaded and do not need locking. 284 func (a *Account) shallowCopy(na *Account) { 285 na.Nkey = a.Nkey 286 na.Issuer = a.Issuer 287 na.traceDest, na.traceDestSampling = a.traceDest, a.traceDestSampling 288 289 if a.imports.streams != nil { 290 na.imports.streams = make([]*streamImport, 0, len(a.imports.streams)) 291 for _, v := range a.imports.streams { 292 si := *v 293 na.imports.streams = append(na.imports.streams, &si) 294 } 295 } 296 if a.imports.services != nil { 297 na.imports.services = make(map[string]*serviceImport) 298 for k, v := range a.imports.services { 299 si := *v 300 na.imports.services[k] = &si 301 } 302 } 303 if a.exports.streams != nil { 304 na.exports.streams = make(map[string]*streamExport) 305 for k, v := range a.exports.streams { 306 if v != nil { 307 se := *v 308 na.exports.streams[k] = &se 309 } else { 310 na.exports.streams[k] = nil 311 } 312 } 313 } 314 if a.exports.services != nil { 315 na.exports.services = make(map[string]*serviceExport) 316 for k, v := range a.exports.services { 317 if v != nil { 318 se := *v 319 na.exports.services[k] = &se 320 } else { 321 na.exports.services[k] = nil 322 } 323 } 324 } 325 na.mappings = a.mappings 326 na.hasMapped.Store(len(na.mappings) > 0) 327 328 // JetStream 329 na.jsLimits = a.jsLimits 330 // Server config account limits. 331 na.limits = a.limits 332 } 333 334 // nextEventID uses its own lock for better concurrency. 335 func (a *Account) nextEventID() string { 336 a.eventIdsMu.Lock() 337 id := a.eventIds.Next() 338 a.eventIdsMu.Unlock() 339 return id 340 } 341 342 // Returns a slice of clients stored in the account, or nil if none is present. 343 // Lock is held on entry. 344 func (a *Account) getClientsLocked() []*client { 345 if len(a.clients) == 0 { 346 return nil 347 } 348 clients := make([]*client, 0, len(a.clients)) 349 for c := range a.clients { 350 clients = append(clients, c) 351 } 352 return clients 353 } 354 355 // Returns a slice of clients stored in the account, or nil if none is present. 356 func (a *Account) getClients() []*client { 357 a.mu.RLock() 358 clients := a.getClientsLocked() 359 a.mu.RUnlock() 360 return clients 361 } 362 363 // Called to track a remote server and connections and leafnodes it 364 // has for this account. 365 func (a *Account) updateRemoteServer(m *AccountNumConns) []*client { 366 a.mu.Lock() 367 if a.strack == nil { 368 a.strack = make(map[string]sconns) 369 } 370 // This does not depend on receiving all updates since each one is idempotent. 371 // FIXME(dlc) - We should cleanup when these both go to zero. 372 prev := a.strack[m.Server.ID] 373 a.strack[m.Server.ID] = sconns{conns: int32(m.Conns), leafs: int32(m.LeafNodes)} 374 a.nrclients += int32(m.Conns) - prev.conns 375 a.nrleafs += int32(m.LeafNodes) - prev.leafs 376 377 mtce := a.mconns != jwt.NoLimit && (len(a.clients)-int(a.sysclients)+int(a.nrclients) > int(a.mconns)) 378 // If we are over here some have snuck in and we need to rebalance. 379 // All others will probably be doing the same thing but better to be 380 // conservative and bit harsh here. Clients will reconnect if we over compensate. 381 var clients []*client 382 if mtce { 383 clients := a.getClientsLocked() 384 sort.Slice(clients, func(i, j int) bool { 385 return clients[i].start.After(clients[j].start) 386 }) 387 over := (len(a.clients) - int(a.sysclients) + int(a.nrclients)) - int(a.mconns) 388 if over < len(clients) { 389 clients = clients[:over] 390 } 391 } 392 // Now check leafnodes. 393 mtlce := a.mleafs != jwt.NoLimit && (a.nleafs+a.nrleafs > a.mleafs) 394 if mtlce { 395 // Take ones from the end. 396 a.lmu.RLock() 397 leafs := a.lleafs 398 over := int(a.nleafs + a.nrleafs - a.mleafs) 399 if over < len(leafs) { 400 leafs = leafs[len(leafs)-over:] 401 } 402 clients = append(clients, leafs...) 403 a.lmu.RUnlock() 404 } 405 a.mu.Unlock() 406 407 // If we have exceeded our max clients this will be populated. 408 return clients 409 } 410 411 // Removes tracking for a remote server that has shutdown. 412 func (a *Account) removeRemoteServer(sid string) { 413 a.mu.Lock() 414 if a.strack != nil { 415 prev := a.strack[sid] 416 delete(a.strack, sid) 417 a.nrclients -= prev.conns 418 a.nrleafs -= prev.leafs 419 } 420 a.mu.Unlock() 421 } 422 423 // When querying for subject interest this is the number of 424 // expected responses. We need to actually check that the entry 425 // has active connections. 426 func (a *Account) expectedRemoteResponses() (expected int32) { 427 a.mu.RLock() 428 for _, sc := range a.strack { 429 if sc.conns > 0 || sc.leafs > 0 { 430 expected++ 431 } 432 } 433 a.mu.RUnlock() 434 return 435 } 436 437 // Clears eventing and tracking for this account. 438 func (a *Account) clearEventing() { 439 a.mu.Lock() 440 a.nrclients = 0 441 // Now clear state 442 clearTimer(&a.etmr) 443 clearTimer(&a.ctmr) 444 a.clients = nil 445 a.strack = nil 446 a.mu.Unlock() 447 } 448 449 // GetName will return the accounts name. 450 func (a *Account) GetName() string { 451 if a == nil { 452 return "n/a" 453 } 454 a.mu.RLock() 455 name := a.Name 456 a.mu.RUnlock() 457 return name 458 } 459 460 // NumConnections returns active number of clients for this account for 461 // all known servers. 462 func (a *Account) NumConnections() int { 463 a.mu.RLock() 464 nc := len(a.clients) - int(a.sysclients) + int(a.nrclients) 465 a.mu.RUnlock() 466 return nc 467 } 468 469 // NumRemoteConnections returns the number of client or leaf connections that 470 // are not on this server. 471 func (a *Account) NumRemoteConnections() int { 472 a.mu.RLock() 473 nc := int(a.nrclients + a.nrleafs) 474 a.mu.RUnlock() 475 return nc 476 } 477 478 // NumLocalConnections returns active number of clients for this account 479 // on this server. 480 func (a *Account) NumLocalConnections() int { 481 a.mu.RLock() 482 nlc := a.numLocalConnections() 483 a.mu.RUnlock() 484 return nlc 485 } 486 487 // Do not account for the system accounts. 488 func (a *Account) numLocalConnections() int { 489 return len(a.clients) - int(a.sysclients) - int(a.nleafs) 490 } 491 492 // This is for extended local interest. 493 // Lock should not be held. 494 func (a *Account) numLocalAndLeafConnections() int { 495 a.mu.RLock() 496 nlc := len(a.clients) - int(a.sysclients) 497 a.mu.RUnlock() 498 return nlc 499 } 500 501 func (a *Account) numLocalLeafNodes() int { 502 return int(a.nleafs) 503 } 504 505 // MaxTotalConnectionsReached returns if we have reached our limit for number of connections. 506 func (a *Account) MaxTotalConnectionsReached() bool { 507 var mtce bool 508 a.mu.RLock() 509 if a.mconns != jwt.NoLimit { 510 mtce = len(a.clients)-int(a.sysclients)+int(a.nrclients) >= int(a.mconns) 511 } 512 a.mu.RUnlock() 513 return mtce 514 } 515 516 // MaxActiveConnections return the set limit for the account system 517 // wide for total number of active connections. 518 func (a *Account) MaxActiveConnections() int { 519 a.mu.RLock() 520 mconns := int(a.mconns) 521 a.mu.RUnlock() 522 return mconns 523 } 524 525 // MaxTotalLeafNodesReached returns if we have reached our limit for number of leafnodes. 526 func (a *Account) MaxTotalLeafNodesReached() bool { 527 a.mu.RLock() 528 mtc := a.maxTotalLeafNodesReached() 529 a.mu.RUnlock() 530 return mtc 531 } 532 533 func (a *Account) maxTotalLeafNodesReached() bool { 534 if a.mleafs != jwt.NoLimit { 535 return a.nleafs+a.nrleafs >= a.mleafs 536 } 537 return false 538 } 539 540 // NumLeafNodes returns the active number of local and remote 541 // leaf node connections. 542 func (a *Account) NumLeafNodes() int { 543 a.mu.RLock() 544 nln := int(a.nleafs + a.nrleafs) 545 a.mu.RUnlock() 546 return nln 547 } 548 549 // NumRemoteLeafNodes returns the active number of remote 550 // leaf node connections. 551 func (a *Account) NumRemoteLeafNodes() int { 552 a.mu.RLock() 553 nrn := int(a.nrleafs) 554 a.mu.RUnlock() 555 return nrn 556 } 557 558 // MaxActiveLeafNodes return the set limit for the account system 559 // wide for total number of leavenode connections. 560 // NOTE: these are tracked separately. 561 func (a *Account) MaxActiveLeafNodes() int { 562 a.mu.RLock() 563 mleafs := int(a.mleafs) 564 a.mu.RUnlock() 565 return mleafs 566 } 567 568 // RoutedSubs returns how many subjects we would send across a route when first 569 // connected or expressing interest. Local client subs. 570 func (a *Account) RoutedSubs() int { 571 a.mu.RLock() 572 defer a.mu.RUnlock() 573 return len(a.rm) 574 } 575 576 // TotalSubs returns total number of Subscriptions for this account. 577 func (a *Account) TotalSubs() int { 578 a.mu.RLock() 579 defer a.mu.RUnlock() 580 if a.sl == nil { 581 return 0 582 } 583 return int(a.sl.Count()) 584 } 585 586 func (a *Account) shouldLogMaxSubErr() bool { 587 if a == nil { 588 return true 589 } 590 a.mu.RLock() 591 last := a.lastLimErr 592 a.mu.RUnlock() 593 if now := time.Now().UnixNano(); now-last >= maxSubLimitReportThreshold { 594 a.mu.Lock() 595 a.lastLimErr = now 596 a.mu.Unlock() 597 return true 598 } 599 return false 600 } 601 602 // MapDest is for mapping published subjects for clients. 603 type MapDest struct { 604 Subject string `json:"subject"` 605 Weight uint8 `json:"weight"` 606 Cluster string `json:"cluster,omitempty"` 607 } 608 609 func NewMapDest(subject string, weight uint8) *MapDest { 610 return &MapDest{subject, weight, _EMPTY_} 611 } 612 613 // destination is for internal representation for a weighted mapped destination. 614 type destination struct { 615 tr *subjectTransform 616 weight uint8 617 } 618 619 // mapping is an internal entry for mapping subjects. 620 type mapping struct { 621 src string 622 wc bool 623 dests []*destination 624 cdests map[string][]*destination 625 } 626 627 // AddMapping adds in a simple route mapping from src subject to dest subject 628 // for inbound client messages. 629 func (a *Account) AddMapping(src, dest string) error { 630 return a.AddWeightedMappings(src, NewMapDest(dest, 100)) 631 } 632 633 // AddWeightedMappings will add in a weighted mappings for the destinations. 634 func (a *Account) AddWeightedMappings(src string, dests ...*MapDest) error { 635 a.mu.Lock() 636 defer a.mu.Unlock() 637 638 if !IsValidSubject(src) { 639 return ErrBadSubject 640 } 641 642 m := &mapping{src: src, wc: subjectHasWildcard(src), dests: make([]*destination, 0, len(dests)+1)} 643 seen := make(map[string]struct{}) 644 645 var tw = make(map[string]uint8) 646 for _, d := range dests { 647 if _, ok := seen[d.Subject]; ok { 648 return fmt.Errorf("duplicate entry for %q", d.Subject) 649 } 650 seen[d.Subject] = struct{}{} 651 if d.Weight > 100 { 652 return fmt.Errorf("individual weights need to be <= 100") 653 } 654 tw[d.Cluster] += d.Weight 655 if tw[d.Cluster] > 100 { 656 return fmt.Errorf("total weight needs to be <= 100") 657 } 658 err := ValidateMappingDestination(d.Subject) 659 if err != nil { 660 return err 661 } 662 tr, err := NewSubjectTransform(src, d.Subject) 663 if err != nil { 664 return err 665 } 666 if d.Cluster == _EMPTY_ { 667 m.dests = append(m.dests, &destination{tr, d.Weight}) 668 } else { 669 // We have a cluster scoped filter. 670 if m.cdests == nil { 671 m.cdests = make(map[string][]*destination) 672 } 673 ad := m.cdests[d.Cluster] 674 ad = append(ad, &destination{tr, d.Weight}) 675 m.cdests[d.Cluster] = ad 676 } 677 } 678 679 processDestinations := func(dests []*destination) ([]*destination, error) { 680 var ltw uint8 681 for _, d := range dests { 682 ltw += d.weight 683 } 684 // Auto add in original at weight difference if all entries weight does not total to 100. 685 // Iff the src was not already added in explicitly, meaning they want loss. 686 _, haveSrc := seen[src] 687 if ltw != 100 && !haveSrc { 688 dest := src 689 if m.wc { 690 // We need to make the appropriate markers for the wildcards etc. 691 dest = transformTokenize(dest) 692 } 693 tr, err := NewSubjectTransform(src, dest) 694 if err != nil { 695 return nil, err 696 } 697 aw := 100 - ltw 698 if len(dests) == 0 { 699 aw = 100 700 } 701 dests = append(dests, &destination{tr, aw}) 702 } 703 sort.Slice(dests, func(i, j int) bool { return dests[i].weight < dests[j].weight }) 704 705 var lw uint8 706 for _, d := range dests { 707 d.weight += lw 708 lw = d.weight 709 } 710 return dests, nil 711 } 712 713 var err error 714 if m.dests, err = processDestinations(m.dests); err != nil { 715 return err 716 } 717 718 // Option cluster scoped destinations 719 for cluster, dests := range m.cdests { 720 if dests, err = processDestinations(dests); err != nil { 721 return err 722 } 723 m.cdests[cluster] = dests 724 } 725 726 // Replace an old one if it exists. 727 for i, em := range a.mappings { 728 if em.src == src { 729 a.mappings[i] = m 730 return nil 731 } 732 } 733 // If we did not replace add to the end. 734 a.mappings = append(a.mappings, m) 735 a.hasMapped.Store(len(a.mappings) > 0) 736 737 // If we have connected leafnodes make sure to update. 738 if a.nleafs > 0 { 739 // Need to release because lock ordering is client -> account 740 a.mu.Unlock() 741 // Now grab the leaf list lock. We can hold client lock under this one. 742 a.lmu.RLock() 743 for _, lc := range a.lleafs { 744 lc.forceAddToSmap(src) 745 } 746 a.lmu.RUnlock() 747 a.mu.Lock() 748 } 749 return nil 750 } 751 752 // RemoveMapping will remove an existing mapping. 753 func (a *Account) RemoveMapping(src string) bool { 754 a.mu.Lock() 755 defer a.mu.Unlock() 756 for i, m := range a.mappings { 757 if m.src == src { 758 // Swap last one into this spot. Its ok to change order. 759 a.mappings[i] = a.mappings[len(a.mappings)-1] 760 a.mappings[len(a.mappings)-1] = nil // gc 761 a.mappings = a.mappings[:len(a.mappings)-1] 762 a.hasMapped.Store(len(a.mappings) > 0) 763 // If we have connected leafnodes make sure to update. 764 if a.nleafs > 0 { 765 // Need to release because lock ordering is client -> account 766 a.mu.Unlock() 767 // Now grab the leaf list lock. We can hold client lock under this one. 768 a.lmu.RLock() 769 for _, lc := range a.lleafs { 770 lc.forceRemoveFromSmap(src) 771 } 772 a.lmu.RUnlock() 773 a.mu.Lock() 774 } 775 return true 776 } 777 } 778 return false 779 } 780 781 // Indicates we have mapping entries. 782 func (a *Account) hasMappings() bool { 783 if a == nil { 784 return false 785 } 786 return a.hasMapped.Load() 787 } 788 789 // This performs the logic to map to a new dest subject based on mappings. 790 // Should only be called from processInboundClientMsg or service import processing. 791 func (a *Account) selectMappedSubject(dest string) (string, bool) { 792 if !a.hasMappings() { 793 return dest, false 794 } 795 796 a.mu.Lock() 797 // In case we have to tokenize for subset matching. 798 tsa := [32]string{} 799 tts := tsa[:0] 800 801 var m *mapping 802 for _, rm := range a.mappings { 803 if !rm.wc && rm.src == dest { 804 m = rm 805 break 806 } else { 807 // tokenize and reuse for subset matching. 808 if len(tts) == 0 { 809 start := 0 810 subject := dest 811 for i := 0; i < len(subject); i++ { 812 if subject[i] == btsep { 813 tts = append(tts, subject[start:i]) 814 start = i + 1 815 } 816 } 817 tts = append(tts, subject[start:]) 818 } 819 if isSubsetMatch(tts, rm.src) { 820 m = rm 821 break 822 } 823 } 824 } 825 826 if m == nil { 827 a.mu.Unlock() 828 return dest, false 829 } 830 831 // The selected destination for the mapping. 832 var d *destination 833 var ndest string 834 835 dests := m.dests 836 if len(m.cdests) > 0 { 837 cn := a.srv.cachedClusterName() 838 dests = m.cdests[cn] 839 if dests == nil { 840 // Fallback to main if we do not match the cluster. 841 dests = m.dests 842 } 843 } 844 845 // Optimize for single entry case. 846 if len(dests) == 1 && dests[0].weight == 100 { 847 d = dests[0] 848 } else { 849 w := uint8(fastrand.Uint32n(100)) 850 for _, rm := range dests { 851 if w < rm.weight { 852 d = rm 853 break 854 } 855 } 856 } 857 858 if d != nil { 859 if len(d.tr.dtokmftokindexesargs) == 0 { 860 ndest = d.tr.dest 861 } else { 862 ndest = d.tr.TransformTokenizedSubject(tts) 863 } 864 } 865 866 a.mu.Unlock() 867 return ndest, true 868 } 869 870 // SubscriptionInterest returns true if this account has a matching subscription 871 // for the given `subject`. 872 func (a *Account) SubscriptionInterest(subject string) bool { 873 return a.Interest(subject) > 0 874 } 875 876 // Interest returns the number of subscriptions for a given subject that match. 877 func (a *Account) Interest(subject string) int { 878 var nms int 879 a.mu.RLock() 880 if a.sl != nil { 881 res := a.sl.Match(subject) 882 nms = len(res.psubs) + len(res.qsubs) 883 } 884 a.mu.RUnlock() 885 return nms 886 } 887 888 // addClient keeps our accounting of local active clients or leafnodes updated. 889 // Returns previous total. 890 func (a *Account) addClient(c *client) int { 891 a.mu.Lock() 892 n := len(a.clients) 893 if a.clients != nil { 894 a.clients[c] = struct{}{} 895 } 896 added := n != len(a.clients) 897 if added { 898 if c.kind != CLIENT && c.kind != LEAF { 899 a.sysclients++ 900 } else if c.kind == LEAF { 901 a.nleafs++ 902 } 903 } 904 a.mu.Unlock() 905 906 // If we added a new leaf use the list lock and add it to the list. 907 if added && c.kind == LEAF { 908 a.lmu.Lock() 909 a.lleafs = append(a.lleafs, c) 910 a.lmu.Unlock() 911 } 912 913 if c != nil && c.srv != nil && added { 914 c.srv.accConnsUpdate(a) 915 } 916 917 return n 918 } 919 920 // For registering clusters for remote leafnodes. 921 // We only register as the hub. 922 func (a *Account) registerLeafNodeCluster(cluster string) { 923 a.mu.Lock() 924 defer a.mu.Unlock() 925 if a.leafClusters == nil { 926 a.leafClusters = make(map[string]uint64) 927 } 928 a.leafClusters[cluster]++ 929 } 930 931 // Check to see if we already have this cluster registered. 932 func (a *Account) hasLeafNodeCluster(cluster string) bool { 933 a.mu.RLock() 934 defer a.mu.RUnlock() 935 return a.leafClusters[cluster] > 0 936 } 937 938 // Check to see if this cluster is isolated, meaning the only one. 939 // Read Lock should be held. 940 func (a *Account) isLeafNodeClusterIsolated(cluster string) bool { 941 if cluster == _EMPTY_ { 942 return false 943 } 944 if len(a.leafClusters) > 1 { 945 return false 946 } 947 return a.leafClusters[cluster] == uint64(a.nleafs) 948 } 949 950 // Helper function to remove leaf nodes. If number of leafnodes gets large 951 // this may need to be optimized out of linear search but believe number 952 // of active leafnodes per account scope to be small and therefore cache friendly. 953 // Lock should not be held on general account lock. 954 func (a *Account) removeLeafNode(c *client) { 955 // Make sure we hold the list lock as well. 956 a.lmu.Lock() 957 defer a.lmu.Unlock() 958 959 ll := len(a.lleafs) 960 for i, l := range a.lleafs { 961 if l == c { 962 a.lleafs[i] = a.lleafs[ll-1] 963 if ll == 1 { 964 a.lleafs = nil 965 } else { 966 a.lleafs = a.lleafs[:ll-1] 967 } 968 return 969 } 970 } 971 } 972 973 // removeClient keeps our accounting of local active clients updated. 974 func (a *Account) removeClient(c *client) int { 975 a.mu.Lock() 976 n := len(a.clients) 977 delete(a.clients, c) 978 removed := n != len(a.clients) 979 if removed { 980 if c.kind != CLIENT && c.kind != LEAF { 981 a.sysclients-- 982 } else if c.kind == LEAF { 983 a.nleafs-- 984 // Need to do cluster accounting here. 985 // Do cluster accounting if we are a hub. 986 if c.isHubLeafNode() { 987 cluster := c.remoteCluster() 988 if count := a.leafClusters[cluster]; count > 1 { 989 a.leafClusters[cluster]-- 990 } else if count == 1 { 991 delete(a.leafClusters, cluster) 992 } 993 } 994 } 995 } 996 a.mu.Unlock() 997 998 if removed && c.kind == LEAF { 999 a.removeLeafNode(c) 1000 } 1001 1002 if c != nil && c.srv != nil && removed { 1003 c.srv.accConnsUpdate(a) 1004 } 1005 1006 return n 1007 } 1008 1009 func setExportAuth(ea *exportAuth, subject string, accounts []*Account, accountPos uint) error { 1010 if accountPos > 0 { 1011 token := strings.Split(subject, tsep) 1012 if len(token) < int(accountPos) || token[accountPos-1] != "*" { 1013 return ErrInvalidSubject 1014 } 1015 } 1016 ea.accountPos = accountPos 1017 // empty means auth required but will be import token. 1018 if accounts == nil { 1019 return nil 1020 } 1021 if len(accounts) == 0 { 1022 ea.tokenReq = true 1023 return nil 1024 } 1025 if ea.approved == nil { 1026 ea.approved = make(map[string]*Account, len(accounts)) 1027 } 1028 for _, acc := range accounts { 1029 ea.approved[acc.Name] = acc 1030 } 1031 return nil 1032 } 1033 1034 // AddServiceExport will configure the account with the defined export. 1035 func (a *Account) AddServiceExport(subject string, accounts []*Account) error { 1036 return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, 0) 1037 } 1038 1039 // AddServiceExport will configure the account with the defined export. 1040 func (a *Account) addServiceExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error { 1041 return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, accountPos) 1042 } 1043 1044 // AddServiceExportWithResponse will configure the account with the defined export and response type. 1045 func (a *Account) AddServiceExportWithResponse(subject string, respType ServiceRespType, accounts []*Account) error { 1046 return a.addServiceExportWithResponseAndAccountPos(subject, respType, accounts, 0) 1047 } 1048 1049 // AddServiceExportWithresponse will configure the account with the defined export and response type. 1050 func (a *Account) addServiceExportWithResponseAndAccountPos( 1051 subject string, respType ServiceRespType, accounts []*Account, accountPos uint) error { 1052 if a == nil { 1053 return ErrMissingAccount 1054 } 1055 1056 a.mu.Lock() 1057 if a.exports.services == nil { 1058 a.exports.services = make(map[string]*serviceExport) 1059 } 1060 1061 se := a.exports.services[subject] 1062 // Always create a service export 1063 if se == nil { 1064 se = &serviceExport{} 1065 } 1066 1067 if respType != Singleton { 1068 se.respType = respType 1069 } 1070 1071 if accounts != nil || accountPos > 0 { 1072 if err := setExportAuth(&se.exportAuth, subject, accounts, accountPos); err != nil { 1073 a.mu.Unlock() 1074 return err 1075 } 1076 } 1077 lrt := a.lowestServiceExportResponseTime() 1078 se.acc = a 1079 se.respThresh = DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD 1080 a.exports.services[subject] = se 1081 1082 var clients []*client 1083 nlrt := a.lowestServiceExportResponseTime() 1084 if nlrt != lrt && len(a.clients) > 0 { 1085 clients = a.getClientsLocked() 1086 } 1087 // Need to release because lock ordering is client -> Account 1088 a.mu.Unlock() 1089 if len(clients) > 0 { 1090 updateAllClientsServiceExportResponseTime(clients, nlrt) 1091 } 1092 return nil 1093 } 1094 1095 // TrackServiceExport will enable latency tracking of the named service. 1096 // Results will be published in this account to the given results subject. 1097 func (a *Account) TrackServiceExport(service, results string) error { 1098 return a.TrackServiceExportWithSampling(service, results, DEFAULT_SERVICE_LATENCY_SAMPLING) 1099 } 1100 1101 // TrackServiceExportWithSampling will enable latency tracking of the named service for the given 1102 // sampling rate (1-100). Results will be published in this account to the given results subject. 1103 func (a *Account) TrackServiceExportWithSampling(service, results string, sampling int) error { 1104 if a == nil { 1105 return ErrMissingAccount 1106 } 1107 1108 if sampling != 0 { // 0 means triggered by header 1109 if sampling < 1 || sampling > 100 { 1110 return ErrBadSampling 1111 } 1112 } 1113 if !IsValidPublishSubject(results) { 1114 return ErrBadPublishSubject 1115 } 1116 // Don't loop back on outselves. 1117 if a.IsExportService(results) { 1118 return ErrBadPublishSubject 1119 } 1120 1121 if a.srv != nil && !a.srv.EventsEnabled() { 1122 return ErrNoSysAccount 1123 } 1124 1125 a.mu.Lock() 1126 if a.exports.services == nil { 1127 a.mu.Unlock() 1128 return ErrMissingService 1129 } 1130 ea, ok := a.exports.services[service] 1131 if !ok { 1132 a.mu.Unlock() 1133 return ErrMissingService 1134 } 1135 if ea == nil { 1136 ea = &serviceExport{} 1137 a.exports.services[service] = ea 1138 } else if ea.respType != Singleton { 1139 a.mu.Unlock() 1140 return ErrBadServiceType 1141 } 1142 ea.latency = &serviceLatency{ 1143 sampling: int8(sampling), 1144 subject: results, 1145 } 1146 s := a.srv 1147 a.mu.Unlock() 1148 1149 if s == nil { 1150 return nil 1151 } 1152 1153 // Now track down the imports and add in latency as needed to enable. 1154 s.accounts.Range(func(k, v interface{}) bool { 1155 acc := v.(*Account) 1156 acc.mu.Lock() 1157 for _, im := range acc.imports.services { 1158 if im != nil && im.acc.Name == a.Name && subjectIsSubsetMatch(im.to, service) { 1159 im.latency = ea.latency 1160 } 1161 } 1162 acc.mu.Unlock() 1163 return true 1164 }) 1165 1166 return nil 1167 } 1168 1169 // UnTrackServiceExport will disable latency tracking of the named service. 1170 func (a *Account) UnTrackServiceExport(service string) { 1171 if a == nil || (a.srv != nil && !a.srv.EventsEnabled()) { 1172 return 1173 } 1174 1175 a.mu.Lock() 1176 if a.exports.services == nil { 1177 a.mu.Unlock() 1178 return 1179 } 1180 ea, ok := a.exports.services[service] 1181 if !ok || ea == nil || ea.latency == nil { 1182 a.mu.Unlock() 1183 return 1184 } 1185 // We have latency here. 1186 ea.latency = nil 1187 s := a.srv 1188 a.mu.Unlock() 1189 1190 if s == nil { 1191 return 1192 } 1193 1194 // Now track down the imports and clean them up. 1195 s.accounts.Range(func(k, v interface{}) bool { 1196 acc := v.(*Account) 1197 acc.mu.Lock() 1198 for _, im := range acc.imports.services { 1199 if im != nil && im.acc.Name == a.Name { 1200 if subjectIsSubsetMatch(im.to, service) { 1201 im.latency, im.m1 = nil, nil 1202 } 1203 } 1204 } 1205 acc.mu.Unlock() 1206 return true 1207 }) 1208 } 1209 1210 // IsExportService will indicate if this service exists. Will check wildcard scenarios. 1211 func (a *Account) IsExportService(service string) bool { 1212 a.mu.RLock() 1213 defer a.mu.RUnlock() 1214 _, ok := a.exports.services[service] 1215 if ok { 1216 return true 1217 } 1218 tokens := strings.Split(service, tsep) 1219 for subj := range a.exports.services { 1220 if isSubsetMatch(tokens, subj) { 1221 return true 1222 } 1223 } 1224 return false 1225 } 1226 1227 // IsExportServiceTracking will indicate if given publish subject is an export service with tracking enabled. 1228 func (a *Account) IsExportServiceTracking(service string) bool { 1229 a.mu.RLock() 1230 ea, ok := a.exports.services[service] 1231 if ok && ea == nil { 1232 a.mu.RUnlock() 1233 return false 1234 } 1235 if ok && ea != nil && ea.latency != nil { 1236 a.mu.RUnlock() 1237 return true 1238 } 1239 // FIXME(dlc) - Might want to cache this is in the hot path checking for latency tracking. 1240 tokens := strings.Split(service, tsep) 1241 for subj, ea := range a.exports.services { 1242 if isSubsetMatch(tokens, subj) && ea != nil && ea.latency != nil { 1243 a.mu.RUnlock() 1244 return true 1245 } 1246 } 1247 a.mu.RUnlock() 1248 return false 1249 } 1250 1251 // ServiceLatency is the JSON message sent out in response to latency tracking for 1252 // an accounts exported services. Additional client info is available in requestor 1253 // and responder. Note that for a requestor, the only information shared by default 1254 // is the RTT used to calculate the total latency. The requestor's account can 1255 // designate to share the additional information in the service import. 1256 type ServiceLatency struct { 1257 TypedEvent 1258 Status int `json:"status"` 1259 Error string `json:"description,omitempty"` 1260 Requestor *ClientInfo `json:"requestor,omitempty"` 1261 Responder *ClientInfo `json:"responder,omitempty"` 1262 RequestHeader http.Header `json:"header,omitempty"` // only contains header(s) triggering the measurement 1263 RequestStart time.Time `json:"start"` 1264 ServiceLatency time.Duration `json:"service"` 1265 SystemLatency time.Duration `json:"system"` 1266 TotalLatency time.Duration `json:"total"` 1267 } 1268 1269 // ServiceLatencyType is the NATS Event Type for ServiceLatency 1270 const ServiceLatencyType = "io.nats.server.metric.v1.service_latency" 1271 1272 // NATSTotalTime is a helper function that totals the NATS latencies. 1273 func (m1 *ServiceLatency) NATSTotalTime() time.Duration { 1274 return m1.Requestor.RTT + m1.Responder.RTT + m1.SystemLatency 1275 } 1276 1277 // Merge function to merge m1 and m2 (requestor and responder) measurements 1278 // when there are two samples. This happens when the requestor and responder 1279 // are on different servers. 1280 // 1281 // m2 ServiceLatency is correct, so use that. 1282 // m1 TotalLatency is correct, so use that. 1283 // Will use those to back into NATS latency. 1284 func (m1 *ServiceLatency) merge(m2 *ServiceLatency) { 1285 rtt := time.Duration(0) 1286 if m2.Responder != nil { 1287 rtt = m2.Responder.RTT 1288 } 1289 m1.SystemLatency = m1.ServiceLatency - (m2.ServiceLatency + rtt) 1290 m1.ServiceLatency = m2.ServiceLatency 1291 m1.Responder = m2.Responder 1292 sanitizeLatencyMetric(m1) 1293 } 1294 1295 // sanitizeLatencyMetric adjusts latency metric values that could go 1296 // negative in some edge conditions since we estimate client RTT 1297 // for both requestor and responder. 1298 // These numbers are never meant to be negative, it just could be 1299 // how we back into the values based on estimated RTT. 1300 func sanitizeLatencyMetric(sl *ServiceLatency) { 1301 if sl.ServiceLatency < 0 { 1302 sl.ServiceLatency = 0 1303 } 1304 if sl.SystemLatency < 0 { 1305 sl.SystemLatency = 0 1306 } 1307 } 1308 1309 // Used for transporting remote latency measurements. 1310 type remoteLatency struct { 1311 Account string `json:"account"` 1312 ReqId string `json:"req_id"` 1313 M2 ServiceLatency `json:"m2"` 1314 respThresh time.Duration 1315 } 1316 1317 // sendLatencyResult will send a latency result and clear the si of the requestor(rc). 1318 func (a *Account) sendLatencyResult(si *serviceImport, sl *ServiceLatency) { 1319 sl.Type = ServiceLatencyType 1320 sl.ID = a.nextEventID() 1321 sl.Time = time.Now().UTC() 1322 a.mu.Lock() 1323 lsubj := si.latency.subject 1324 si.rc = nil 1325 a.mu.Unlock() 1326 1327 a.srv.sendInternalAccountMsg(a, lsubj, sl) 1328 } 1329 1330 // Used to send a bad request metric when we do not have a reply subject 1331 func (a *Account) sendBadRequestTrackingLatency(si *serviceImport, requestor *client, header http.Header) { 1332 sl := &ServiceLatency{ 1333 Status: 400, 1334 Error: "Bad Request", 1335 Requestor: requestor.getClientInfo(si.share), 1336 } 1337 sl.RequestHeader = header 1338 sl.RequestStart = time.Now().Add(-sl.Requestor.RTT).UTC() 1339 a.sendLatencyResult(si, sl) 1340 } 1341 1342 // Used to send a latency result when the requestor interest was lost before the 1343 // response could be delivered. 1344 func (a *Account) sendReplyInterestLostTrackLatency(si *serviceImport) { 1345 sl := &ServiceLatency{ 1346 Status: 408, 1347 Error: "Request Timeout", 1348 } 1349 a.mu.RLock() 1350 rc, share, ts := si.rc, si.share, si.ts 1351 sl.RequestHeader = si.trackingHdr 1352 a.mu.RUnlock() 1353 if rc != nil { 1354 sl.Requestor = rc.getClientInfo(share) 1355 } 1356 sl.RequestStart = time.Unix(0, ts-int64(sl.Requestor.RTT)).UTC() 1357 a.sendLatencyResult(si, sl) 1358 } 1359 1360 func (a *Account) sendBackendErrorTrackingLatency(si *serviceImport, reason rsiReason) { 1361 sl := &ServiceLatency{} 1362 a.mu.RLock() 1363 rc, share, ts := si.rc, si.share, si.ts 1364 sl.RequestHeader = si.trackingHdr 1365 a.mu.RUnlock() 1366 if rc != nil { 1367 sl.Requestor = rc.getClientInfo(share) 1368 } 1369 var reqRTT time.Duration 1370 if sl.Requestor != nil { 1371 reqRTT = sl.Requestor.RTT 1372 } 1373 sl.RequestStart = time.Unix(0, ts-int64(reqRTT)).UTC() 1374 if reason == rsiNoDelivery { 1375 sl.Status = 503 1376 sl.Error = "Service Unavailable" 1377 } else if reason == rsiTimeout { 1378 sl.Status = 504 1379 sl.Error = "Service Timeout" 1380 } 1381 a.sendLatencyResult(si, sl) 1382 } 1383 1384 // sendTrackingLatency will send out the appropriate tracking information for the 1385 // service request/response latency. This is called when the requestor's server has 1386 // received the response. 1387 // TODO(dlc) - holding locks for RTTs may be too much long term. Should revisit. 1388 func (a *Account) sendTrackingLatency(si *serviceImport, responder *client) bool { 1389 a.mu.RLock() 1390 rc := si.rc 1391 a.mu.RUnlock() 1392 if rc == nil { 1393 return true 1394 } 1395 1396 ts := time.Now() 1397 serviceRTT := time.Duration(ts.UnixNano() - si.ts) 1398 requestor := si.rc 1399 1400 sl := &ServiceLatency{ 1401 Status: 200, 1402 Requestor: requestor.getClientInfo(si.share), 1403 Responder: responder.getClientInfo(true), 1404 } 1405 var respRTT, reqRTT time.Duration 1406 if sl.Responder != nil { 1407 respRTT = sl.Responder.RTT 1408 } 1409 if sl.Requestor != nil { 1410 reqRTT = sl.Requestor.RTT 1411 } 1412 sl.RequestStart = time.Unix(0, si.ts-int64(reqRTT)).UTC() 1413 sl.ServiceLatency = serviceRTT - respRTT 1414 sl.TotalLatency = reqRTT + serviceRTT 1415 if respRTT > 0 { 1416 sl.SystemLatency = time.Since(ts) 1417 sl.TotalLatency += sl.SystemLatency 1418 } 1419 sl.RequestHeader = si.trackingHdr 1420 sanitizeLatencyMetric(sl) 1421 1422 sl.Type = ServiceLatencyType 1423 sl.ID = a.nextEventID() 1424 sl.Time = time.Now().UTC() 1425 1426 // If we are expecting a remote measurement, store our sl here. 1427 // We need to account for the race between this and us receiving the 1428 // remote measurement. 1429 // FIXME(dlc) - We need to clean these up but this should happen 1430 // already with the auto-expire logic. 1431 if responder != nil && responder.kind != CLIENT { 1432 si.acc.mu.Lock() 1433 if si.m1 != nil { 1434 m1, m2 := sl, si.m1 1435 m1.merge(m2) 1436 si.acc.mu.Unlock() 1437 a.srv.sendInternalAccountMsg(a, si.latency.subject, m1) 1438 a.mu.Lock() 1439 si.rc = nil 1440 a.mu.Unlock() 1441 return true 1442 } 1443 si.m1 = sl 1444 si.acc.mu.Unlock() 1445 return false 1446 } else { 1447 a.srv.sendInternalAccountMsg(a, si.latency.subject, sl) 1448 a.mu.Lock() 1449 si.rc = nil 1450 a.mu.Unlock() 1451 } 1452 return true 1453 } 1454 1455 // This will check to make sure our response lower threshold is set 1456 // properly in any clients doing rrTracking. 1457 func updateAllClientsServiceExportResponseTime(clients []*client, lrt time.Duration) { 1458 for _, c := range clients { 1459 c.mu.Lock() 1460 if c.rrTracking != nil && lrt != c.rrTracking.lrt { 1461 c.rrTracking.lrt = lrt 1462 if c.rrTracking.ptmr.Stop() { 1463 c.rrTracking.ptmr.Reset(lrt) 1464 } 1465 } 1466 c.mu.Unlock() 1467 } 1468 } 1469 1470 // Will select the lowest respThresh from all service exports. 1471 // Read lock should be held. 1472 func (a *Account) lowestServiceExportResponseTime() time.Duration { 1473 // Lowest we will allow is 5 minutes. Its an upper bound for this function. 1474 lrt := 5 * time.Minute 1475 for _, se := range a.exports.services { 1476 if se.respThresh < lrt { 1477 lrt = se.respThresh 1478 } 1479 } 1480 return lrt 1481 } 1482 1483 // AddServiceImportWithClaim will add in the service import via the jwt claim. 1484 func (a *Account) AddServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import) error { 1485 return a.addServiceImportWithClaim(destination, from, to, imClaim, false) 1486 } 1487 1488 // addServiceImportWithClaim will add in the service import via the jwt claim. 1489 // It will also skip the authorization check in cases where internal is true 1490 func (a *Account) addServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import, internal bool) error { 1491 if destination == nil { 1492 return ErrMissingAccount 1493 } 1494 // Empty means use from. 1495 if to == _EMPTY_ { 1496 to = from 1497 } 1498 if !IsValidSubject(from) || !IsValidSubject(to) { 1499 return ErrInvalidSubject 1500 } 1501 1502 // First check to see if the account has authorized us to route to the "to" subject. 1503 if !internal && !destination.checkServiceImportAuthorized(a, to, imClaim) { 1504 return ErrServiceImportAuthorization 1505 } 1506 1507 // Check if this introduces a cycle before proceeding. 1508 if err := a.serviceImportFormsCycle(destination, from); err != nil { 1509 return err 1510 } 1511 1512 _, err := a.addServiceImport(destination, from, to, imClaim) 1513 1514 return err 1515 } 1516 1517 const MaxAccountCycleSearchDepth = 1024 1518 1519 func (a *Account) serviceImportFormsCycle(dest *Account, from string) error { 1520 return dest.checkServiceImportsForCycles(from, map[string]bool{a.Name: true}) 1521 } 1522 1523 func (a *Account) checkServiceImportsForCycles(from string, visited map[string]bool) error { 1524 if len(visited) >= MaxAccountCycleSearchDepth { 1525 return ErrCycleSearchDepth 1526 } 1527 a.mu.RLock() 1528 for _, si := range a.imports.services { 1529 if SubjectsCollide(from, si.to) { 1530 a.mu.RUnlock() 1531 if visited[si.acc.Name] { 1532 return ErrImportFormsCycle 1533 } 1534 // Push ourselves and check si.acc 1535 visited[a.Name] = true 1536 if subjectIsSubsetMatch(si.from, from) { 1537 from = si.from 1538 } 1539 if err := si.acc.checkServiceImportsForCycles(from, visited); err != nil { 1540 return err 1541 } 1542 a.mu.RLock() 1543 } 1544 } 1545 a.mu.RUnlock() 1546 return nil 1547 } 1548 1549 func (a *Account) streamImportFormsCycle(dest *Account, to string) error { 1550 return dest.checkStreamImportsForCycles(to, map[string]bool{a.Name: true}) 1551 } 1552 1553 // Lock should be held. 1554 func (a *Account) hasServiceExportMatching(to string) bool { 1555 for subj := range a.exports.services { 1556 if subjectIsSubsetMatch(to, subj) { 1557 return true 1558 } 1559 } 1560 return false 1561 } 1562 1563 // Lock should be held. 1564 func (a *Account) hasStreamExportMatching(to string) bool { 1565 for subj := range a.exports.streams { 1566 if subjectIsSubsetMatch(to, subj) { 1567 return true 1568 } 1569 } 1570 return false 1571 } 1572 1573 func (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool) error { 1574 if len(visited) >= MaxAccountCycleSearchDepth { 1575 return ErrCycleSearchDepth 1576 } 1577 1578 a.mu.RLock() 1579 1580 if !a.hasStreamExportMatching(to) { 1581 a.mu.RUnlock() 1582 return nil 1583 } 1584 1585 for _, si := range a.imports.streams { 1586 if SubjectsCollide(to, si.to) { 1587 a.mu.RUnlock() 1588 if visited[si.acc.Name] { 1589 return ErrImportFormsCycle 1590 } 1591 // Push ourselves and check si.acc 1592 visited[a.Name] = true 1593 if subjectIsSubsetMatch(si.to, to) { 1594 to = si.to 1595 } 1596 if err := si.acc.checkStreamImportsForCycles(to, visited); err != nil { 1597 return err 1598 } 1599 a.mu.RLock() 1600 } 1601 } 1602 a.mu.RUnlock() 1603 return nil 1604 } 1605 1606 // SetServiceImportSharing will allow sharing of information about requests with the export account. 1607 // Used for service latency tracking at the moment. 1608 func (a *Account) SetServiceImportSharing(destination *Account, to string, allow bool) error { 1609 return a.setServiceImportSharing(destination, to, true, allow) 1610 } 1611 1612 // setServiceImportSharing will allow sharing of information about requests with the export account. 1613 func (a *Account) setServiceImportSharing(destination *Account, to string, check, allow bool) error { 1614 a.mu.Lock() 1615 defer a.mu.Unlock() 1616 if check && a.isClaimAccount() { 1617 return fmt.Errorf("claim based accounts can not be updated directly") 1618 } 1619 for _, si := range a.imports.services { 1620 if si.acc == destination && si.to == to { 1621 si.share = allow 1622 return nil 1623 } 1624 } 1625 return fmt.Errorf("service import not found") 1626 } 1627 1628 // AddServiceImport will add a route to an account to send published messages / requests 1629 // to the destination account. From is the local subject to map, To is the 1630 // subject that will appear on the destination account. Destination will need 1631 // to have an import rule to allow access via addService. 1632 func (a *Account) AddServiceImport(destination *Account, from, to string) error { 1633 return a.AddServiceImportWithClaim(destination, from, to, nil) 1634 } 1635 1636 // NumPendingReverseResponses returns the number of response mappings we have for all outstanding 1637 // requests for service imports. 1638 func (a *Account) NumPendingReverseResponses() int { 1639 a.mu.RLock() 1640 defer a.mu.RUnlock() 1641 return len(a.imports.rrMap) 1642 } 1643 1644 // NumPendingAllResponses return the number of all responses outstanding for service exports. 1645 func (a *Account) NumPendingAllResponses() int { 1646 return a.NumPendingResponses(_EMPTY_) 1647 } 1648 1649 // NumPendingResponses returns the number of responses outstanding for service exports 1650 // on this account. An empty filter string returns all responses regardless of which export. 1651 // If you specify the filter we will only return ones that are for that export. 1652 // NOTE this is only for what this server is tracking. 1653 func (a *Account) NumPendingResponses(filter string) int { 1654 a.mu.RLock() 1655 defer a.mu.RUnlock() 1656 if filter == _EMPTY_ { 1657 return len(a.exports.responses) 1658 } 1659 se := a.getServiceExport(filter) 1660 if se == nil { 1661 return 0 1662 } 1663 var nre int 1664 for _, si := range a.exports.responses { 1665 if si.se == se { 1666 nre++ 1667 } 1668 } 1669 return nre 1670 } 1671 1672 // NumServiceImports returns the number of service imports we have configured. 1673 func (a *Account) NumServiceImports() int { 1674 a.mu.RLock() 1675 defer a.mu.RUnlock() 1676 return len(a.imports.services) 1677 } 1678 1679 // Reason why we are removing this response serviceImport. 1680 type rsiReason int 1681 1682 const ( 1683 rsiOk = rsiReason(iota) 1684 rsiNoDelivery 1685 rsiTimeout 1686 ) 1687 1688 // removeRespServiceImport removes a response si mapping and the reverse entries for interest detection. 1689 func (a *Account) removeRespServiceImport(si *serviceImport, reason rsiReason) { 1690 if si == nil { 1691 return 1692 } 1693 1694 a.mu.Lock() 1695 c := a.ic 1696 delete(a.exports.responses, si.from) 1697 dest, to, tracking, rc, didDeliver := si.acc, si.to, si.tracking, si.rc, si.didDeliver 1698 a.mu.Unlock() 1699 1700 // If we have a sid make sure to unsub. 1701 if len(si.sid) > 0 && c != nil { 1702 c.processUnsub(si.sid) 1703 } 1704 1705 if tracking && rc != nil && !didDeliver { 1706 a.sendBackendErrorTrackingLatency(si, reason) 1707 } 1708 1709 dest.checkForReverseEntry(to, si, false) 1710 } 1711 1712 // removeServiceImport will remove the route by subject. 1713 func (a *Account) removeServiceImport(subject string) { 1714 a.mu.Lock() 1715 si, ok := a.imports.services[subject] 1716 delete(a.imports.services, subject) 1717 1718 var sid []byte 1719 c := a.ic 1720 1721 if ok && si != nil { 1722 if a.ic != nil && si.sid != nil { 1723 sid = si.sid 1724 } 1725 } 1726 a.mu.Unlock() 1727 1728 if sid != nil { 1729 c.processUnsub(sid) 1730 } 1731 } 1732 1733 // This tracks responses to service requests mappings. This is used for cleanup. 1734 func (a *Account) addReverseRespMapEntry(acc *Account, reply, from string) { 1735 a.mu.Lock() 1736 if a.imports.rrMap == nil { 1737 a.imports.rrMap = make(map[string][]*serviceRespEntry) 1738 } 1739 sre := &serviceRespEntry{acc, from} 1740 sra := a.imports.rrMap[reply] 1741 a.imports.rrMap[reply] = append(sra, sre) 1742 a.mu.Unlock() 1743 } 1744 1745 // checkForReverseEntries is for when we are trying to match reverse entries to a wildcard. 1746 // This will be called from checkForReverseEntry when the reply arg is a wildcard subject. 1747 // This will usually be called in a go routine since we need to walk all the entries. 1748 func (a *Account) checkForReverseEntries(reply string, checkInterest, recursed bool) { 1749 if subjectIsLiteral(reply) { 1750 a._checkForReverseEntry(reply, nil, checkInterest, recursed) 1751 return 1752 } 1753 1754 a.mu.RLock() 1755 if len(a.imports.rrMap) == 0 { 1756 a.mu.RUnlock() 1757 return 1758 } 1759 1760 var _rs [64]string 1761 rs := _rs[:0] 1762 if n := len(a.imports.rrMap); n > cap(rs) { 1763 rs = make([]string, 0, n) 1764 } 1765 1766 for k := range a.imports.rrMap { 1767 rs = append(rs, k) 1768 } 1769 a.mu.RUnlock() 1770 1771 tsa := [32]string{} 1772 tts := tokenizeSubjectIntoSlice(tsa[:0], reply) 1773 1774 rsa := [32]string{} 1775 for _, r := range rs { 1776 rts := tokenizeSubjectIntoSlice(rsa[:0], r) 1777 // isSubsetMatchTokenized is heavy so make sure we do this without the lock. 1778 if isSubsetMatchTokenized(rts, tts) { 1779 a._checkForReverseEntry(r, nil, checkInterest, recursed) 1780 } 1781 } 1782 } 1783 1784 // This checks for any response map entries. If you specify an si we will only match and 1785 // clean up for that one, otherwise we remove them all. 1786 func (a *Account) checkForReverseEntry(reply string, si *serviceImport, checkInterest bool) { 1787 a._checkForReverseEntry(reply, si, checkInterest, false) 1788 } 1789 1790 // Callers should use checkForReverseEntry instead. This function exists to help prevent 1791 // infinite recursion. 1792 func (a *Account) _checkForReverseEntry(reply string, si *serviceImport, checkInterest, recursed bool) { 1793 a.mu.RLock() 1794 if len(a.imports.rrMap) == 0 { 1795 a.mu.RUnlock() 1796 return 1797 } 1798 1799 if subjectHasWildcard(reply) { 1800 if recursed { 1801 // If we have reached this condition then it is because the reverse entries also 1802 // contain wildcards (that shouldn't happen but a client *could* provide an inbox 1803 // prefix that is illegal because it ends in a wildcard character), at which point 1804 // we will end up with infinite recursion between this func and checkForReverseEntries. 1805 // To avoid a stack overflow panic, we'll give up instead. 1806 a.mu.RUnlock() 1807 return 1808 } 1809 1810 doInline := len(a.imports.rrMap) <= 64 1811 a.mu.RUnlock() 1812 1813 if doInline { 1814 a.checkForReverseEntries(reply, checkInterest, true) 1815 } else { 1816 go a.checkForReverseEntries(reply, checkInterest, true) 1817 } 1818 return 1819 } 1820 1821 if sres := a.imports.rrMap[reply]; sres == nil { 1822 a.mu.RUnlock() 1823 return 1824 } 1825 1826 // If we are here we have an entry we should check. 1827 // If requested we will first check if there is any 1828 // interest for this subject for the entire account. 1829 // If there is we can not delete any entries yet. 1830 // Note that if we are here reply has to be a literal subject. 1831 if checkInterest { 1832 // If interest still exists we can not clean these up yet. 1833 if rr := a.sl.Match(reply); len(rr.psubs)+len(rr.qsubs) > 0 { 1834 a.mu.RUnlock() 1835 return 1836 } 1837 } 1838 a.mu.RUnlock() 1839 1840 // Delete the appropriate entries here based on optional si. 1841 a.mu.Lock() 1842 // We need a new lookup here because we have released the lock. 1843 sres := a.imports.rrMap[reply] 1844 if si == nil { 1845 delete(a.imports.rrMap, reply) 1846 } else if sres != nil { 1847 // Find the one we are looking for.. 1848 for i, sre := range sres { 1849 if sre.msub == si.from { 1850 sres = append(sres[:i], sres[i+1:]...) 1851 break 1852 } 1853 } 1854 if len(sres) > 0 { 1855 a.imports.rrMap[si.to] = sres 1856 } else { 1857 delete(a.imports.rrMap, si.to) 1858 } 1859 } 1860 a.mu.Unlock() 1861 1862 // If we are here we no longer have interest and we have 1863 // response entries that we should clean up. 1864 if si == nil { 1865 // sres is now known to have been removed from a.imports.rrMap, so we 1866 // can safely (data race wise) iterate through. 1867 for _, sre := range sres { 1868 acc := sre.acc 1869 var trackingCleanup bool 1870 var rsi *serviceImport 1871 acc.mu.Lock() 1872 c := acc.ic 1873 if rsi = acc.exports.responses[sre.msub]; rsi != nil && !rsi.didDeliver { 1874 delete(acc.exports.responses, rsi.from) 1875 trackingCleanup = rsi.tracking && rsi.rc != nil 1876 } 1877 acc.mu.Unlock() 1878 // If we are doing explicit subs for all responses (e.g. bound to leafnode) 1879 // we will have a non-empty sid here. 1880 if rsi != nil && len(rsi.sid) > 0 && c != nil { 1881 c.processUnsub(rsi.sid) 1882 } 1883 if trackingCleanup { 1884 acc.sendReplyInterestLostTrackLatency(rsi) 1885 } 1886 } 1887 } 1888 } 1889 1890 // Checks to see if a potential service import subject is already overshadowed. 1891 func (a *Account) serviceImportShadowed(from string) bool { 1892 a.mu.RLock() 1893 defer a.mu.RUnlock() 1894 if a.imports.services[from] != nil { 1895 return true 1896 } 1897 // We did not find a direct match, so check individually. 1898 for subj := range a.imports.services { 1899 if subjectIsSubsetMatch(from, subj) { 1900 return true 1901 } 1902 } 1903 return false 1904 } 1905 1906 // Internal check to see if a service import exists. 1907 func (a *Account) serviceImportExists(from string) bool { 1908 a.mu.RLock() 1909 dup := a.imports.services[from] 1910 a.mu.RUnlock() 1911 return dup != nil 1912 } 1913 1914 // Add a service import. 1915 // This does no checks and should only be called by the msg processing code. 1916 // Use AddServiceImport from above if responding to user input or config changes, etc. 1917 func (a *Account) addServiceImport(dest *Account, from, to string, claim *jwt.Import) (*serviceImport, error) { 1918 rt := Singleton 1919 var lat *serviceLatency 1920 1921 if dest == nil { 1922 return nil, ErrMissingAccount 1923 } 1924 1925 var atrc bool 1926 dest.mu.RLock() 1927 se := dest.getServiceExport(to) 1928 if se != nil { 1929 rt = se.respType 1930 lat = se.latency 1931 atrc = se.atrc 1932 } 1933 dest.mu.RUnlock() 1934 1935 a.mu.Lock() 1936 if a.imports.services == nil { 1937 a.imports.services = make(map[string]*serviceImport) 1938 } else if dup := a.imports.services[from]; dup != nil { 1939 a.mu.Unlock() 1940 return nil, fmt.Errorf("duplicate service import subject %q, previously used in import for account %q, subject %q", 1941 from, dup.acc.Name, dup.to) 1942 } 1943 1944 if to == _EMPTY_ { 1945 to = from 1946 } 1947 // Check to see if we have a wildcard 1948 var ( 1949 usePub bool 1950 tr *subjectTransform 1951 err error 1952 ) 1953 if subjectHasWildcard(to) { 1954 // If to and from match, then we use the published subject. 1955 if to == from { 1956 usePub = true 1957 } else { 1958 to, _ = transformUntokenize(to) 1959 // Create a transform. Do so in reverse such that $ symbols only exist in to 1960 if tr, err = NewSubjectTransformStrict(to, transformTokenize(from)); err != nil { 1961 a.mu.Unlock() 1962 return nil, fmt.Errorf("failed to create mapping transform for service import subject from %q to %q: %v", 1963 from, to, err) 1964 } else { 1965 // un-tokenize and reverse transform so we get the transform needed 1966 from, _ = transformUntokenize(from) 1967 tr = tr.reverse() 1968 } 1969 } 1970 } 1971 var share bool 1972 if claim != nil { 1973 share = claim.Share 1974 } 1975 si := &serviceImport{dest, claim, se, nil, from, to, tr, 0, rt, lat, nil, nil, usePub, false, false, share, false, false, atrc, nil} 1976 a.imports.services[from] = si 1977 a.mu.Unlock() 1978 1979 if err := a.addServiceImportSub(si); err != nil { 1980 a.removeServiceImport(si.from) 1981 return nil, err 1982 } 1983 return si, nil 1984 } 1985 1986 // Returns the internal client, will create one if not present. 1987 // Lock should be held. 1988 func (a *Account) internalClient() *client { 1989 if a.ic == nil && a.srv != nil { 1990 a.ic = a.srv.createInternalAccountClient() 1991 a.ic.acc = a 1992 } 1993 return a.ic 1994 } 1995 1996 // Internal account scoped subscriptions. 1997 func (a *Account) subscribeInternal(subject string, cb msgHandler) (*subscription, error) { 1998 return a.subscribeInternalEx(subject, cb, false) 1999 } 2000 2001 // Unsubscribe from an internal account subscription. 2002 func (a *Account) unsubscribeInternal(sub *subscription) { 2003 if ic := a.internalClient(); ic != nil { 2004 ic.processUnsub(sub.sid) 2005 } 2006 } 2007 2008 // Creates internal subscription for service import responses. 2009 func (a *Account) subscribeServiceImportResponse(subject string) (*subscription, error) { 2010 return a.subscribeInternalEx(subject, a.processServiceImportResponse, true) 2011 } 2012 2013 func (a *Account) subscribeInternalEx(subject string, cb msgHandler, ri bool) (*subscription, error) { 2014 a.mu.Lock() 2015 a.isid++ 2016 c, sid := a.internalClient(), strconv.FormatUint(a.isid, 10) 2017 a.mu.Unlock() 2018 2019 // This will happen in parsing when the account has not been properly setup. 2020 if c == nil { 2021 return nil, fmt.Errorf("no internal account client") 2022 } 2023 2024 return c.processSubEx([]byte(subject), nil, []byte(sid), cb, false, false, ri) 2025 } 2026 2027 // This will add an account subscription that matches the "from" from a service import entry. 2028 func (a *Account) addServiceImportSub(si *serviceImport) error { 2029 a.mu.Lock() 2030 c := a.internalClient() 2031 // This will happen in parsing when the account has not been properly setup. 2032 if c == nil { 2033 a.mu.Unlock() 2034 return nil 2035 } 2036 if si.sid != nil { 2037 a.mu.Unlock() 2038 return fmt.Errorf("duplicate call to create subscription for service import") 2039 } 2040 a.isid++ 2041 sid := strconv.FormatUint(a.isid, 10) 2042 si.sid = []byte(sid) 2043 subject := si.from 2044 a.mu.Unlock() 2045 2046 cb := func(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) { 2047 c.processServiceImport(si, acc, msg) 2048 } 2049 sub, err := c.processSubEx([]byte(subject), nil, []byte(sid), cb, true, true, false) 2050 if err != nil { 2051 return err 2052 } 2053 // Leafnodes introduce a new way to introduce messages into the system. Therefore forward import subscription 2054 // This is similar to what initLeafNodeSmapAndSendSubs does 2055 // TODO we need to consider performing this update as we get client subscriptions. 2056 // This behavior would result in subscription propagation only where actually used. 2057 a.updateLeafNodes(sub, 1) 2058 return nil 2059 } 2060 2061 // Remove all the subscriptions associated with service imports. 2062 func (a *Account) removeAllServiceImportSubs() { 2063 a.mu.RLock() 2064 var sids [][]byte 2065 for _, si := range a.imports.services { 2066 if si.sid != nil { 2067 sids = append(sids, si.sid) 2068 si.sid = nil 2069 } 2070 } 2071 c := a.ic 2072 a.ic = nil 2073 a.mu.RUnlock() 2074 2075 if c == nil { 2076 return 2077 } 2078 for _, sid := range sids { 2079 c.processUnsub(sid) 2080 } 2081 c.closeConnection(InternalClient) 2082 } 2083 2084 // Add in subscriptions for all registered service imports. 2085 func (a *Account) addAllServiceImportSubs() { 2086 var sis [32]*serviceImport 2087 serviceImports := sis[:0] 2088 a.mu.RLock() 2089 for _, si := range a.imports.services { 2090 serviceImports = append(serviceImports, si) 2091 } 2092 a.mu.RUnlock() 2093 for _, si := range serviceImports { 2094 a.addServiceImportSub(si) 2095 } 2096 } 2097 2098 var ( 2099 // header where all information is encoded in one value. 2100 trcUber = textproto.CanonicalMIMEHeaderKey("Uber-Trace-Id") 2101 trcCtx = textproto.CanonicalMIMEHeaderKey("Traceparent") 2102 trcB3 = textproto.CanonicalMIMEHeaderKey("B3") 2103 // openzipkin header to check 2104 trcB3Sm = textproto.CanonicalMIMEHeaderKey("X-B3-Sampled") 2105 trcB3Id = textproto.CanonicalMIMEHeaderKey("X-B3-TraceId") 2106 // additional header needed to include when present 2107 trcB3PSId = textproto.CanonicalMIMEHeaderKey("X-B3-ParentSpanId") 2108 trcB3SId = textproto.CanonicalMIMEHeaderKey("X-B3-SpanId") 2109 trcCtxSt = textproto.CanonicalMIMEHeaderKey("Tracestate") 2110 trcUberCtxPrefix = textproto.CanonicalMIMEHeaderKey("Uberctx-") 2111 ) 2112 2113 func newB3Header(h http.Header) http.Header { 2114 retHdr := http.Header{} 2115 if v, ok := h[trcB3Sm]; ok { 2116 retHdr[trcB3Sm] = v 2117 } 2118 if v, ok := h[trcB3Id]; ok { 2119 retHdr[trcB3Id] = v 2120 } 2121 if v, ok := h[trcB3PSId]; ok { 2122 retHdr[trcB3PSId] = v 2123 } 2124 if v, ok := h[trcB3SId]; ok { 2125 retHdr[trcB3SId] = v 2126 } 2127 return retHdr 2128 } 2129 2130 func newUberHeader(h http.Header, tId []string) http.Header { 2131 retHdr := http.Header{trcUber: tId} 2132 for k, v := range h { 2133 if strings.HasPrefix(k, trcUberCtxPrefix) { 2134 retHdr[k] = v 2135 } 2136 } 2137 return retHdr 2138 } 2139 2140 func newTraceCtxHeader(h http.Header, tId []string) http.Header { 2141 retHdr := http.Header{trcCtx: tId} 2142 if v, ok := h[trcCtxSt]; ok { 2143 retHdr[trcCtxSt] = v 2144 } 2145 return retHdr 2146 } 2147 2148 // Helper to determine when to sample. When header has a value, sampling is driven by header 2149 func shouldSample(l *serviceLatency, c *client) (bool, http.Header) { 2150 if l == nil { 2151 return false, nil 2152 } 2153 if l.sampling < 0 { 2154 return false, nil 2155 } 2156 if l.sampling >= 100 { 2157 return true, nil 2158 } 2159 if l.sampling > 0 && rand.Int31n(100) <= int32(l.sampling) { 2160 return true, nil 2161 } 2162 h := c.parseState.getHeader() 2163 if len(h) == 0 { 2164 return false, nil 2165 } 2166 if tId := h[trcUber]; len(tId) != 0 { 2167 // sample 479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1 2168 tk := strings.Split(tId[0], ":") 2169 if len(tk) == 4 && len(tk[3]) > 0 && len(tk[3]) <= 2 { 2170 dst := [2]byte{} 2171 src := [2]byte{'0', tk[3][0]} 2172 if len(tk[3]) == 2 { 2173 src[1] = tk[3][1] 2174 } 2175 if _, err := hex.Decode(dst[:], src[:]); err == nil && dst[0]&1 == 1 { 2176 return true, newUberHeader(h, tId) 2177 } 2178 } 2179 return false, nil 2180 } else if sampled := h[trcB3Sm]; len(sampled) != 0 && sampled[0] == "1" { 2181 return true, newB3Header(h) // allowed 2182 } else if len(sampled) != 0 && sampled[0] == "0" { 2183 return false, nil // denied 2184 } else if _, ok := h[trcB3Id]; ok { 2185 // sample 80f198ee56343ba864fe8b2a57d3eff7 2186 // presence (with X-B3-Sampled not being 0) means sampling left to recipient 2187 return true, newB3Header(h) 2188 } else if b3 := h[trcB3]; len(b3) != 0 { 2189 // sample 80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90 2190 // sample 0 2191 tk := strings.Split(b3[0], "-") 2192 if len(tk) > 2 && tk[2] == "0" { 2193 return false, nil // denied 2194 } else if len(tk) == 1 && tk[0] == "0" { 2195 return false, nil // denied 2196 } 2197 return true, http.Header{trcB3: b3} // sampling allowed or left to recipient of header 2198 } else if tId := h[trcCtx]; len(tId) != 0 { 2199 var sample bool 2200 // sample 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01 2201 tk := strings.Split(tId[0], "-") 2202 if len(tk) == 4 && len([]byte(tk[3])) == 2 { 2203 if hexVal, err := strconv.ParseInt(tk[3], 16, 8); err == nil { 2204 sample = hexVal&0x1 == 0x1 2205 } 2206 } 2207 if sample { 2208 return true, newTraceCtxHeader(h, tId) 2209 } else { 2210 return false, nil 2211 } 2212 } 2213 return false, nil 2214 } 2215 2216 // Used to mimic client like replies. 2217 const ( 2218 replyPrefix = "_R_." 2219 replyPrefixLen = len(replyPrefix) 2220 baseServerLen = 10 2221 replyLen = 6 2222 minReplyLen = 15 2223 digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 2224 base = 62 2225 ) 2226 2227 // This is where all service export responses are handled. 2228 func (a *Account) processServiceImportResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { 2229 a.mu.RLock() 2230 if a.expired || len(a.exports.responses) == 0 { 2231 a.mu.RUnlock() 2232 return 2233 } 2234 si := a.exports.responses[subject] 2235 2236 if si == nil || si.invalid { 2237 a.mu.RUnlock() 2238 return 2239 } 2240 a.mu.RUnlock() 2241 2242 // Send for normal processing. 2243 c.processServiceImport(si, a, msg) 2244 } 2245 2246 // Will create the response prefix for fast generation of responses. 2247 // A wildcard subscription may be used handle interest graph propagation 2248 // for all service replies, unless we are bound to a leafnode. 2249 // Lock should be held. 2250 func (a *Account) createRespWildcard() { 2251 var b = [baseServerLen]byte{'_', 'R', '_', '.'} 2252 rn := fastrand.Uint64() 2253 for i, l := replyPrefixLen, rn; i < len(b); i++ { 2254 b[i] = digits[l%base] 2255 l /= base 2256 } 2257 a.siReply = append(b[:], '.') 2258 } 2259 2260 // Test whether this is a tracked reply. 2261 func isTrackedReply(reply []byte) bool { 2262 lreply := len(reply) - 1 2263 return lreply > 3 && reply[lreply-1] == '.' && reply[lreply] == 'T' 2264 } 2265 2266 // Generate a new service reply from the wildcard prefix. 2267 // FIXME(dlc) - probably do not have to use rand here. about 25ns per. 2268 func (a *Account) newServiceReply(tracking bool) []byte { 2269 a.mu.Lock() 2270 s := a.srv 2271 rn := fastrand.Uint64() 2272 2273 // Check if we need to create the reply here. 2274 var createdSiReply bool 2275 if a.siReply == nil { 2276 a.createRespWildcard() 2277 createdSiReply = true 2278 } 2279 replyPre := a.siReply 2280 a.mu.Unlock() 2281 2282 // If we created the siReply and we are not bound to a leafnode 2283 // we need to do the wildcard subscription. 2284 if createdSiReply { 2285 a.subscribeServiceImportResponse(string(append(replyPre, '>'))) 2286 } 2287 2288 var b [replyLen]byte 2289 for i, l := 0, rn; i < len(b); i++ { 2290 b[i] = digits[l%base] 2291 l /= base 2292 } 2293 // Make sure to copy. 2294 reply := make([]byte, 0, len(replyPre)+len(b)) 2295 reply = append(reply, replyPre...) 2296 reply = append(reply, b[:]...) 2297 2298 if tracking && s.sys != nil { 2299 // Add in our tracking identifier. This allows the metrics to get back to only 2300 // this server without needless SUBS/UNSUBS. 2301 reply = append(reply, '.') 2302 reply = append(reply, s.sys.shash...) 2303 reply = append(reply, '.', 'T') 2304 } 2305 2306 return reply 2307 } 2308 2309 // Checks if a serviceImport was created to map responses. 2310 func (si *serviceImport) isRespServiceImport() bool { 2311 return si != nil && si.response 2312 } 2313 2314 // Sets the response threshold timer for a service export. 2315 // Account lock should be held 2316 func (se *serviceExport) setResponseThresholdTimer() { 2317 if se.rtmr != nil { 2318 return // Already set 2319 } 2320 se.rtmr = time.AfterFunc(se.respThresh, se.checkExpiredResponses) 2321 } 2322 2323 // Account lock should be held 2324 func (se *serviceExport) clearResponseThresholdTimer() bool { 2325 if se.rtmr == nil { 2326 return true 2327 } 2328 stopped := se.rtmr.Stop() 2329 se.rtmr = nil 2330 return stopped 2331 } 2332 2333 // checkExpiredResponses will check for any pending responses that need to 2334 // be cleaned up. 2335 func (se *serviceExport) checkExpiredResponses() { 2336 acc := se.acc 2337 if acc == nil { 2338 se.clearResponseThresholdTimer() 2339 return 2340 } 2341 2342 var expired []*serviceImport 2343 mints := time.Now().UnixNano() - int64(se.respThresh) 2344 2345 // TODO(dlc) - Should we release lock while doing this? Or only do these in batches? 2346 // Should we break this up for responses only from this service export? 2347 // Responses live on acc directly for fast inbound processsing for the _R_ wildcard. 2348 // We could do another indirection at this level but just to get to the service export? 2349 var totalResponses int 2350 acc.mu.RLock() 2351 for _, si := range acc.exports.responses { 2352 if si.se == se { 2353 totalResponses++ 2354 if si.ts <= mints { 2355 expired = append(expired, si) 2356 } 2357 } 2358 } 2359 acc.mu.RUnlock() 2360 2361 for _, si := range expired { 2362 acc.removeRespServiceImport(si, rsiTimeout) 2363 } 2364 2365 // Pull out expired to determine if we have any left for timer. 2366 totalResponses -= len(expired) 2367 2368 // Redo timer as needed. 2369 acc.mu.Lock() 2370 if totalResponses > 0 && se.rtmr != nil { 2371 se.rtmr.Stop() 2372 se.rtmr.Reset(se.respThresh) 2373 } else { 2374 se.clearResponseThresholdTimer() 2375 } 2376 acc.mu.Unlock() 2377 } 2378 2379 // ServiceExportResponseThreshold returns the current threshold. 2380 func (a *Account) ServiceExportResponseThreshold(export string) (time.Duration, error) { 2381 a.mu.Lock() 2382 defer a.mu.Unlock() 2383 se := a.getServiceExport(export) 2384 if se == nil { 2385 return 0, fmt.Errorf("no export defined for %q", export) 2386 } 2387 return se.respThresh, nil 2388 } 2389 2390 // SetServiceExportResponseThreshold sets the maximum time the system will a response to be delivered 2391 // from a service export responder. 2392 func (a *Account) SetServiceExportResponseThreshold(export string, maxTime time.Duration) error { 2393 a.mu.Lock() 2394 if a.isClaimAccount() { 2395 a.mu.Unlock() 2396 return fmt.Errorf("claim based accounts can not be updated directly") 2397 } 2398 lrt := a.lowestServiceExportResponseTime() 2399 se := a.getServiceExport(export) 2400 if se == nil { 2401 a.mu.Unlock() 2402 return fmt.Errorf("no export defined for %q", export) 2403 } 2404 se.respThresh = maxTime 2405 2406 var clients []*client 2407 nlrt := a.lowestServiceExportResponseTime() 2408 if nlrt != lrt && len(a.clients) > 0 { 2409 clients = a.getClientsLocked() 2410 } 2411 // Need to release because lock ordering is client -> Account 2412 a.mu.Unlock() 2413 if len(clients) > 0 { 2414 updateAllClientsServiceExportResponseTime(clients, nlrt) 2415 } 2416 return nil 2417 } 2418 2419 func (a *Account) SetServiceExportAllowTrace(export string, allowTrace bool) error { 2420 a.mu.Lock() 2421 se := a.getServiceExport(export) 2422 if se == nil { 2423 a.mu.Unlock() 2424 return fmt.Errorf("no export defined for %q", export) 2425 } 2426 se.atrc = allowTrace 2427 a.mu.Unlock() 2428 return nil 2429 } 2430 2431 // This is for internal service import responses. 2432 func (a *Account) addRespServiceImport(dest *Account, to string, osi *serviceImport, tracking bool, header http.Header) *serviceImport { 2433 nrr := string(osi.acc.newServiceReply(tracking)) 2434 2435 a.mu.Lock() 2436 rt := osi.rt 2437 2438 // dest is the requestor's account. a is the service responder with the export. 2439 // Marked as internal here, that is how we distinguish. 2440 si := &serviceImport{dest, nil, osi.se, nil, nrr, to, nil, 0, rt, nil, nil, nil, false, true, false, osi.share, false, false, false, nil} 2441 2442 if a.exports.responses == nil { 2443 a.exports.responses = make(map[string]*serviceImport) 2444 } 2445 a.exports.responses[nrr] = si 2446 2447 // Always grab time and make sure response threshold timer is running. 2448 si.ts = time.Now().UnixNano() 2449 if osi.se != nil { 2450 osi.se.setResponseThresholdTimer() 2451 } 2452 2453 if rt == Singleton && tracking { 2454 si.latency = osi.latency 2455 si.tracking = true 2456 si.trackingHdr = header 2457 } 2458 a.mu.Unlock() 2459 2460 // We do add in the reverse map such that we can detect loss of interest and do proper 2461 // cleanup of this si as interest goes away. 2462 dest.addReverseRespMapEntry(a, to, nrr) 2463 2464 return si 2465 } 2466 2467 // AddStreamImportWithClaim will add in the stream import from a specific account with optional token. 2468 func (a *Account) AddStreamImportWithClaim(account *Account, from, prefix string, imClaim *jwt.Import) error { 2469 return a.addStreamImportWithClaim(account, from, prefix, false, imClaim) 2470 } 2471 2472 func (a *Account) addStreamImportWithClaim(account *Account, from, prefix string, allowTrace bool, imClaim *jwt.Import) error { 2473 if account == nil { 2474 return ErrMissingAccount 2475 } 2476 2477 // First check to see if the account has authorized export of the subject. 2478 if !account.checkStreamImportAuthorized(a, from, imClaim) { 2479 return ErrStreamImportAuthorization 2480 } 2481 2482 // Check prefix if it exists and make sure its a literal. 2483 // Append token separator if not already present. 2484 if prefix != _EMPTY_ { 2485 // Make sure there are no wildcards here, this prefix needs to be a literal 2486 // since it will be prepended to a publish subject. 2487 if !subjectIsLiteral(prefix) { 2488 return ErrStreamImportBadPrefix 2489 } 2490 if prefix[len(prefix)-1] != btsep { 2491 prefix = prefix + string(btsep) 2492 } 2493 } 2494 2495 return a.addMappedStreamImportWithClaim(account, from, prefix+from, allowTrace, imClaim) 2496 } 2497 2498 // AddMappedStreamImport helper for AddMappedStreamImportWithClaim 2499 func (a *Account) AddMappedStreamImport(account *Account, from, to string) error { 2500 return a.AddMappedStreamImportWithClaim(account, from, to, nil) 2501 } 2502 2503 // AddMappedStreamImportWithClaim will add in the stream import from a specific account with optional token. 2504 func (a *Account) AddMappedStreamImportWithClaim(account *Account, from, to string, imClaim *jwt.Import) error { 2505 return a.addMappedStreamImportWithClaim(account, from, to, false, imClaim) 2506 } 2507 2508 func (a *Account) addMappedStreamImportWithClaim(account *Account, from, to string, allowTrace bool, imClaim *jwt.Import) error { 2509 if account == nil { 2510 return ErrMissingAccount 2511 } 2512 2513 // First check to see if the account has authorized export of the subject. 2514 if !account.checkStreamImportAuthorized(a, from, imClaim) { 2515 return ErrStreamImportAuthorization 2516 } 2517 2518 if to == _EMPTY_ { 2519 to = from 2520 } 2521 2522 // Check if this forms a cycle. 2523 if err := a.streamImportFormsCycle(account, to); err != nil { 2524 return err 2525 } 2526 2527 var ( 2528 usePub bool 2529 tr *subjectTransform 2530 err error 2531 ) 2532 if subjectHasWildcard(from) { 2533 if to == from { 2534 usePub = true 2535 } else { 2536 // Create a transform 2537 if tr, err = NewSubjectTransformStrict(from, transformTokenize(to)); err != nil { 2538 return fmt.Errorf("failed to create mapping transform for stream import subject from %q to %q: %v", 2539 from, to, err) 2540 } 2541 to, _ = transformUntokenize(to) 2542 } 2543 } 2544 2545 a.mu.Lock() 2546 if a.isStreamImportDuplicate(account, from) { 2547 a.mu.Unlock() 2548 return ErrStreamImportDuplicate 2549 } 2550 if imClaim != nil { 2551 allowTrace = imClaim.AllowTrace 2552 } 2553 a.imports.streams = append(a.imports.streams, &streamImport{account, from, to, tr, nil, imClaim, usePub, false, allowTrace}) 2554 a.mu.Unlock() 2555 return nil 2556 } 2557 2558 // isStreamImportDuplicate checks for duplicate. 2559 // Lock should be held. 2560 func (a *Account) isStreamImportDuplicate(acc *Account, from string) bool { 2561 for _, si := range a.imports.streams { 2562 if si.acc == acc && si.from == from { 2563 return true 2564 } 2565 } 2566 return false 2567 } 2568 2569 // AddStreamImport will add in the stream import from a specific account. 2570 func (a *Account) AddStreamImport(account *Account, from, prefix string) error { 2571 return a.addStreamImportWithClaim(account, from, prefix, false, nil) 2572 } 2573 2574 // IsPublicExport is a placeholder to denote a public export. 2575 var IsPublicExport = []*Account(nil) 2576 2577 // AddStreamExport will add an export to the account. If accounts is nil 2578 // it will signify a public export, meaning anyone can import. 2579 func (a *Account) AddStreamExport(subject string, accounts []*Account) error { 2580 return a.addStreamExportWithAccountPos(subject, accounts, 0) 2581 } 2582 2583 // AddStreamExport will add an export to the account. If accounts is nil 2584 // it will signify a public export, meaning anyone can import. 2585 // if accountPos is > 0, all imports will be granted where the following holds: 2586 // strings.Split(subject, tsep)[accountPos] == account id will be granted. 2587 func (a *Account) addStreamExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error { 2588 if a == nil { 2589 return ErrMissingAccount 2590 } 2591 2592 a.mu.Lock() 2593 defer a.mu.Unlock() 2594 2595 if a.exports.streams == nil { 2596 a.exports.streams = make(map[string]*streamExport) 2597 } 2598 ea := a.exports.streams[subject] 2599 if accounts != nil || accountPos > 0 { 2600 if ea == nil { 2601 ea = &streamExport{} 2602 } 2603 if err := setExportAuth(&ea.exportAuth, subject, accounts, accountPos); err != nil { 2604 return err 2605 } 2606 } 2607 a.exports.streams[subject] = ea 2608 return nil 2609 } 2610 2611 // Check if another account is authorized to import from us. 2612 func (a *Account) checkStreamImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool { 2613 // Find the subject in the exports list. 2614 a.mu.RLock() 2615 auth := a.checkStreamImportAuthorizedNoLock(account, subject, imClaim) 2616 a.mu.RUnlock() 2617 return auth 2618 } 2619 2620 func (a *Account) checkStreamImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool { 2621 if a.exports.streams == nil || !IsValidSubject(subject) { 2622 return false 2623 } 2624 return a.checkStreamExportApproved(account, subject, imClaim) 2625 } 2626 2627 func (a *Account) checkAuth(ea *exportAuth, account *Account, imClaim *jwt.Import, tokens []string) bool { 2628 // if ea is nil or ea.approved is nil, that denotes a public export 2629 if ea == nil || (len(ea.approved) == 0 && !ea.tokenReq && ea.accountPos == 0) { 2630 return true 2631 } 2632 // Check if the export is protected and enforces presence of importing account identity 2633 if ea.accountPos > 0 { 2634 return ea.accountPos <= uint(len(tokens)) && tokens[ea.accountPos-1] == account.Name 2635 } 2636 // Check if token required 2637 if ea.tokenReq { 2638 return a.checkActivation(account, imClaim, ea, true) 2639 } 2640 if ea.approved == nil { 2641 return false 2642 } 2643 // If we have a matching account we are authorized 2644 _, ok := ea.approved[account.Name] 2645 return ok 2646 } 2647 2648 func (a *Account) checkStreamExportApproved(account *Account, subject string, imClaim *jwt.Import) bool { 2649 // Check direct match of subject first 2650 ea, ok := a.exports.streams[subject] 2651 if ok { 2652 // if ea is nil or eq.approved is nil, that denotes a public export 2653 if ea == nil { 2654 return true 2655 } 2656 return a.checkAuth(&ea.exportAuth, account, imClaim, nil) 2657 } 2658 2659 // ok if we are here we did not match directly so we need to test each one. 2660 // The import subject arg has to take precedence, meaning the export 2661 // has to be a true subset of the import claim. We already checked for 2662 // exact matches above. 2663 tokens := strings.Split(subject, tsep) 2664 for subj, ea := range a.exports.streams { 2665 if isSubsetMatch(tokens, subj) { 2666 if ea == nil { 2667 return true 2668 } 2669 return a.checkAuth(&ea.exportAuth, account, imClaim, tokens) 2670 } 2671 } 2672 return false 2673 } 2674 2675 func (a *Account) checkServiceExportApproved(account *Account, subject string, imClaim *jwt.Import) bool { 2676 // Check direct match of subject first 2677 se, ok := a.exports.services[subject] 2678 if ok { 2679 // if se is nil or eq.approved is nil, that denotes a public export 2680 if se == nil { 2681 return true 2682 } 2683 return a.checkAuth(&se.exportAuth, account, imClaim, nil) 2684 } 2685 // ok if we are here we did not match directly so we need to test each one. 2686 // The import subject arg has to take precedence, meaning the export 2687 // has to be a true subset of the import claim. We already checked for 2688 // exact matches above. 2689 tokens := strings.Split(subject, tsep) 2690 for subj, se := range a.exports.services { 2691 if isSubsetMatch(tokens, subj) { 2692 if se == nil { 2693 return true 2694 } 2695 return a.checkAuth(&se.exportAuth, account, imClaim, tokens) 2696 } 2697 } 2698 return false 2699 } 2700 2701 // Helper function to get a serviceExport. 2702 // Lock should be held on entry. 2703 func (a *Account) getServiceExport(subj string) *serviceExport { 2704 se, ok := a.exports.services[subj] 2705 // The export probably has a wildcard, so lookup that up. 2706 if !ok { 2707 se = a.getWildcardServiceExport(subj) 2708 } 2709 return se 2710 } 2711 2712 // This helper is used when trying to match a serviceExport record that is 2713 // represented by a wildcard. 2714 // Lock should be held on entry. 2715 func (a *Account) getWildcardServiceExport(from string) *serviceExport { 2716 tokens := strings.Split(from, tsep) 2717 for subj, se := range a.exports.services { 2718 if isSubsetMatch(tokens, subj) { 2719 return se 2720 } 2721 } 2722 return nil 2723 } 2724 2725 // These are import stream specific versions for when an activation expires. 2726 func (a *Account) streamActivationExpired(exportAcc *Account, subject string) { 2727 a.mu.RLock() 2728 if a.expired || a.imports.streams == nil { 2729 a.mu.RUnlock() 2730 return 2731 } 2732 var si *streamImport 2733 for _, si = range a.imports.streams { 2734 if si.acc == exportAcc && si.from == subject { 2735 break 2736 } 2737 } 2738 2739 if si == nil || si.invalid { 2740 a.mu.RUnlock() 2741 return 2742 } 2743 a.mu.RUnlock() 2744 2745 if si.acc.checkActivation(a, si.claim, nil, false) { 2746 // The token has been updated most likely and we are good to go. 2747 return 2748 } 2749 2750 a.mu.Lock() 2751 si.invalid = true 2752 clients := a.getClientsLocked() 2753 awcsti := map[string]struct{}{a.Name: {}} 2754 a.mu.Unlock() 2755 for _, c := range clients { 2756 c.processSubsOnConfigReload(awcsti) 2757 } 2758 } 2759 2760 // These are import service specific versions for when an activation expires. 2761 func (a *Account) serviceActivationExpired(subject string) { 2762 a.mu.RLock() 2763 if a.expired || a.imports.services == nil { 2764 a.mu.RUnlock() 2765 return 2766 } 2767 si := a.imports.services[subject] 2768 if si == nil || si.invalid { 2769 a.mu.RUnlock() 2770 return 2771 } 2772 a.mu.RUnlock() 2773 2774 if si.acc.checkActivation(a, si.claim, nil, false) { 2775 // The token has been updated most likely and we are good to go. 2776 return 2777 } 2778 2779 a.mu.Lock() 2780 si.invalid = true 2781 a.mu.Unlock() 2782 } 2783 2784 // Fires for expired activation tokens. We could track this with timers etc. 2785 // Instead we just re-analyze where we are and if we need to act. 2786 func (a *Account) activationExpired(exportAcc *Account, subject string, kind jwt.ExportType) { 2787 switch kind { 2788 case jwt.Stream: 2789 a.streamActivationExpired(exportAcc, subject) 2790 case jwt.Service: 2791 a.serviceActivationExpired(subject) 2792 } 2793 } 2794 2795 func isRevoked(revocations map[string]int64, subject string, issuedAt int64) bool { 2796 if len(revocations) == 0 { 2797 return false 2798 } 2799 if t, ok := revocations[subject]; !ok || t < issuedAt { 2800 if t, ok := revocations[jwt.All]; !ok || t < issuedAt { 2801 return false 2802 } 2803 } 2804 return true 2805 } 2806 2807 // checkActivation will check the activation token for validity. 2808 // ea may only be nil in cases where revocation may not be checked, say triggered by expiration timer. 2809 func (a *Account) checkActivation(importAcc *Account, claim *jwt.Import, ea *exportAuth, expTimer bool) bool { 2810 if claim == nil || claim.Token == _EMPTY_ { 2811 return false 2812 } 2813 // Create a quick clone so we can inline Token JWT. 2814 clone := *claim 2815 2816 vr := jwt.CreateValidationResults() 2817 clone.Validate(importAcc.Name, vr) 2818 if vr.IsBlocking(true) { 2819 return false 2820 } 2821 act, err := jwt.DecodeActivationClaims(clone.Token) 2822 if err != nil { 2823 return false 2824 } 2825 if !a.isIssuerClaimTrusted(act) { 2826 return false 2827 } 2828 vr = jwt.CreateValidationResults() 2829 act.Validate(vr) 2830 if vr.IsBlocking(true) { 2831 return false 2832 } 2833 if act.Expires != 0 { 2834 tn := time.Now().Unix() 2835 if act.Expires <= tn { 2836 return false 2837 } 2838 if expTimer { 2839 expiresAt := time.Duration(act.Expires - tn) 2840 time.AfterFunc(expiresAt*time.Second, func() { 2841 importAcc.activationExpired(a, string(act.ImportSubject), claim.Type) 2842 }) 2843 } 2844 } 2845 if ea == nil { 2846 return true 2847 } 2848 // Check for token revocation.. 2849 return !isRevoked(ea.actsRevoked, act.Subject, act.IssuedAt) 2850 } 2851 2852 // Returns true if the activation claim is trusted. That is the issuer matches 2853 // the account or is an entry in the signing keys. 2854 func (a *Account) isIssuerClaimTrusted(claims *jwt.ActivationClaims) bool { 2855 // if no issuer account, issuer is the account 2856 if claims.IssuerAccount == _EMPTY_ { 2857 return true 2858 } 2859 // If the IssuerAccount is not us, then this is considered an error. 2860 if a.Name != claims.IssuerAccount { 2861 if a.srv != nil { 2862 a.srv.Errorf("Invalid issuer account %q in activation claim (subject: %q - type: %q) for account %q", 2863 claims.IssuerAccount, claims.Activation.ImportSubject, claims.Activation.ImportType, a.Name) 2864 } 2865 return false 2866 } 2867 _, ok := a.hasIssuerNoLock(claims.Issuer) 2868 return ok 2869 } 2870 2871 // Returns true if `a` and `b` stream imports are the same. Note that the 2872 // check is done with the account's name, not the pointer. This is used 2873 // during config reload where we are comparing current and new config 2874 // in which pointers are different. 2875 // No lock is acquired in this function, so it is assumed that the 2876 // import maps are not changed while this executes. 2877 func (a *Account) checkStreamImportsEqual(b *Account) bool { 2878 if len(a.imports.streams) != len(b.imports.streams) { 2879 return false 2880 } 2881 // Load the b imports into a map index by what we are looking for. 2882 bm := make(map[string]*streamImport, len(b.imports.streams)) 2883 for _, bim := range b.imports.streams { 2884 bm[bim.acc.Name+bim.from+bim.to] = bim 2885 } 2886 for _, aim := range a.imports.streams { 2887 if bim, ok := bm[aim.acc.Name+aim.from+aim.to]; !ok { 2888 return false 2889 } else if aim.atrc != bim.atrc { 2890 return false 2891 } 2892 } 2893 return true 2894 } 2895 2896 // Returns true if `a` and `b` stream exports are the same. 2897 // Acquires `a` read lock, but `b` is assumed to not be accessed 2898 // by anyone but the caller (`b` is not registered anywhere). 2899 func (a *Account) checkStreamExportsEqual(b *Account) bool { 2900 a.mu.RLock() 2901 defer a.mu.RUnlock() 2902 if len(a.exports.streams) != len(b.exports.streams) { 2903 return false 2904 } 2905 for subj, aea := range a.exports.streams { 2906 bea, ok := b.exports.streams[subj] 2907 if !ok { 2908 return false 2909 } 2910 if !isStreamExportEqual(aea, bea) { 2911 return false 2912 } 2913 } 2914 return true 2915 } 2916 2917 func isStreamExportEqual(a, b *streamExport) bool { 2918 if a == nil && b == nil { 2919 return true 2920 } 2921 if (a == nil && b != nil) || (a != nil && b == nil) { 2922 return false 2923 } 2924 return isExportAuthEqual(&a.exportAuth, &b.exportAuth) 2925 } 2926 2927 // Returns true if `a` and `b` service exports are the same. 2928 // Acquires `a` read lock, but `b` is assumed to not be accessed 2929 // by anyone but the caller (`b` is not registered anywhere). 2930 func (a *Account) checkServiceExportsEqual(b *Account) bool { 2931 a.mu.RLock() 2932 defer a.mu.RUnlock() 2933 if len(a.exports.services) != len(b.exports.services) { 2934 return false 2935 } 2936 for subj, aea := range a.exports.services { 2937 bea, ok := b.exports.services[subj] 2938 if !ok { 2939 return false 2940 } 2941 if !isServiceExportEqual(aea, bea) { 2942 return false 2943 } 2944 } 2945 return true 2946 } 2947 2948 func isServiceExportEqual(a, b *serviceExport) bool { 2949 if a == nil && b == nil { 2950 return true 2951 } 2952 if (a == nil && b != nil) || (a != nil && b == nil) { 2953 return false 2954 } 2955 if !isExportAuthEqual(&a.exportAuth, &b.exportAuth) { 2956 return false 2957 } 2958 if a.acc.Name != b.acc.Name { 2959 return false 2960 } 2961 if a.respType != b.respType { 2962 return false 2963 } 2964 if a.latency != nil || b.latency != nil { 2965 if (a.latency != nil && b.latency == nil) || (a.latency == nil && b.latency != nil) { 2966 return false 2967 } 2968 if a.latency.sampling != b.latency.sampling { 2969 return false 2970 } 2971 if a.latency.subject != b.latency.subject { 2972 return false 2973 } 2974 } 2975 if a.atrc != b.atrc { 2976 return false 2977 } 2978 return true 2979 } 2980 2981 // Returns true if `a` and `b` exportAuth structures are equal. 2982 // Both `a` and `b` are guaranteed to be non-nil. 2983 // Locking is handled by the caller. 2984 func isExportAuthEqual(a, b *exportAuth) bool { 2985 if a.tokenReq != b.tokenReq { 2986 return false 2987 } 2988 if a.accountPos != b.accountPos { 2989 return false 2990 } 2991 if len(a.approved) != len(b.approved) { 2992 return false 2993 } 2994 for ak, av := range a.approved { 2995 if bv, ok := b.approved[ak]; !ok || av.Name != bv.Name { 2996 return false 2997 } 2998 } 2999 if len(a.actsRevoked) != len(b.actsRevoked) { 3000 return false 3001 } 3002 for ak, av := range a.actsRevoked { 3003 if bv, ok := b.actsRevoked[ak]; !ok || av != bv { 3004 return false 3005 } 3006 } 3007 return true 3008 } 3009 3010 // Check if another account is authorized to route requests to this service. 3011 func (a *Account) checkServiceImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool { 3012 a.mu.RLock() 3013 authorized := a.checkServiceImportAuthorizedNoLock(account, subject, imClaim) 3014 a.mu.RUnlock() 3015 return authorized 3016 } 3017 3018 // Check if another account is authorized to route requests to this service. 3019 func (a *Account) checkServiceImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool { 3020 // Find the subject in the services list. 3021 if a.exports.services == nil { 3022 return false 3023 } 3024 return a.checkServiceExportApproved(account, subject, imClaim) 3025 } 3026 3027 // IsExpired returns expiration status. 3028 func (a *Account) IsExpired() bool { 3029 a.mu.RLock() 3030 exp := a.expired 3031 a.mu.RUnlock() 3032 return exp 3033 } 3034 3035 // Called when an account has expired. 3036 func (a *Account) expiredTimeout() { 3037 // Mark expired first. 3038 a.mu.Lock() 3039 a.expired = true 3040 a.mu.Unlock() 3041 3042 // Collect the clients and expire them. 3043 cs := a.getClients() 3044 for _, c := range cs { 3045 c.accountAuthExpired() 3046 } 3047 } 3048 3049 // Sets the expiration timer for an account JWT that has it set. 3050 func (a *Account) setExpirationTimer(d time.Duration) { 3051 a.etmr = time.AfterFunc(d, a.expiredTimeout) 3052 } 3053 3054 // Lock should be held 3055 func (a *Account) clearExpirationTimer() bool { 3056 if a.etmr == nil { 3057 return true 3058 } 3059 stopped := a.etmr.Stop() 3060 a.etmr = nil 3061 return stopped 3062 } 3063 3064 // checkUserRevoked will check if a user has been revoked. 3065 func (a *Account) checkUserRevoked(nkey string, issuedAt int64) bool { 3066 a.mu.RLock() 3067 defer a.mu.RUnlock() 3068 return isRevoked(a.usersRevoked, nkey, issuedAt) 3069 } 3070 3071 // failBearer will return if bearer token are allowed (false) or disallowed (true) 3072 func (a *Account) failBearer() bool { 3073 a.mu.RLock() 3074 defer a.mu.RUnlock() 3075 return a.disallowBearer 3076 } 3077 3078 // Check expiration and set the proper state as needed. 3079 func (a *Account) checkExpiration(claims *jwt.ClaimsData) { 3080 a.mu.Lock() 3081 defer a.mu.Unlock() 3082 3083 a.clearExpirationTimer() 3084 if claims.Expires == 0 { 3085 a.expired = false 3086 return 3087 } 3088 tn := time.Now().Unix() 3089 if claims.Expires <= tn { 3090 a.expired = true 3091 return 3092 } 3093 expiresAt := time.Duration(claims.Expires - tn) 3094 a.setExpirationTimer(expiresAt * time.Second) 3095 a.expired = false 3096 } 3097 3098 // hasIssuer returns true if the issuer matches the account 3099 // If the issuer is a scoped signing key, the scope will be returned as well 3100 // issuer or it is a signing key for the account. 3101 func (a *Account) hasIssuer(issuer string) (jwt.Scope, bool) { 3102 a.mu.RLock() 3103 scope, ok := a.hasIssuerNoLock(issuer) 3104 a.mu.RUnlock() 3105 return scope, ok 3106 } 3107 3108 // hasIssuerNoLock is the unlocked version of hasIssuer 3109 func (a *Account) hasIssuerNoLock(issuer string) (jwt.Scope, bool) { 3110 scope, ok := a.signingKeys[issuer] 3111 return scope, ok 3112 } 3113 3114 // Returns the loop detection subject used for leafnodes 3115 func (a *Account) getLDSubject() string { 3116 a.mu.RLock() 3117 lds := a.lds 3118 a.mu.RUnlock() 3119 return lds 3120 } 3121 3122 // Placeholder for signaling token auth required. 3123 var tokenAuthReq = []*Account{} 3124 3125 func authAccounts(tokenReq bool) []*Account { 3126 if tokenReq { 3127 return tokenAuthReq 3128 } 3129 return nil 3130 } 3131 3132 // SetAccountResolver will assign the account resolver. 3133 func (s *Server) SetAccountResolver(ar AccountResolver) { 3134 s.mu.Lock() 3135 s.accResolver = ar 3136 s.mu.Unlock() 3137 } 3138 3139 // AccountResolver returns the registered account resolver. 3140 func (s *Server) AccountResolver() AccountResolver { 3141 s.mu.Lock() 3142 ar := s.accResolver 3143 s.mu.Unlock() 3144 return ar 3145 } 3146 3147 // isClaimAccount returns if this account is backed by a JWT claim. 3148 // Lock should be held. 3149 func (a *Account) isClaimAccount() bool { 3150 return a.claimJWT != _EMPTY_ 3151 } 3152 3153 // UpdateAccountClaims will update an existing account with new claims. 3154 // This will replace any exports or imports previously defined. 3155 // Lock MUST NOT be held upon entry. 3156 func (s *Server) UpdateAccountClaims(a *Account, ac *jwt.AccountClaims) { 3157 s.updateAccountClaimsWithRefresh(a, ac, true) 3158 } 3159 3160 func (a *Account) traceLabel() string { 3161 if a == nil { 3162 return _EMPTY_ 3163 } 3164 if a.nameTag != _EMPTY_ { 3165 return fmt.Sprintf("%s/%s", a.Name, a.nameTag) 3166 } 3167 return a.Name 3168 } 3169 3170 // Check if an account has external auth set. 3171 // Operator/Account Resolver only. 3172 func (a *Account) hasExternalAuth() bool { 3173 if a == nil { 3174 return false 3175 } 3176 a.mu.RLock() 3177 defer a.mu.RUnlock() 3178 return a.extAuth != nil 3179 } 3180 3181 // Deterimine if this is an external auth user. 3182 func (a *Account) isExternalAuthUser(userID string) bool { 3183 if a == nil { 3184 return false 3185 } 3186 a.mu.RLock() 3187 defer a.mu.RUnlock() 3188 if a.extAuth != nil { 3189 for _, u := range a.extAuth.AuthUsers { 3190 if userID == u { 3191 return true 3192 } 3193 } 3194 } 3195 return false 3196 } 3197 3198 // Return the external authorization xkey if external authorization is enabled and the xkey is set. 3199 // Operator/Account Resolver only. 3200 func (a *Account) externalAuthXKey() string { 3201 if a == nil { 3202 return _EMPTY_ 3203 } 3204 a.mu.RLock() 3205 defer a.mu.RUnlock() 3206 if a.extAuth != nil && a.extAuth.XKey != _EMPTY_ { 3207 return a.extAuth.XKey 3208 } 3209 return _EMPTY_ 3210 } 3211 3212 // Check if an account switch for external authorization is allowed. 3213 func (a *Account) isAllowedAcount(acc string) bool { 3214 if a == nil { 3215 return false 3216 } 3217 a.mu.RLock() 3218 defer a.mu.RUnlock() 3219 if a.extAuth != nil { 3220 // if we have a single allowed account, and we have a wildcard 3221 // we accept it 3222 if len(a.extAuth.AllowedAccounts) == 1 && 3223 a.extAuth.AllowedAccounts[0] == jwt.AnyAccount { 3224 return true 3225 } 3226 // otherwise must match exactly 3227 for _, a := range a.extAuth.AllowedAccounts { 3228 if a == acc { 3229 return true 3230 } 3231 } 3232 } 3233 return false 3234 } 3235 3236 // updateAccountClaimsWithRefresh will update an existing account with new claims. 3237 // If refreshImportingAccounts is true it will also update incomplete dependent accounts 3238 // This will replace any exports or imports previously defined. 3239 // Lock MUST NOT be held upon entry. 3240 func (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaims, refreshImportingAccounts bool) { 3241 if a == nil { 3242 return 3243 } 3244 s.Debugf("Updating account claims: %s/%s", a.Name, ac.Name) 3245 a.checkExpiration(ac.Claims()) 3246 3247 a.mu.Lock() 3248 // Clone to update, only select certain fields. 3249 old := &Account{Name: a.Name, exports: a.exports, limits: a.limits, signingKeys: a.signingKeys} 3250 3251 // overwrite claim meta data 3252 a.nameTag = ac.Name 3253 a.tags = ac.Tags 3254 3255 var td string 3256 var tds int 3257 if ac.Trace != nil { 3258 // Update trace destination and sampling 3259 td, tds = string(ac.Trace.Destination), ac.Trace.Sampling 3260 if !IsValidPublishSubject(td) { 3261 td, tds = _EMPTY_, 0 3262 } else if tds <= 0 || tds > 100 { 3263 tds = 100 3264 } 3265 } 3266 a.traceDest, a.traceDestSampling = td, tds 3267 3268 // Check for external authorization. 3269 if ac.HasExternalAuthorization() { 3270 a.extAuth = &jwt.ExternalAuthorization{} 3271 a.extAuth.AuthUsers.Add(ac.Authorization.AuthUsers...) 3272 a.extAuth.AllowedAccounts.Add(ac.Authorization.AllowedAccounts...) 3273 a.extAuth.XKey = ac.Authorization.XKey 3274 } 3275 3276 // Reset exports and imports here. 3277 3278 // Exports is creating a whole new map. 3279 a.exports = exportMap{} 3280 3281 // Imports are checked unlocked in processInbound, so we can't change out the struct here. Need to process inline. 3282 if a.imports.streams != nil { 3283 old.imports.streams = a.imports.streams 3284 a.imports.streams = nil 3285 } 3286 if a.imports.services != nil { 3287 old.imports.services = make(map[string]*serviceImport, len(a.imports.services)) 3288 } 3289 for k, v := range a.imports.services { 3290 old.imports.services[k] = v 3291 delete(a.imports.services, k) 3292 } 3293 3294 alteredScope := map[string]struct{}{} 3295 3296 // update account signing keys 3297 a.signingKeys = nil 3298 _, strict := s.strictSigningKeyUsage[a.Issuer] 3299 if len(ac.SigningKeys) > 0 || !strict { 3300 a.signingKeys = make(map[string]jwt.Scope) 3301 } 3302 signersChanged := false 3303 for k, scope := range ac.SigningKeys { 3304 a.signingKeys[k] = scope 3305 } 3306 if !strict { 3307 a.signingKeys[a.Name] = nil 3308 } 3309 if len(a.signingKeys) != len(old.signingKeys) { 3310 signersChanged = true 3311 } 3312 for k, scope := range a.signingKeys { 3313 if oldScope, ok := old.signingKeys[k]; !ok { 3314 signersChanged = true 3315 } else if !reflect.DeepEqual(scope, oldScope) { 3316 signersChanged = true 3317 alteredScope[k] = struct{}{} 3318 } 3319 } 3320 // collect mappings that need to be removed 3321 removeList := []string{} 3322 for _, m := range a.mappings { 3323 if _, ok := ac.Mappings[jwt.Subject(m.src)]; !ok { 3324 removeList = append(removeList, m.src) 3325 } 3326 } 3327 a.mu.Unlock() 3328 3329 for sub, wm := range ac.Mappings { 3330 mappings := make([]*MapDest, len(wm)) 3331 for i, m := range wm { 3332 mappings[i] = &MapDest{ 3333 Subject: string(m.Subject), 3334 Weight: m.GetWeight(), 3335 Cluster: m.Cluster, 3336 } 3337 } 3338 // This will overwrite existing entries 3339 a.AddWeightedMappings(string(sub), mappings...) 3340 } 3341 // remove mappings 3342 for _, rmMapping := range removeList { 3343 a.RemoveMapping(rmMapping) 3344 } 3345 3346 // Re-register system exports/imports. 3347 if a == s.SystemAccount() { 3348 s.addSystemAccountExports(a) 3349 } else { 3350 s.registerSystemImports(a) 3351 } 3352 3353 jsEnabled := s.JetStreamEnabled() 3354 3355 streamTokenExpirationChanged := false 3356 serviceTokenExpirationChanged := false 3357 3358 for _, e := range ac.Exports { 3359 switch e.Type { 3360 case jwt.Stream: 3361 s.Debugf("Adding stream export %q for %s", e.Subject, a.traceLabel()) 3362 if err := a.addStreamExportWithAccountPos( 3363 string(e.Subject), authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil { 3364 s.Debugf("Error adding stream export to account [%s]: %v", a.traceLabel(), err.Error()) 3365 } 3366 case jwt.Service: 3367 s.Debugf("Adding service export %q for %s", e.Subject, a.traceLabel()) 3368 rt := Singleton 3369 switch e.ResponseType { 3370 case jwt.ResponseTypeStream: 3371 rt = Streamed 3372 case jwt.ResponseTypeChunked: 3373 rt = Chunked 3374 } 3375 if err := a.addServiceExportWithResponseAndAccountPos( 3376 string(e.Subject), rt, authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil { 3377 s.Debugf("Error adding service export to account [%s]: %v", a.traceLabel(), err) 3378 continue 3379 } 3380 sub := string(e.Subject) 3381 if e.Latency != nil { 3382 if err := a.TrackServiceExportWithSampling(sub, string(e.Latency.Results), int(e.Latency.Sampling)); err != nil { 3383 hdrNote := _EMPTY_ 3384 if e.Latency.Sampling == jwt.Headers { 3385 hdrNote = " (using headers)" 3386 } 3387 s.Debugf("Error adding latency tracking%s for service export to account [%s]: %v", hdrNote, a.traceLabel(), err) 3388 } 3389 } 3390 if e.ResponseThreshold != 0 { 3391 // Response threshold was set in options. 3392 if err := a.SetServiceExportResponseThreshold(sub, e.ResponseThreshold); err != nil { 3393 s.Debugf("Error adding service export response threshold for [%s]: %v", a.traceLabel(), err) 3394 } 3395 } 3396 if err := a.SetServiceExportAllowTrace(sub, e.AllowTrace); err != nil { 3397 s.Debugf("Error adding allow_trace for %q: %v", sub, err) 3398 } 3399 } 3400 3401 var revocationChanged *bool 3402 var ea *exportAuth 3403 3404 a.mu.Lock() 3405 switch e.Type { 3406 case jwt.Stream: 3407 revocationChanged = &streamTokenExpirationChanged 3408 if se, ok := a.exports.streams[string(e.Subject)]; ok && se != nil { 3409 ea = &se.exportAuth 3410 } 3411 case jwt.Service: 3412 revocationChanged = &serviceTokenExpirationChanged 3413 if se, ok := a.exports.services[string(e.Subject)]; ok && se != nil { 3414 ea = &se.exportAuth 3415 } 3416 } 3417 if ea != nil { 3418 oldRevocations := ea.actsRevoked 3419 if len(e.Revocations) == 0 { 3420 // remove all, no need to evaluate existing imports 3421 ea.actsRevoked = nil 3422 } else if len(oldRevocations) == 0 { 3423 // add all, existing imports need to be re evaluated 3424 ea.actsRevoked = e.Revocations 3425 *revocationChanged = true 3426 } else { 3427 ea.actsRevoked = e.Revocations 3428 // diff, existing imports need to be conditionally re evaluated, depending on: 3429 // if a key was added, or it's timestamp increased 3430 for k, t := range e.Revocations { 3431 if tOld, ok := oldRevocations[k]; !ok || tOld < t { 3432 *revocationChanged = true 3433 } 3434 } 3435 } 3436 } 3437 a.mu.Unlock() 3438 } 3439 var incompleteImports []*jwt.Import 3440 for _, i := range ac.Imports { 3441 // check tmpAccounts with priority 3442 var acc *Account 3443 var err error 3444 if v, ok := s.tmpAccounts.Load(i.Account); ok { 3445 acc = v.(*Account) 3446 } else { 3447 acc, err = s.lookupAccount(i.Account) 3448 } 3449 if acc == nil || err != nil { 3450 s.Errorf("Can't locate account [%s] for import of [%v] %s (err=%v)", i.Account, i.Subject, i.Type, err) 3451 incompleteImports = append(incompleteImports, i) 3452 continue 3453 } 3454 from := string(i.Subject) 3455 to := i.GetTo() 3456 switch i.Type { 3457 case jwt.Stream: 3458 if i.LocalSubject != _EMPTY_ { 3459 // set local subject implies to is empty 3460 to = string(i.LocalSubject) 3461 s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to) 3462 err = a.AddMappedStreamImportWithClaim(acc, from, to, i) 3463 } else { 3464 s.Debugf("Adding stream import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to) 3465 err = a.AddStreamImportWithClaim(acc, from, to, i) 3466 } 3467 if err != nil { 3468 s.Debugf("Error adding stream import to account [%s]: %v", a.traceLabel(), err.Error()) 3469 incompleteImports = append(incompleteImports, i) 3470 } 3471 case jwt.Service: 3472 if i.LocalSubject != _EMPTY_ { 3473 from = string(i.LocalSubject) 3474 to = string(i.Subject) 3475 } 3476 s.Debugf("Adding service import %s:%q for %s:%q", acc.traceLabel(), from, a.traceLabel(), to) 3477 if err := a.AddServiceImportWithClaim(acc, from, to, i); err != nil { 3478 s.Debugf("Error adding service import to account [%s]: %v", a.traceLabel(), err.Error()) 3479 incompleteImports = append(incompleteImports, i) 3480 } 3481 } 3482 } 3483 // Now let's apply any needed changes from import/export changes. 3484 if !a.checkStreamImportsEqual(old) { 3485 awcsti := map[string]struct{}{a.Name: {}} 3486 for _, c := range a.getClients() { 3487 c.processSubsOnConfigReload(awcsti) 3488 } 3489 } 3490 // Now check if stream exports have changed. 3491 if !a.checkStreamExportsEqual(old) || signersChanged || streamTokenExpirationChanged { 3492 clients := map[*client]struct{}{} 3493 // We need to check all accounts that have an import claim from this account. 3494 awcsti := map[string]struct{}{} 3495 s.accounts.Range(func(k, v interface{}) bool { 3496 acc := v.(*Account) 3497 // Move to the next if this account is actually account "a". 3498 if acc.Name == a.Name { 3499 return true 3500 } 3501 // TODO: checkStreamImportAuthorized() stack should not be trying 3502 // to lock "acc". If we find that to be needed, we will need to 3503 // rework this to ensure we don't lock acc. 3504 acc.mu.Lock() 3505 for _, im := range acc.imports.streams { 3506 if im != nil && im.acc.Name == a.Name { 3507 // Check for if we are still authorized for an import. 3508 im.invalid = !a.checkStreamImportAuthorized(acc, im.from, im.claim) 3509 awcsti[acc.Name] = struct{}{} 3510 for c := range acc.clients { 3511 clients[c] = struct{}{} 3512 } 3513 } 3514 } 3515 acc.mu.Unlock() 3516 return true 3517 }) 3518 // Now walk clients. 3519 for c := range clients { 3520 c.processSubsOnConfigReload(awcsti) 3521 } 3522 } 3523 // Now check if service exports have changed. 3524 if !a.checkServiceExportsEqual(old) || signersChanged || serviceTokenExpirationChanged { 3525 s.accounts.Range(func(k, v interface{}) bool { 3526 acc := v.(*Account) 3527 // Move to the next if this account is actually account "a". 3528 if acc.Name == a.Name { 3529 return true 3530 } 3531 // TODO: checkServiceImportAuthorized() stack should not be trying 3532 // to lock "acc". If we find that to be needed, we will need to 3533 // rework this to ensure we don't lock acc. 3534 acc.mu.Lock() 3535 for _, si := range acc.imports.services { 3536 if si != nil && si.acc.Name == a.Name { 3537 // Check for if we are still authorized for an import. 3538 si.invalid = !a.checkServiceImportAuthorized(acc, si.to, si.claim) 3539 // Make sure we should still be tracking latency and if we 3540 // are allowed to trace. 3541 if !si.response { 3542 if se := a.getServiceExport(si.to); se != nil { 3543 if si.latency != nil { 3544 si.latency = se.latency 3545 } 3546 // Update allow trace. 3547 si.atrc = se.atrc 3548 } 3549 } 3550 } 3551 } 3552 acc.mu.Unlock() 3553 return true 3554 }) 3555 } 3556 3557 // Now make sure we shutdown the old service import subscriptions. 3558 var sids [][]byte 3559 a.mu.RLock() 3560 c := a.ic 3561 for _, si := range old.imports.services { 3562 if c != nil && si.sid != nil { 3563 sids = append(sids, si.sid) 3564 } 3565 } 3566 a.mu.RUnlock() 3567 for _, sid := range sids { 3568 c.processUnsub(sid) 3569 } 3570 3571 // Now do limits if they are present. 3572 a.mu.Lock() 3573 a.msubs = int32(ac.Limits.Subs) 3574 a.mpay = int32(ac.Limits.Payload) 3575 a.mconns = int32(ac.Limits.Conn) 3576 a.mleafs = int32(ac.Limits.LeafNodeConn) 3577 a.disallowBearer = ac.Limits.DisallowBearer 3578 // Check for any revocations 3579 if len(ac.Revocations) > 0 { 3580 // We will always replace whatever we had with most current, so no 3581 // need to look at what we have. 3582 a.usersRevoked = make(map[string]int64, len(ac.Revocations)) 3583 for pk, t := range ac.Revocations { 3584 a.usersRevoked[pk] = t 3585 } 3586 } else { 3587 a.usersRevoked = nil 3588 } 3589 a.defaultPerms = buildPermissionsFromJwt(&ac.DefaultPermissions) 3590 a.incomplete = len(incompleteImports) != 0 3591 for _, i := range incompleteImports { 3592 s.incompleteAccExporterMap.Store(i.Account, struct{}{}) 3593 } 3594 if a.srv == nil { 3595 a.srv = s 3596 } 3597 3598 if ac.Limits.IsJSEnabled() { 3599 toUnlimited := func(value int64) int64 { 3600 if value > 0 { 3601 return value 3602 } 3603 return -1 3604 } 3605 if ac.Limits.JetStreamLimits.DiskStorage != 0 || ac.Limits.JetStreamLimits.MemoryStorage != 0 { 3606 // JetStreamAccountLimits and jwt.JetStreamLimits use same value for unlimited 3607 a.jsLimits = map[string]JetStreamAccountLimits{ 3608 _EMPTY_: { 3609 MaxMemory: ac.Limits.JetStreamLimits.MemoryStorage, 3610 MaxStore: ac.Limits.JetStreamLimits.DiskStorage, 3611 MaxStreams: int(ac.Limits.JetStreamLimits.Streams), 3612 MaxConsumers: int(ac.Limits.JetStreamLimits.Consumer), 3613 MemoryMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.MemoryMaxStreamBytes), 3614 StoreMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.DiskMaxStreamBytes), 3615 MaxBytesRequired: ac.Limits.JetStreamLimits.MaxBytesRequired, 3616 MaxAckPending: int(toUnlimited(ac.Limits.JetStreamLimits.MaxAckPending)), 3617 }, 3618 } 3619 } else { 3620 a.jsLimits = map[string]JetStreamAccountLimits{} 3621 for t, l := range ac.Limits.JetStreamTieredLimits { 3622 a.jsLimits[t] = JetStreamAccountLimits{ 3623 MaxMemory: l.MemoryStorage, 3624 MaxStore: l.DiskStorage, 3625 MaxStreams: int(l.Streams), 3626 MaxConsumers: int(l.Consumer), 3627 MemoryMaxStreamBytes: toUnlimited(l.MemoryMaxStreamBytes), 3628 StoreMaxStreamBytes: toUnlimited(l.DiskMaxStreamBytes), 3629 MaxBytesRequired: l.MaxBytesRequired, 3630 MaxAckPending: int(toUnlimited(l.MaxAckPending)), 3631 } 3632 } 3633 } 3634 } else if a.jsLimits != nil { 3635 // covers failed update followed by disable 3636 a.jsLimits = nil 3637 } 3638 3639 a.updated = time.Now() 3640 clients := a.getClientsLocked() 3641 a.mu.Unlock() 3642 3643 // Sort if we are over the limit. 3644 if a.MaxTotalConnectionsReached() { 3645 sort.Slice(clients, func(i, j int) bool { 3646 return clients[i].start.After(clients[j].start) 3647 }) 3648 } 3649 3650 // If JetStream is enabled for this server we will call into configJetStream for the account 3651 // regardless of enabled or disabled. It handles both cases. 3652 if jsEnabled { 3653 if err := s.configJetStream(a); err != nil { 3654 s.Errorf("Error configuring jetstream for account [%s]: %v", a.traceLabel(), err.Error()) 3655 a.mu.Lock() 3656 // Absent reload of js server cfg, this is going to be broken until js is disabled 3657 a.incomplete = true 3658 a.mu.Unlock() 3659 } 3660 } else if a.jsLimits != nil { 3661 // We do not have JS enabled for this server, but the account has it enabled so setup 3662 // our imports properly. This allows this server to proxy JS traffic correctly. 3663 s.checkJetStreamExports() 3664 a.enableAllJetStreamServiceImportsAndMappings() 3665 } 3666 3667 for i, c := range clients { 3668 a.mu.RLock() 3669 exceeded := a.mconns != jwt.NoLimit && i >= int(a.mconns) 3670 a.mu.RUnlock() 3671 if exceeded { 3672 c.maxAccountConnExceeded() 3673 continue 3674 } 3675 c.mu.Lock() 3676 c.applyAccountLimits() 3677 theJWT := c.opts.JWT 3678 c.mu.Unlock() 3679 // Check for being revoked here. We use ac one to avoid the account lock. 3680 if (ac.Revocations != nil || ac.Limits.DisallowBearer) && theJWT != _EMPTY_ { 3681 if juc, err := jwt.DecodeUserClaims(theJWT); err != nil { 3682 c.Debugf("User JWT not valid: %v", err) 3683 c.authViolation() 3684 continue 3685 } else if juc.BearerToken && ac.Limits.DisallowBearer { 3686 c.Debugf("Bearer User JWT not allowed") 3687 c.authViolation() 3688 continue 3689 } else if ok := ac.IsClaimRevoked(juc); ok { 3690 c.sendErrAndDebug("User Authentication Revoked") 3691 c.closeConnection(Revocation) 3692 continue 3693 } 3694 } 3695 } 3696 3697 // Check if the signing keys changed, might have to evict 3698 if signersChanged { 3699 for _, c := range clients { 3700 c.mu.Lock() 3701 if c.user == nil { 3702 c.mu.Unlock() 3703 continue 3704 } 3705 sk := c.user.SigningKey 3706 c.mu.Unlock() 3707 if sk == _EMPTY_ { 3708 continue 3709 } 3710 if _, ok := alteredScope[sk]; ok { 3711 c.closeConnection(AuthenticationViolation) 3712 } else if _, ok := a.hasIssuer(sk); !ok { 3713 c.closeConnection(AuthenticationViolation) 3714 } 3715 } 3716 } 3717 3718 if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok && refreshImportingAccounts { 3719 s.incompleteAccExporterMap.Delete(old.Name) 3720 s.accounts.Range(func(key, value interface{}) bool { 3721 acc := value.(*Account) 3722 acc.mu.RLock() 3723 incomplete := acc.incomplete 3724 name := acc.Name 3725 label := acc.traceLabel() 3726 // Must use jwt in account or risk failing on fetch 3727 // This jwt may not be the same that caused exportingAcc to be in incompleteAccExporterMap 3728 claimJWT := acc.claimJWT 3729 acc.mu.RUnlock() 3730 if incomplete && name != old.Name { 3731 if accClaims, _, err := s.verifyAccountClaims(claimJWT); err == nil { 3732 // Since claimJWT has not changed, acc can become complete 3733 // but it won't alter incomplete for it's dependents accounts. 3734 s.updateAccountClaimsWithRefresh(acc, accClaims, false) 3735 // old.Name was deleted before ranging over accounts 3736 // If it exists again, UpdateAccountClaims set it for failed imports of acc. 3737 // So there was one import of acc that imported this account and failed again. 3738 // Since this account just got updated, the import itself may be in error. So trace that. 3739 if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok { 3740 s.incompleteAccExporterMap.Delete(old.Name) 3741 s.Errorf("Account %s has issues importing account %s", label, old.Name) 3742 } 3743 } 3744 } 3745 return true 3746 }) 3747 } 3748 } 3749 3750 // Helper to build an internal account structure from a jwt.AccountClaims. 3751 // Lock MUST NOT be held upon entry. 3752 func (s *Server) buildInternalAccount(ac *jwt.AccountClaims) *Account { 3753 acc := NewAccount(ac.Subject) 3754 acc.Issuer = ac.Issuer 3755 // Set this here since we are placing in s.tmpAccounts below and may be 3756 // referenced by an route RS+, etc. 3757 s.setAccountSublist(acc) 3758 3759 // We don't want to register an account that is in the process of 3760 // being built, however, to solve circular import dependencies, we 3761 // need to store it here. 3762 s.tmpAccounts.Store(ac.Subject, acc) 3763 s.UpdateAccountClaims(acc, ac) 3764 return acc 3765 } 3766 3767 // Helper to build Permissions from jwt.Permissions 3768 // or return nil if none were specified 3769 func buildPermissionsFromJwt(uc *jwt.Permissions) *Permissions { 3770 if uc == nil { 3771 return nil 3772 } 3773 var p *Permissions 3774 if len(uc.Pub.Allow) > 0 || len(uc.Pub.Deny) > 0 { 3775 p = &Permissions{} 3776 p.Publish = &SubjectPermission{} 3777 p.Publish.Allow = uc.Pub.Allow 3778 p.Publish.Deny = uc.Pub.Deny 3779 } 3780 if len(uc.Sub.Allow) > 0 || len(uc.Sub.Deny) > 0 { 3781 if p == nil { 3782 p = &Permissions{} 3783 } 3784 p.Subscribe = &SubjectPermission{} 3785 p.Subscribe.Allow = uc.Sub.Allow 3786 p.Subscribe.Deny = uc.Sub.Deny 3787 } 3788 if uc.Resp != nil { 3789 if p == nil { 3790 p = &Permissions{} 3791 } 3792 p.Response = &ResponsePermission{ 3793 MaxMsgs: uc.Resp.MaxMsgs, 3794 Expires: uc.Resp.Expires, 3795 } 3796 validateResponsePermissions(p) 3797 } 3798 return p 3799 } 3800 3801 // Helper to build internal NKeyUser. 3802 func buildInternalNkeyUser(uc *jwt.UserClaims, acts map[string]struct{}, acc *Account) *NkeyUser { 3803 nu := &NkeyUser{Nkey: uc.Subject, Account: acc, AllowedConnectionTypes: acts} 3804 if uc.IssuerAccount != _EMPTY_ { 3805 nu.SigningKey = uc.Issuer 3806 } 3807 3808 // Now check for permissions. 3809 var p = buildPermissionsFromJwt(&uc.Permissions) 3810 if p == nil && acc.defaultPerms != nil { 3811 p = acc.defaultPerms.clone() 3812 } 3813 nu.Permissions = p 3814 return nu 3815 } 3816 3817 func fetchAccount(res AccountResolver, name string) (string, error) { 3818 if !nkeys.IsValidPublicAccountKey(name) { 3819 return _EMPTY_, fmt.Errorf("will only fetch valid account keys") 3820 } 3821 return res.Fetch(name) 3822 } 3823 3824 // AccountResolver interface. This is to fetch Account JWTs by public nkeys 3825 type AccountResolver interface { 3826 Fetch(name string) (string, error) 3827 Store(name, jwt string) error 3828 IsReadOnly() bool 3829 Start(server *Server) error 3830 IsTrackingUpdate() bool 3831 Reload() error 3832 Close() 3833 } 3834 3835 // Default implementations of IsReadOnly/Start so only need to be written when changed 3836 type resolverDefaultsOpsImpl struct{} 3837 3838 func (*resolverDefaultsOpsImpl) IsReadOnly() bool { 3839 return true 3840 } 3841 3842 func (*resolverDefaultsOpsImpl) IsTrackingUpdate() bool { 3843 return false 3844 } 3845 3846 func (*resolverDefaultsOpsImpl) Start(*Server) error { 3847 return nil 3848 } 3849 3850 func (*resolverDefaultsOpsImpl) Reload() error { 3851 return nil 3852 } 3853 3854 func (*resolverDefaultsOpsImpl) Close() { 3855 } 3856 3857 func (*resolverDefaultsOpsImpl) Store(_, _ string) error { 3858 return fmt.Errorf("store operation not supported for URL Resolver") 3859 } 3860 3861 // MemAccResolver is a memory only resolver. 3862 // Mostly for testing. 3863 type MemAccResolver struct { 3864 sm sync.Map 3865 resolverDefaultsOpsImpl 3866 } 3867 3868 // Fetch will fetch the account jwt claims from the internal sync.Map. 3869 func (m *MemAccResolver) Fetch(name string) (string, error) { 3870 if j, ok := m.sm.Load(name); ok { 3871 return j.(string), nil 3872 } 3873 return _EMPTY_, ErrMissingAccount 3874 } 3875 3876 // Store will store the account jwt claims in the internal sync.Map. 3877 func (m *MemAccResolver) Store(name, jwt string) error { 3878 m.sm.Store(name, jwt) 3879 return nil 3880 } 3881 3882 func (m *MemAccResolver) IsReadOnly() bool { 3883 return false 3884 } 3885 3886 // URLAccResolver implements an http fetcher. 3887 type URLAccResolver struct { 3888 url string 3889 c *http.Client 3890 resolverDefaultsOpsImpl 3891 } 3892 3893 // NewURLAccResolver returns a new resolver for the given base URL. 3894 func NewURLAccResolver(url string) (*URLAccResolver, error) { 3895 if !strings.HasSuffix(url, "/") { 3896 url += "/" 3897 } 3898 // FIXME(dlc) - Make timeout and others configurable. 3899 // We create our own transport to amortize TLS. 3900 tr := &http.Transport{ 3901 MaxIdleConns: 10, 3902 IdleConnTimeout: 30 * time.Second, 3903 } 3904 ur := &URLAccResolver{ 3905 url: url, 3906 c: &http.Client{Timeout: DEFAULT_ACCOUNT_FETCH_TIMEOUT, Transport: tr}, 3907 } 3908 return ur, nil 3909 } 3910 3911 // Fetch will fetch the account jwt claims from the base url, appending the 3912 // account name onto the end. 3913 func (ur *URLAccResolver) Fetch(name string) (string, error) { 3914 url := ur.url + name 3915 resp, err := ur.c.Get(url) 3916 if err != nil { 3917 return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), err) 3918 } else if resp == nil { 3919 return _EMPTY_, fmt.Errorf("could not fetch <%q>: no response", redactURLString(url)) 3920 } 3921 defer resp.Body.Close() 3922 if resp.StatusCode != http.StatusOK { 3923 return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), resp.Status) 3924 } 3925 body, err := io.ReadAll(resp.Body) 3926 if err != nil { 3927 return _EMPTY_, err 3928 } 3929 return string(body), nil 3930 } 3931 3932 // Resolver based on nats for synchronization and backing directory for storage. 3933 type DirAccResolver struct { 3934 *DirJWTStore 3935 *Server 3936 syncInterval time.Duration 3937 fetchTimeout time.Duration 3938 } 3939 3940 func (dr *DirAccResolver) IsTrackingUpdate() bool { 3941 return true 3942 } 3943 3944 func (dr *DirAccResolver) Reload() error { 3945 return dr.DirJWTStore.Reload() 3946 } 3947 3948 // ServerAPIClaimUpdateResponse is the response to $SYS.REQ.ACCOUNT.<id>.CLAIMS.UPDATE and $SYS.REQ.CLAIMS.UPDATE 3949 type ServerAPIClaimUpdateResponse struct { 3950 Server *ServerInfo `json:"server"` 3951 Data *ClaimUpdateStatus `json:"data,omitempty"` 3952 Error *ClaimUpdateError `json:"error,omitempty"` 3953 } 3954 3955 type ClaimUpdateError struct { 3956 Account string `json:"account,omitempty"` 3957 Code int `json:"code"` 3958 Description string `json:"description,omitempty"` 3959 } 3960 3961 type ClaimUpdateStatus struct { 3962 Account string `json:"account,omitempty"` 3963 Code int `json:"code,omitempty"` 3964 Message string `json:"message,omitempty"` 3965 } 3966 3967 func respondToUpdate(s *Server, respSubj string, acc string, message string, err error) { 3968 if err == nil { 3969 if acc == _EMPTY_ { 3970 s.Debugf("%s", message) 3971 } else { 3972 s.Debugf("%s - %s", message, acc) 3973 } 3974 } else { 3975 if acc == _EMPTY_ { 3976 s.Errorf("%s - %s", message, err) 3977 } else { 3978 s.Errorf("%s - %s - %s", message, acc, err) 3979 } 3980 } 3981 if respSubj == _EMPTY_ { 3982 return 3983 } 3984 3985 response := ServerAPIClaimUpdateResponse{ 3986 Server: &ServerInfo{}, 3987 } 3988 3989 if err == nil { 3990 response.Data = &ClaimUpdateStatus{ 3991 Account: acc, 3992 Code: http.StatusOK, 3993 Message: message, 3994 } 3995 } else { 3996 response.Error = &ClaimUpdateError{ 3997 Account: acc, 3998 Code: http.StatusInternalServerError, 3999 Description: fmt.Sprintf("%s - %v", message, err), 4000 } 4001 } 4002 4003 s.sendInternalMsgLocked(respSubj, _EMPTY_, response.Server, response) 4004 } 4005 4006 func handleListRequest(store *DirJWTStore, s *Server, reply string) { 4007 if reply == _EMPTY_ { 4008 return 4009 } 4010 accIds := make([]string, 0, 1024) 4011 if err := store.PackWalk(1, func(partialPackMsg string) { 4012 if tk := strings.Split(partialPackMsg, "|"); len(tk) == 2 { 4013 accIds = append(accIds, tk[0]) 4014 } 4015 }); err != nil { 4016 // let them timeout 4017 s.Errorf("list request error: %v", err) 4018 } else { 4019 s.Debugf("list request responded with %d account ids", len(accIds)) 4020 server := &ServerInfo{} 4021 response := map[string]interface{}{"server": server, "data": accIds} 4022 s.sendInternalMsgLocked(reply, _EMPTY_, server, response) 4023 } 4024 } 4025 4026 func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) { 4027 var accIds []interface{} 4028 var subj, sysAccName string 4029 if sysAcc := s.SystemAccount(); sysAcc != nil { 4030 sysAccName = sysAcc.GetName() 4031 } 4032 // Only operator and operator signing key are allowed to delete 4033 gk, err := jwt.DecodeGeneric(string(msg)) 4034 if err == nil { 4035 subj = gk.Subject 4036 if store.deleteType == NoDelete { 4037 err = fmt.Errorf("delete must be enabled in server config") 4038 } else if subj != gk.Issuer { 4039 err = fmt.Errorf("not self signed") 4040 } else if _, ok := store.operator[gk.Issuer]; !ok { 4041 err = fmt.Errorf("not trusted") 4042 } else if list, ok := gk.Data["accounts"]; !ok { 4043 err = fmt.Errorf("malformed request") 4044 } else if accIds, ok = list.([]interface{}); !ok { 4045 err = fmt.Errorf("malformed request") 4046 } else { 4047 for _, entry := range accIds { 4048 if acc, ok := entry.(string); !ok || 4049 acc == _EMPTY_ || !nkeys.IsValidPublicAccountKey(acc) { 4050 err = fmt.Errorf("malformed request") 4051 break 4052 } else if acc == sysAccName { 4053 err = fmt.Errorf("not allowed to delete system account") 4054 break 4055 } 4056 } 4057 } 4058 } 4059 if err != nil { 4060 respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("delete accounts request by %s failed", subj), err) 4061 return 4062 } 4063 errs := []string{} 4064 passCnt := 0 4065 for _, acc := range accIds { 4066 if err := store.delete(acc.(string)); err != nil { 4067 errs = append(errs, err.Error()) 4068 } else { 4069 passCnt++ 4070 } 4071 } 4072 if len(errs) == 0 { 4073 respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts", passCnt), nil) 4074 } else { 4075 respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts, failed for %d", passCnt, len(errs)), 4076 errors.New(strings.Join(errs, "\n"))) 4077 } 4078 } 4079 4080 func getOperatorKeys(s *Server) (string, map[string]struct{}, bool, error) { 4081 var op string 4082 var strict bool 4083 keys := make(map[string]struct{}) 4084 if opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 { 4085 op = opts.TrustedOperators[0].Subject 4086 strict = opts.TrustedOperators[0].StrictSigningKeyUsage 4087 if !strict { 4088 keys[opts.TrustedOperators[0].Subject] = struct{}{} 4089 } 4090 for _, key := range opts.TrustedOperators[0].SigningKeys { 4091 keys[key] = struct{}{} 4092 } 4093 } 4094 if len(keys) == 0 { 4095 return _EMPTY_, nil, false, fmt.Errorf("no operator key found") 4096 } 4097 return op, keys, strict, nil 4098 } 4099 4100 func claimValidate(claim *jwt.AccountClaims) error { 4101 vr := &jwt.ValidationResults{} 4102 claim.Validate(vr) 4103 if vr.IsBlocking(false) { 4104 return fmt.Errorf("validation errors: %v", vr.Errors()) 4105 } 4106 return nil 4107 } 4108 4109 func removeCb(s *Server, pubKey string) { 4110 v, ok := s.accounts.Load(pubKey) 4111 if !ok { 4112 return 4113 } 4114 a := v.(*Account) 4115 s.Debugf("Disable account %s due to remove", pubKey) 4116 a.mu.Lock() 4117 // lock out new clients 4118 a.msubs = 0 4119 a.mpay = 0 4120 a.mconns = 0 4121 a.mleafs = 0 4122 a.updated = time.Now() 4123 jsa := a.js 4124 a.mu.Unlock() 4125 // set the account to be expired and disconnect clients 4126 a.expiredTimeout() 4127 // For JS, we need also to disable it. 4128 if js := s.getJetStream(); js != nil && jsa != nil { 4129 js.disableJetStream(jsa) 4130 // Remove JetStream state in memory, this will be reset 4131 // on the changed callback from the account in case it is 4132 // enabled again. 4133 a.js = nil 4134 } 4135 // We also need to remove all ServerImport subscriptions 4136 a.removeAllServiceImportSubs() 4137 a.mu.Lock() 4138 a.clearExpirationTimer() 4139 a.mu.Unlock() 4140 } 4141 4142 func (dr *DirAccResolver) Start(s *Server) error { 4143 op, opKeys, strict, err := getOperatorKeys(s) 4144 if err != nil { 4145 return err 4146 } 4147 dr.Lock() 4148 defer dr.Unlock() 4149 dr.Server = s 4150 dr.operator = opKeys 4151 dr.DirJWTStore.changed = func(pubKey string) { 4152 if v, ok := s.accounts.Load(pubKey); ok { 4153 if theJwt, err := dr.LoadAcc(pubKey); err != nil { 4154 s.Errorf("DirResolver - Update got error on load: %v", err) 4155 } else { 4156 acc := v.(*Account) 4157 if err = s.updateAccountWithClaimJWT(acc, theJwt); err != nil { 4158 s.Errorf("DirResolver - Update for account %q resulted in error %v", pubKey, err) 4159 } else { 4160 if _, jsa, err := acc.checkForJetStream(); err != nil { 4161 if !IsNatsErr(err, JSNotEnabledForAccountErr) { 4162 s.Warnf("DirResolver - Error checking for JetStream support for account %q: %v", pubKey, err) 4163 } 4164 } else if jsa == nil { 4165 if err = s.configJetStream(acc); err != nil { 4166 s.Errorf("DirResolver - Error configuring JetStream for account %q: %v", pubKey, err) 4167 } 4168 } 4169 } 4170 } 4171 } 4172 } 4173 dr.DirJWTStore.deleted = func(pubKey string) { 4174 removeCb(s, pubKey) 4175 } 4176 packRespIb := s.newRespInbox() 4177 for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { 4178 // subscribe to account jwt update requests 4179 if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) { 4180 var pubKey string 4181 tk := strings.Split(subj, tsep) 4182 if len(tk) == accUpdateTokensNew { 4183 pubKey = tk[accReqAccIndex] 4184 } else if len(tk) == accUpdateTokensOld { 4185 pubKey = tk[accUpdateAccIdxOld] 4186 } else { 4187 s.Debugf("DirResolver - jwt update skipped due to bad subject %q", subj) 4188 return 4189 } 4190 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4191 respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err) 4192 } else if err := claimValidate(claim); err != nil { 4193 respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err) 4194 } else if claim.Subject != pubKey { 4195 err := errors.New("subject does not match jwt content") 4196 respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) 4197 } else if claim.Issuer == op && strict { 4198 err := errors.New("operator requires issuer to be a signing key") 4199 respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) 4200 } else if err := dr.save(pubKey, string(msg)); err != nil { 4201 respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) 4202 } else { 4203 respondToUpdate(s, resp, pubKey, "jwt updated", nil) 4204 } 4205 }); err != nil { 4206 return fmt.Errorf("error setting up update handling: %v", err) 4207 } 4208 } 4209 if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) { 4210 // As this is a raw message, we need to extract payload and only decode claims from it, 4211 // in case request is sent with headers. 4212 _, msg = c.msgParts(msg) 4213 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4214 respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err) 4215 } else if claim.Issuer == op && strict { 4216 err := errors.New("operator requires issuer to be a signing key") 4217 respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err) 4218 } else if err := claimValidate(claim); err != nil { 4219 respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err) 4220 } else if err := dr.save(claim.Subject, string(msg)); err != nil { 4221 respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err) 4222 } else { 4223 respondToUpdate(s, resp, claim.Subject, "jwt updated", nil) 4224 } 4225 }); err != nil { 4226 return fmt.Errorf("error setting up update handling: %v", err) 4227 } 4228 // respond to lookups with our version 4229 if _, err := s.sysSubscribe(fmt.Sprintf(accLookupReqSubj, "*"), func(_ *subscription, _ *client, _ *Account, subj, reply string, msg []byte) { 4230 if reply == _EMPTY_ { 4231 return 4232 } 4233 tk := strings.Split(subj, tsep) 4234 if len(tk) != accLookupReqTokens { 4235 return 4236 } 4237 accName := tk[accReqAccIndex] 4238 if theJWT, err := dr.DirJWTStore.LoadAcc(accName); err != nil { 4239 if errors.Is(err, fs.ErrNotExist) { 4240 s.Debugf("DirResolver - Could not find account %q", accName) 4241 // Reply with empty response to signal absence of JWT to others. 4242 s.sendInternalMsgLocked(reply, _EMPTY_, nil, nil) 4243 } else { 4244 s.Errorf("DirResolver - Error looking up account %q: %v", accName, err) 4245 } 4246 } else { 4247 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(theJWT)) 4248 } 4249 }); err != nil { 4250 return fmt.Errorf("error setting up lookup request handling: %v", err) 4251 } 4252 // respond to pack requests with one or more pack messages 4253 // an empty message signifies the end of the response responder. 4254 if _, err := s.sysSubscribeQ(accPackReqSubj, "responder", func(_ *subscription, _ *client, _ *Account, _, reply string, theirHash []byte) { 4255 if reply == _EMPTY_ { 4256 return 4257 } 4258 ourHash := dr.DirJWTStore.Hash() 4259 if bytes.Equal(theirHash, ourHash[:]) { 4260 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{}) 4261 s.Debugf("DirResolver - Pack request matches hash %x", ourHash[:]) 4262 } else if err := dr.DirJWTStore.PackWalk(1, func(partialPackMsg string) { 4263 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(partialPackMsg)) 4264 }); err != nil { 4265 // let them timeout 4266 s.Errorf("DirResolver - Pack request error: %v", err) 4267 } else { 4268 s.Debugf("DirResolver - Pack request hash %x - finished responding with hash %x", theirHash, ourHash) 4269 s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{}) 4270 } 4271 }); err != nil { 4272 return fmt.Errorf("error setting up pack request handling: %v", err) 4273 } 4274 // respond to list requests with one message containing all account ids 4275 if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) { 4276 handleListRequest(dr.DirJWTStore, s, reply) 4277 }); err != nil { 4278 return fmt.Errorf("error setting up list request handling: %v", err) 4279 } 4280 if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { 4281 handleDeleteRequest(dr.DirJWTStore, s, msg, reply) 4282 }); err != nil { 4283 return fmt.Errorf("error setting up delete request handling: %v", err) 4284 } 4285 // embed pack responses into store 4286 if _, err := s.sysSubscribe(packRespIb, func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) { 4287 hash := dr.DirJWTStore.Hash() 4288 if len(msg) == 0 { // end of response stream 4289 s.Debugf("DirResolver - Merging finished and resulting in: %x", dr.DirJWTStore.Hash()) 4290 return 4291 } else if err := dr.DirJWTStore.Merge(string(msg)); err != nil { 4292 s.Errorf("DirResolver - Merging resulted in error: %v", err) 4293 } else { 4294 s.Debugf("DirResolver - Merging succeeded and changed %x to %x", hash, dr.DirJWTStore.Hash()) 4295 } 4296 }); err != nil { 4297 return fmt.Errorf("error setting up pack response handling: %v", err) 4298 } 4299 // periodically send out pack message 4300 quit := s.quitCh 4301 s.startGoRoutine(func() { 4302 defer s.grWG.Done() 4303 ticker := time.NewTicker(dr.syncInterval) 4304 for { 4305 select { 4306 case <-quit: 4307 ticker.Stop() 4308 return 4309 case <-ticker.C: 4310 } 4311 ourHash := dr.DirJWTStore.Hash() 4312 s.Debugf("DirResolver - Checking store state: %x", ourHash) 4313 s.sendInternalMsgLocked(accPackReqSubj, packRespIb, nil, ourHash[:]) 4314 } 4315 }) 4316 s.Noticef("Managing all jwt in exclusive directory %s", dr.directory) 4317 return nil 4318 } 4319 4320 func (dr *DirAccResolver) Fetch(name string) (string, error) { 4321 if theJWT, err := dr.LoadAcc(name); theJWT != _EMPTY_ { 4322 return theJWT, nil 4323 } else { 4324 dr.Lock() 4325 srv := dr.Server 4326 to := dr.fetchTimeout 4327 dr.Unlock() 4328 if srv == nil { 4329 return _EMPTY_, err 4330 } 4331 return srv.fetch(dr, name, to) // lookup from other server 4332 } 4333 } 4334 4335 func (dr *DirAccResolver) Store(name, jwt string) error { 4336 return dr.saveIfNewer(name, jwt) 4337 } 4338 4339 type DirResOption func(s *DirAccResolver) error 4340 4341 // limits the amount of time spent waiting for an account fetch to complete 4342 func FetchTimeout(to time.Duration) DirResOption { 4343 return func(r *DirAccResolver) error { 4344 if to <= time.Duration(0) { 4345 return fmt.Errorf("Fetch timeout %v is too smal", to) 4346 } 4347 r.fetchTimeout = to 4348 return nil 4349 } 4350 } 4351 4352 func (dr *DirAccResolver) apply(opts ...DirResOption) error { 4353 for _, o := range opts { 4354 if err := o(dr); err != nil { 4355 return err 4356 } 4357 } 4358 return nil 4359 } 4360 4361 func NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete deleteType, opts ...DirResOption) (*DirAccResolver, error) { 4362 if limit == 0 { 4363 limit = math.MaxInt64 4364 } 4365 if syncInterval <= 0 { 4366 syncInterval = time.Minute 4367 } 4368 store, err := NewExpiringDirJWTStore(path, false, true, delete, 0, limit, false, 0, nil) 4369 if err != nil { 4370 return nil, err 4371 } 4372 4373 res := &DirAccResolver{store, nil, syncInterval, DEFAULT_ACCOUNT_FETCH_TIMEOUT} 4374 if err := res.apply(opts...); err != nil { 4375 return nil, err 4376 } 4377 return res, nil 4378 } 4379 4380 // Caching resolver using nats for lookups and making use of a directory for storage 4381 type CacheDirAccResolver struct { 4382 DirAccResolver 4383 ttl time.Duration 4384 } 4385 4386 func (s *Server) fetch(res AccountResolver, name string, timeout time.Duration) (string, error) { 4387 if s == nil { 4388 return _EMPTY_, ErrNoAccountResolver 4389 } 4390 respC := make(chan []byte, 1) 4391 accountLookupRequest := fmt.Sprintf(accLookupReqSubj, name) 4392 s.mu.Lock() 4393 if s.sys == nil || s.sys.replies == nil { 4394 s.mu.Unlock() 4395 return _EMPTY_, fmt.Errorf("eventing shut down") 4396 } 4397 // Resolver will wait for detected active servers to reply 4398 // before serving an error in case there weren't any found. 4399 expectedServers := len(s.sys.servers) 4400 replySubj := s.newRespInbox() 4401 replies := s.sys.replies 4402 4403 // Store our handler. 4404 replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { 4405 var clone []byte 4406 isEmpty := len(msg) == 0 4407 if !isEmpty { 4408 clone = make([]byte, len(msg)) 4409 copy(clone, msg) 4410 } 4411 s.mu.Lock() 4412 defer s.mu.Unlock() 4413 expectedServers-- 4414 // Skip empty responses until getting all the available servers. 4415 if isEmpty && expectedServers > 0 { 4416 return 4417 } 4418 // Use the first valid response if there is still interest or 4419 // one of the empty responses to signal that it was not found. 4420 if _, ok := replies[replySubj]; ok { 4421 select { 4422 case respC <- clone: 4423 default: 4424 } 4425 } 4426 } 4427 s.sendInternalMsg(accountLookupRequest, replySubj, nil, []byte{}) 4428 quit := s.quitCh 4429 s.mu.Unlock() 4430 var err error 4431 var theJWT string 4432 select { 4433 case <-quit: 4434 err = errors.New("fetching jwt failed due to shutdown") 4435 case <-time.After(timeout): 4436 err = errors.New("fetching jwt timed out") 4437 case m := <-respC: 4438 if len(m) == 0 { 4439 err = errors.New("account jwt not found") 4440 } else if err = res.Store(name, string(m)); err == nil { 4441 theJWT = string(m) 4442 } 4443 } 4444 s.mu.Lock() 4445 delete(replies, replySubj) 4446 s.mu.Unlock() 4447 close(respC) 4448 return theJWT, err 4449 } 4450 4451 func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, opts ...DirResOption) (*CacheDirAccResolver, error) { 4452 if limit <= 0 { 4453 limit = 1_000 4454 } 4455 store, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil) 4456 if err != nil { 4457 return nil, err 4458 } 4459 res := &CacheDirAccResolver{DirAccResolver{store, nil, 0, DEFAULT_ACCOUNT_FETCH_TIMEOUT}, ttl} 4460 if err := res.apply(opts...); err != nil { 4461 return nil, err 4462 } 4463 return res, nil 4464 } 4465 4466 func (dr *CacheDirAccResolver) Start(s *Server) error { 4467 op, opKeys, strict, err := getOperatorKeys(s) 4468 if err != nil { 4469 return err 4470 } 4471 dr.Lock() 4472 defer dr.Unlock() 4473 dr.Server = s 4474 dr.operator = opKeys 4475 dr.DirJWTStore.changed = func(pubKey string) { 4476 if v, ok := s.accounts.Load(pubKey); !ok { 4477 } else if theJwt, err := dr.LoadAcc(pubKey); err != nil { 4478 s.Errorf("DirResolver - Update got error on load: %v", err) 4479 } else if err := s.updateAccountWithClaimJWT(v.(*Account), theJwt); err != nil { 4480 s.Errorf("DirResolver - Update resulted in error %v", err) 4481 } 4482 } 4483 dr.DirJWTStore.deleted = func(pubKey string) { 4484 removeCb(s, pubKey) 4485 } 4486 for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { 4487 // subscribe to account jwt update requests 4488 if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) { 4489 var pubKey string 4490 tk := strings.Split(subj, tsep) 4491 if len(tk) == accUpdateTokensNew { 4492 pubKey = tk[accReqAccIndex] 4493 } else if len(tk) == accUpdateTokensOld { 4494 pubKey = tk[accUpdateAccIdxOld] 4495 } else { 4496 s.Debugf("DirResolver - jwt update cache skipped due to bad subject %q", subj) 4497 return 4498 } 4499 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4500 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4501 } else if claim.Subject != pubKey { 4502 err := errors.New("subject does not match jwt content") 4503 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4504 } else if claim.Issuer == op && strict { 4505 err := errors.New("operator requires issuer to be a signing key") 4506 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4507 } else if _, ok := s.accounts.Load(pubKey); !ok { 4508 respondToUpdate(s, resp, pubKey, "jwt update cache skipped", nil) 4509 } else if err := claimValidate(claim); err != nil { 4510 respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err) 4511 } else if err := dr.save(pubKey, string(msg)); err != nil { 4512 respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) 4513 } else { 4514 respondToUpdate(s, resp, pubKey, "jwt updated cache", nil) 4515 } 4516 }); err != nil { 4517 return fmt.Errorf("error setting up update handling: %v", err) 4518 } 4519 } 4520 if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) { 4521 // As this is a raw message, we need to extract payload and only decode claims from it, 4522 // in case request is sent with headers. 4523 _, msg = c.msgParts(msg) 4524 if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { 4525 respondToUpdate(s, resp, "n/a", "jwt update cache resulted in error", err) 4526 } else if claim.Issuer == op && strict { 4527 err := errors.New("operator requires issuer to be a signing key") 4528 respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err) 4529 } else if _, ok := s.accounts.Load(claim.Subject); !ok { 4530 respondToUpdate(s, resp, claim.Subject, "jwt update cache skipped", nil) 4531 } else if err := claimValidate(claim); err != nil { 4532 respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err) 4533 } else if err := dr.save(claim.Subject, string(msg)); err != nil { 4534 respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err) 4535 } else { 4536 respondToUpdate(s, resp, claim.Subject, "jwt updated cache", nil) 4537 } 4538 }); err != nil { 4539 return fmt.Errorf("error setting up update handling: %v", err) 4540 } 4541 // respond to list requests with one message containing all account ids 4542 if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) { 4543 handleListRequest(dr.DirJWTStore, s, reply) 4544 }); err != nil { 4545 return fmt.Errorf("error setting up list request handling: %v", err) 4546 } 4547 if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { 4548 handleDeleteRequest(dr.DirJWTStore, s, msg, reply) 4549 }); err != nil { 4550 return fmt.Errorf("error setting up list request handling: %v", err) 4551 } 4552 s.Noticef("Managing some jwt in exclusive directory %s", dr.directory) 4553 return nil 4554 } 4555 4556 func (dr *CacheDirAccResolver) Reload() error { 4557 return dr.DirAccResolver.Reload() 4558 }