github.com/xzl8028/xenia-server@v0.0.0-20190809101854-18450a97da63/app/web_hub.go (about) 1 // Copyright (c) 2015-present Xenia, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "fmt" 8 "hash/fnv" 9 "runtime" 10 "runtime/debug" 11 "strconv" 12 "strings" 13 "sync/atomic" 14 "time" 15 16 "github.com/xzl8028/xenia-server/mlog" 17 "github.com/xzl8028/xenia-server/model" 18 ) 19 20 const ( 21 BROADCAST_QUEUE_SIZE = 4096 22 DEADLOCK_TICKER = 15 * time.Second // check every 15 seconds 23 DEADLOCK_WARN = (BROADCAST_QUEUE_SIZE * 99) / 100 // number of buffered messages before printing stack trace 24 ) 25 26 type WebConnActivityMessage struct { 27 UserId string 28 SessionToken string 29 ActivityAt int64 30 } 31 32 type Hub struct { 33 // connectionCount should be kept first. 34 // See https://github.com/xzl8028/xenia-server/pull/7281 35 connectionCount int64 36 app *App 37 connectionIndex int 38 register chan *WebConn 39 unregister chan *WebConn 40 broadcast chan *model.WebSocketEvent 41 stop chan struct{} 42 didStop chan struct{} 43 invalidateUser chan string 44 activity chan *WebConnActivityMessage 45 ExplicitStop bool 46 goroutineId int 47 } 48 49 func (a *App) NewWebHub() *Hub { 50 return &Hub{ 51 app: a, 52 register: make(chan *WebConn, 1), 53 unregister: make(chan *WebConn, 1), 54 broadcast: make(chan *model.WebSocketEvent, BROADCAST_QUEUE_SIZE), 55 stop: make(chan struct{}), 56 didStop: make(chan struct{}), 57 invalidateUser: make(chan string), 58 activity: make(chan *WebConnActivityMessage), 59 ExplicitStop: false, 60 } 61 } 62 63 func (a *App) TotalWebsocketConnections() int { 64 count := int64(0) 65 for _, hub := range a.Srv.Hubs { 66 count = count + atomic.LoadInt64(&hub.connectionCount) 67 } 68 69 return int(count) 70 } 71 72 func (a *App) HubStart() { 73 // Total number of hubs is twice the number of CPUs. 74 numberOfHubs := runtime.NumCPU() * 2 75 mlog.Info(fmt.Sprintf("Starting %v websocket hubs", numberOfHubs)) 76 77 a.Srv.Hubs = make([]*Hub, numberOfHubs) 78 a.Srv.HubsStopCheckingForDeadlock = make(chan bool, 1) 79 80 for i := 0; i < len(a.Srv.Hubs); i++ { 81 a.Srv.Hubs[i] = a.NewWebHub() 82 a.Srv.Hubs[i].connectionIndex = i 83 a.Srv.Hubs[i].Start() 84 } 85 86 go func() { 87 ticker := time.NewTicker(DEADLOCK_TICKER) 88 89 defer func() { 90 ticker.Stop() 91 }() 92 93 for { 94 select { 95 case <-ticker.C: 96 for _, hub := range a.Srv.Hubs { 97 if len(hub.broadcast) >= DEADLOCK_WARN { 98 mlog.Error(fmt.Sprintf("Hub processing might be deadlock on hub %v goroutine %v with %v events in the buffer", hub.connectionIndex, hub.goroutineId, len(hub.broadcast))) 99 buf := make([]byte, 1<<16) 100 runtime.Stack(buf, true) 101 output := fmt.Sprintf("%s", buf) 102 splits := strings.Split(output, "goroutine ") 103 104 for _, part := range splits { 105 if strings.Contains(part, fmt.Sprintf("%v", hub.goroutineId)) { 106 mlog.Error(fmt.Sprintf("Trace for possible deadlock goroutine %v", part)) 107 } 108 } 109 } 110 } 111 112 case <-a.Srv.HubsStopCheckingForDeadlock: 113 return 114 } 115 } 116 }() 117 } 118 119 func (a *App) HubStop() { 120 mlog.Info("stopping websocket hub connections") 121 122 select { 123 case a.Srv.HubsStopCheckingForDeadlock <- true: 124 default: 125 mlog.Warn("We appear to have already sent the stop checking for deadlocks command") 126 } 127 128 for _, hub := range a.Srv.Hubs { 129 hub.Stop() 130 } 131 132 a.Srv.Hubs = []*Hub{} 133 } 134 135 func (a *App) GetHubForUserId(userId string) *Hub { 136 if len(a.Srv.Hubs) == 0 { 137 return nil 138 } 139 140 hash := fnv.New32a() 141 hash.Write([]byte(userId)) 142 index := hash.Sum32() % uint32(len(a.Srv.Hubs)) 143 return a.Srv.Hubs[index] 144 } 145 146 func (a *App) HubRegister(webConn *WebConn) { 147 hub := a.GetHubForUserId(webConn.UserId) 148 if hub != nil { 149 hub.Register(webConn) 150 } 151 } 152 153 func (a *App) HubUnregister(webConn *WebConn) { 154 hub := a.GetHubForUserId(webConn.UserId) 155 if hub != nil { 156 hub.Unregister(webConn) 157 } 158 } 159 160 func (a *App) Publish(message *model.WebSocketEvent) { 161 if metrics := a.Metrics; metrics != nil { 162 metrics.IncrementWebsocketEvent(message.Event) 163 } 164 165 a.PublishSkipClusterSend(message) 166 167 if a.Cluster != nil { 168 cm := &model.ClusterMessage{ 169 Event: model.CLUSTER_EVENT_PUBLISH, 170 SendType: model.CLUSTER_SEND_BEST_EFFORT, 171 Data: message.ToJson(), 172 } 173 174 if message.Event == model.WEBSOCKET_EVENT_POSTED || 175 message.Event == model.WEBSOCKET_EVENT_POST_EDITED || 176 message.Event == model.WEBSOCKET_EVENT_DIRECT_ADDED || 177 message.Event == model.WEBSOCKET_EVENT_GROUP_ADDED || 178 message.Event == model.WEBSOCKET_EVENT_ADDED_TO_TEAM { 179 cm.SendType = model.CLUSTER_SEND_RELIABLE 180 } 181 182 a.Cluster.SendClusterMessage(cm) 183 } 184 } 185 186 func (a *App) PublishSkipClusterSend(message *model.WebSocketEvent) { 187 if message.Broadcast.UserId != "" { 188 hub := a.GetHubForUserId(message.Broadcast.UserId) 189 if hub != nil { 190 hub.Broadcast(message) 191 } 192 } else { 193 for _, hub := range a.Srv.Hubs { 194 hub.Broadcast(message) 195 } 196 } 197 } 198 199 func (a *App) InvalidateCacheForChannel(channel *model.Channel) { 200 a.InvalidateCacheForChannelSkipClusterSend(channel.Id) 201 a.InvalidateCacheForChannelByNameSkipClusterSend(channel.TeamId, channel.Name) 202 203 if a.Cluster != nil { 204 msg := &model.ClusterMessage{ 205 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL, 206 SendType: model.CLUSTER_SEND_BEST_EFFORT, 207 Data: channel.Id, 208 } 209 210 a.Cluster.SendClusterMessage(msg) 211 212 nameMsg := &model.ClusterMessage{ 213 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_BY_NAME, 214 SendType: model.CLUSTER_SEND_BEST_EFFORT, 215 Props: make(map[string]string), 216 } 217 218 nameMsg.Props["name"] = channel.Name 219 if channel.TeamId == "" { 220 nameMsg.Props["id"] = "dm" 221 } else { 222 nameMsg.Props["id"] = channel.TeamId 223 } 224 225 a.Cluster.SendClusterMessage(nameMsg) 226 } 227 } 228 229 func (a *App) InvalidateCacheForChannelSkipClusterSend(channelId string) { 230 a.Srv.Store.Channel().InvalidateChannel(channelId) 231 } 232 233 func (a *App) InvalidateCacheForChannelMembers(channelId string) { 234 a.InvalidateCacheForChannelMembersSkipClusterSend(channelId) 235 236 if a.Cluster != nil { 237 msg := &model.ClusterMessage{ 238 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS, 239 SendType: model.CLUSTER_SEND_BEST_EFFORT, 240 Data: channelId, 241 } 242 a.Cluster.SendClusterMessage(msg) 243 } 244 } 245 246 func (a *App) InvalidateCacheForChannelMembersSkipClusterSend(channelId string) { 247 a.Srv.Store.User().InvalidateProfilesInChannelCache(channelId) 248 a.Srv.Store.Channel().InvalidateMemberCount(channelId) 249 } 250 251 func (a *App) InvalidateCacheForChannelMembersNotifyProps(channelId string) { 252 a.InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId) 253 254 if a.Cluster != nil { 255 msg := &model.ClusterMessage{ 256 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS_NOTIFY_PROPS, 257 SendType: model.CLUSTER_SEND_BEST_EFFORT, 258 Data: channelId, 259 } 260 a.Cluster.SendClusterMessage(msg) 261 } 262 } 263 264 func (a *App) InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId string) { 265 a.Srv.Store.Channel().InvalidateCacheForChannelMembersNotifyProps(channelId) 266 } 267 268 func (a *App) InvalidateCacheForChannelByNameSkipClusterSend(teamId, name string) { 269 if teamId == "" { 270 teamId = "dm" 271 } 272 273 a.Srv.Store.Channel().InvalidateChannelByName(teamId, name) 274 } 275 276 func (a *App) InvalidateCacheForChannelPosts(channelId string) { 277 a.InvalidateCacheForChannelPostsSkipClusterSend(channelId) 278 279 if a.Cluster != nil { 280 msg := &model.ClusterMessage{ 281 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_POSTS, 282 SendType: model.CLUSTER_SEND_BEST_EFFORT, 283 Data: channelId, 284 } 285 a.Cluster.SendClusterMessage(msg) 286 } 287 } 288 289 func (a *App) InvalidateCacheForChannelPostsSkipClusterSend(channelId string) { 290 a.Srv.Store.Post().InvalidateLastPostTimeCache(channelId) 291 } 292 293 func (a *App) InvalidateCacheForUser(userId string) { 294 a.InvalidateCacheForUserSkipClusterSend(userId) 295 296 if a.Cluster != nil { 297 msg := &model.ClusterMessage{ 298 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_USER, 299 SendType: model.CLUSTER_SEND_BEST_EFFORT, 300 Data: userId, 301 } 302 a.Cluster.SendClusterMessage(msg) 303 } 304 } 305 306 func (a *App) InvalidateCacheForUserTeams(userId string) { 307 a.InvalidateCacheForUserTeamsSkipClusterSend(userId) 308 309 if a.Cluster != nil { 310 msg := &model.ClusterMessage{ 311 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_USER_TEAMS, 312 SendType: model.CLUSTER_SEND_BEST_EFFORT, 313 Data: userId, 314 } 315 a.Cluster.SendClusterMessage(msg) 316 } 317 } 318 319 func (a *App) InvalidateCacheForUserSkipClusterSend(userId string) { 320 a.Srv.Store.Channel().InvalidateAllChannelMembersForUser(userId) 321 a.Srv.Store.User().InvalidateProfilesInChannelCacheByUser(userId) 322 a.Srv.Store.User().InvalidatProfileCacheForUser(userId) 323 324 hub := a.GetHubForUserId(userId) 325 if hub != nil { 326 hub.InvalidateUser(userId) 327 } 328 } 329 330 func (a *App) InvalidateCacheForUserTeamsSkipClusterSend(userId string) { 331 a.Srv.Store.Team().InvalidateAllTeamIdsForUser(userId) 332 333 hub := a.GetHubForUserId(userId) 334 if hub != nil { 335 hub.InvalidateUser(userId) 336 } 337 } 338 339 func (a *App) InvalidateCacheForWebhook(webhookId string) { 340 a.InvalidateCacheForWebhookSkipClusterSend(webhookId) 341 342 if a.Cluster != nil { 343 msg := &model.ClusterMessage{ 344 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_WEBHOOK, 345 SendType: model.CLUSTER_SEND_BEST_EFFORT, 346 Data: webhookId, 347 } 348 a.Cluster.SendClusterMessage(msg) 349 } 350 } 351 352 func (a *App) InvalidateCacheForWebhookSkipClusterSend(webhookId string) { 353 a.Srv.Store.Webhook().InvalidateWebhookCache(webhookId) 354 } 355 356 func (a *App) InvalidateWebConnSessionCacheForUser(userId string) { 357 hub := a.GetHubForUserId(userId) 358 if hub != nil { 359 hub.InvalidateUser(userId) 360 } 361 } 362 363 func (a *App) UpdateWebConnUserActivity(session model.Session, activityAt int64) { 364 hub := a.GetHubForUserId(session.UserId) 365 if hub != nil { 366 hub.UpdateActivity(session.UserId, session.Token, activityAt) 367 } 368 } 369 370 func (h *Hub) Register(webConn *WebConn) { 371 select { 372 case h.register <- webConn: 373 case <-h.didStop: 374 } 375 376 if webConn.IsAuthenticated() { 377 webConn.SendHello() 378 } 379 } 380 381 func (h *Hub) Unregister(webConn *WebConn) { 382 select { 383 case h.unregister <- webConn: 384 case <-h.stop: 385 } 386 } 387 388 func (h *Hub) Broadcast(message *model.WebSocketEvent) { 389 if h != nil && h.broadcast != nil && message != nil { 390 select { 391 case h.broadcast <- message: 392 case <-h.didStop: 393 } 394 } 395 } 396 397 func (h *Hub) InvalidateUser(userId string) { 398 select { 399 case h.invalidateUser <- userId: 400 case <-h.didStop: 401 } 402 } 403 404 func (h *Hub) UpdateActivity(userId, sessionToken string, activityAt int64) { 405 select { 406 case h.activity <- &WebConnActivityMessage{UserId: userId, SessionToken: sessionToken, ActivityAt: activityAt}: 407 case <-h.didStop: 408 } 409 } 410 411 func getGoroutineId() int { 412 var buf [64]byte 413 n := runtime.Stack(buf[:], false) 414 idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0] 415 id, err := strconv.Atoi(idField) 416 if err != nil { 417 id = -1 418 } 419 return id 420 } 421 422 func (h *Hub) Stop() { 423 close(h.stop) 424 <-h.didStop 425 } 426 427 func (h *Hub) Start() { 428 var doStart func() 429 var doRecoverableStart func() 430 var doRecover func() 431 432 doStart = func() { 433 h.goroutineId = getGoroutineId() 434 mlog.Debug(fmt.Sprintf("Hub for index %v is starting with goroutine %v", h.connectionIndex, h.goroutineId)) 435 436 connections := newHubConnectionIndex() 437 438 for { 439 select { 440 case webCon := <-h.register: 441 connections.Add(webCon) 442 atomic.StoreInt64(&h.connectionCount, int64(len(connections.All()))) 443 case webCon := <-h.unregister: 444 connections.Remove(webCon) 445 atomic.StoreInt64(&h.connectionCount, int64(len(connections.All()))) 446 447 if len(webCon.UserId) == 0 { 448 continue 449 } 450 451 conns := connections.ForUser(webCon.UserId) 452 if len(conns) == 0 { 453 h.app.Srv.Go(func() { 454 h.app.SetStatusOffline(webCon.UserId, false) 455 }) 456 } else { 457 var latestActivity int64 = 0 458 for _, conn := range conns { 459 if conn.LastUserActivityAt > latestActivity { 460 latestActivity = conn.LastUserActivityAt 461 } 462 } 463 if h.app.IsUserAway(latestActivity) { 464 h.app.Srv.Go(func() { 465 h.app.SetStatusLastActivityAt(webCon.UserId, latestActivity) 466 }) 467 } 468 } 469 case userId := <-h.invalidateUser: 470 for _, webCon := range connections.ForUser(userId) { 471 webCon.InvalidateCache() 472 } 473 case activity := <-h.activity: 474 for _, webCon := range connections.ForUser(activity.UserId) { 475 if webCon.GetSessionToken() == activity.SessionToken { 476 webCon.LastUserActivityAt = activity.ActivityAt 477 } 478 } 479 case msg := <-h.broadcast: 480 candidates := connections.All() 481 if msg.Broadcast.UserId != "" { 482 candidates = connections.ForUser(msg.Broadcast.UserId) 483 } 484 msg.PrecomputeJSON() 485 for _, webCon := range candidates { 486 if webCon.ShouldSendEvent(msg) { 487 select { 488 case webCon.Send <- msg: 489 default: 490 mlog.Error(fmt.Sprintf("webhub.broadcast: cannot send, closing websocket for userId=%v", webCon.UserId)) 491 close(webCon.Send) 492 connections.Remove(webCon) 493 } 494 } 495 } 496 case <-h.stop: 497 userIds := make(map[string]bool) 498 499 for _, webCon := range connections.All() { 500 userIds[webCon.UserId] = true 501 webCon.Close() 502 } 503 504 for userId := range userIds { 505 h.app.SetStatusOffline(userId, false) 506 } 507 508 h.ExplicitStop = true 509 close(h.didStop) 510 511 return 512 } 513 } 514 } 515 516 doRecoverableStart = func() { 517 defer doRecover() 518 doStart() 519 } 520 521 doRecover = func() { 522 if !h.ExplicitStop { 523 if r := recover(); r != nil { 524 mlog.Error(fmt.Sprintf("Recovering from Hub panic. Panic was: %v", r)) 525 } else { 526 mlog.Error("Webhub stopped unexpectedly. Recovering.") 527 } 528 529 mlog.Error(string(debug.Stack())) 530 531 go doRecoverableStart() 532 } 533 } 534 535 go doRecoverableStart() 536 } 537 538 type hubConnectionIndexIndexes struct { 539 connections int 540 connectionsByUserId int 541 } 542 543 // hubConnectionIndex provides fast addition, removal, and iteration of web connections. 544 type hubConnectionIndex struct { 545 connections []*WebConn 546 connectionsByUserId map[string][]*WebConn 547 connectionIndexes map[*WebConn]*hubConnectionIndexIndexes 548 } 549 550 func newHubConnectionIndex() *hubConnectionIndex { 551 return &hubConnectionIndex{ 552 connections: make([]*WebConn, 0, model.SESSION_CACHE_SIZE), 553 connectionsByUserId: make(map[string][]*WebConn), 554 connectionIndexes: make(map[*WebConn]*hubConnectionIndexIndexes), 555 } 556 } 557 558 func (i *hubConnectionIndex) Add(wc *WebConn) { 559 i.connections = append(i.connections, wc) 560 i.connectionsByUserId[wc.UserId] = append(i.connectionsByUserId[wc.UserId], wc) 561 i.connectionIndexes[wc] = &hubConnectionIndexIndexes{ 562 connections: len(i.connections) - 1, 563 connectionsByUserId: len(i.connectionsByUserId[wc.UserId]) - 1, 564 } 565 } 566 567 func (i *hubConnectionIndex) Remove(wc *WebConn) { 568 indexes, ok := i.connectionIndexes[wc] 569 if !ok { 570 return 571 } 572 573 last := i.connections[len(i.connections)-1] 574 i.connections[indexes.connections] = last 575 i.connections = i.connections[:len(i.connections)-1] 576 i.connectionIndexes[last].connections = indexes.connections 577 578 userConnections := i.connectionsByUserId[wc.UserId] 579 last = userConnections[len(userConnections)-1] 580 userConnections[indexes.connectionsByUserId] = last 581 i.connectionsByUserId[wc.UserId] = userConnections[:len(userConnections)-1] 582 i.connectionIndexes[last].connectionsByUserId = indexes.connectionsByUserId 583 584 delete(i.connectionIndexes, wc) 585 } 586 587 func (i *hubConnectionIndex) ForUser(id string) []*WebConn { 588 return i.connectionsByUserId[id] 589 } 590 591 func (i *hubConnectionIndex) All() []*WebConn { 592 return i.connections 593 }