github.com/adacta-ru/mattermost-server@v5.11.1+incompatible/app/web_hub.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "fmt" 8 "hash/fnv" 9 "runtime" 10 "runtime/debug" 11 "strconv" 12 "strings" 13 "sync/atomic" 14 "time" 15 16 "github.com/mattermost/mattermost-server/mlog" 17 "github.com/mattermost/mattermost-server/model" 18 ) 19 20 const ( 21 BROADCAST_QUEUE_SIZE = 4096 22 DEADLOCK_TICKER = 15 * time.Second // check every 15 seconds 23 DEADLOCK_WARN = (BROADCAST_QUEUE_SIZE * 99) / 100 // number of buffered messages before printing stack trace 24 ) 25 26 type WebConnActivityMessage struct { 27 UserId string 28 SessionToken string 29 ActivityAt int64 30 } 31 32 type Hub struct { 33 // connectionCount should be kept first. 34 // See https://github.com/mattermost/mattermost-server/pull/7281 35 connectionCount int64 36 app *App 37 connectionIndex int 38 register chan *WebConn 39 unregister chan *WebConn 40 broadcast chan *model.WebSocketEvent 41 stop chan struct{} 42 didStop chan struct{} 43 invalidateUser chan string 44 activity chan *WebConnActivityMessage 45 ExplicitStop bool 46 goroutineId int 47 } 48 49 func (a *App) NewWebHub() *Hub { 50 return &Hub{ 51 app: a, 52 register: make(chan *WebConn, 1), 53 unregister: make(chan *WebConn, 1), 54 broadcast: make(chan *model.WebSocketEvent, BROADCAST_QUEUE_SIZE), 55 stop: make(chan struct{}), 56 didStop: make(chan struct{}), 57 invalidateUser: make(chan string), 58 activity: make(chan *WebConnActivityMessage), 59 ExplicitStop: false, 60 } 61 } 62 63 func (a *App) TotalWebsocketConnections() int { 64 count := int64(0) 65 for _, hub := range a.Srv.Hubs { 66 count = count + atomic.LoadInt64(&hub.connectionCount) 67 } 68 69 return int(count) 70 } 71 72 func (a *App) HubStart() { 73 // Total number of hubs is twice the number of CPUs. 74 numberOfHubs := runtime.NumCPU() * 2 75 mlog.Info(fmt.Sprintf("Starting %v websocket hubs", numberOfHubs)) 76 77 a.Srv.Hubs = make([]*Hub, numberOfHubs) 78 a.Srv.HubsStopCheckingForDeadlock = make(chan bool, 1) 79 80 for i := 0; i < len(a.Srv.Hubs); i++ { 81 a.Srv.Hubs[i] = a.NewWebHub() 82 a.Srv.Hubs[i].connectionIndex = i 83 a.Srv.Hubs[i].Start() 84 } 85 86 go func() { 87 ticker := time.NewTicker(DEADLOCK_TICKER) 88 89 defer func() { 90 ticker.Stop() 91 }() 92 93 for { 94 select { 95 case <-ticker.C: 96 for _, hub := range a.Srv.Hubs { 97 if len(hub.broadcast) >= DEADLOCK_WARN { 98 mlog.Error(fmt.Sprintf("Hub processing might be deadlock on hub %v goroutine %v with %v events in the buffer", hub.connectionIndex, hub.goroutineId, len(hub.broadcast))) 99 buf := make([]byte, 1<<16) 100 runtime.Stack(buf, true) 101 output := fmt.Sprintf("%s", buf) 102 splits := strings.Split(output, "goroutine ") 103 104 for _, part := range splits { 105 if strings.Contains(part, fmt.Sprintf("%v", hub.goroutineId)) { 106 mlog.Error(fmt.Sprintf("Trace for possible deadlock goroutine %v", part)) 107 } 108 } 109 } 110 } 111 112 case <-a.Srv.HubsStopCheckingForDeadlock: 113 return 114 } 115 } 116 }() 117 } 118 119 func (a *App) HubStop() { 120 mlog.Info("stopping websocket hub connections") 121 122 select { 123 case a.Srv.HubsStopCheckingForDeadlock <- true: 124 default: 125 mlog.Warn("We appear to have already sent the stop checking for deadlocks command") 126 } 127 128 for _, hub := range a.Srv.Hubs { 129 hub.Stop() 130 } 131 132 a.Srv.Hubs = []*Hub{} 133 } 134 135 func (a *App) GetHubForUserId(userId string) *Hub { 136 if len(a.Srv.Hubs) == 0 { 137 return nil 138 } 139 140 hash := fnv.New32a() 141 hash.Write([]byte(userId)) 142 index := hash.Sum32() % uint32(len(a.Srv.Hubs)) 143 return a.Srv.Hubs[index] 144 } 145 146 func (a *App) HubRegister(webConn *WebConn) { 147 hub := a.GetHubForUserId(webConn.UserId) 148 if hub != nil { 149 hub.Register(webConn) 150 } 151 } 152 153 func (a *App) HubUnregister(webConn *WebConn) { 154 hub := a.GetHubForUserId(webConn.UserId) 155 if hub != nil { 156 hub.Unregister(webConn) 157 } 158 } 159 160 func (a *App) Publish(message *model.WebSocketEvent) { 161 if metrics := a.Metrics; metrics != nil { 162 metrics.IncrementWebsocketEvent(message.Event) 163 } 164 165 a.PublishSkipClusterSend(message) 166 167 if a.Cluster != nil { 168 cm := &model.ClusterMessage{ 169 Event: model.CLUSTER_EVENT_PUBLISH, 170 SendType: model.CLUSTER_SEND_BEST_EFFORT, 171 Data: message.ToJson(), 172 } 173 174 if message.Event == model.WEBSOCKET_EVENT_POSTED || 175 message.Event == model.WEBSOCKET_EVENT_POST_EDITED || 176 message.Event == model.WEBSOCKET_EVENT_DIRECT_ADDED || 177 message.Event == model.WEBSOCKET_EVENT_GROUP_ADDED || 178 message.Event == model.WEBSOCKET_EVENT_ADDED_TO_TEAM { 179 cm.SendType = model.CLUSTER_SEND_RELIABLE 180 } 181 182 a.Cluster.SendClusterMessage(cm) 183 } 184 } 185 186 func (a *App) PublishSkipClusterSend(message *model.WebSocketEvent) { 187 if message.Broadcast.UserId != "" { 188 hub := a.GetHubForUserId(message.Broadcast.UserId) 189 if hub != nil { 190 hub.Broadcast(message) 191 } 192 } else { 193 for _, hub := range a.Srv.Hubs { 194 hub.Broadcast(message) 195 } 196 } 197 } 198 199 func (a *App) InvalidateCacheForChannel(channel *model.Channel) { 200 a.InvalidateCacheForChannelSkipClusterSend(channel.Id) 201 a.InvalidateCacheForChannelByNameSkipClusterSend(channel.TeamId, channel.Name) 202 203 if a.Cluster != nil { 204 msg := &model.ClusterMessage{ 205 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL, 206 SendType: model.CLUSTER_SEND_BEST_EFFORT, 207 Data: channel.Id, 208 } 209 210 a.Cluster.SendClusterMessage(msg) 211 212 nameMsg := &model.ClusterMessage{ 213 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_BY_NAME, 214 SendType: model.CLUSTER_SEND_BEST_EFFORT, 215 Props: make(map[string]string), 216 } 217 218 nameMsg.Props["name"] = channel.Name 219 if channel.TeamId == "" { 220 nameMsg.Props["id"] = "dm" 221 } else { 222 nameMsg.Props["id"] = channel.TeamId 223 } 224 225 a.Cluster.SendClusterMessage(nameMsg) 226 } 227 } 228 229 func (a *App) InvalidateCacheForChannelSkipClusterSend(channelId string) { 230 a.Srv.Store.Channel().InvalidateChannel(channelId) 231 } 232 233 func (a *App) InvalidateCacheForChannelMembers(channelId string) { 234 a.InvalidateCacheForChannelMembersSkipClusterSend(channelId) 235 236 if a.Cluster != nil { 237 msg := &model.ClusterMessage{ 238 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS, 239 SendType: model.CLUSTER_SEND_BEST_EFFORT, 240 Data: channelId, 241 } 242 a.Cluster.SendClusterMessage(msg) 243 } 244 } 245 246 func (a *App) InvalidateCacheForChannelMembersSkipClusterSend(channelId string) { 247 a.Srv.Store.User().InvalidateProfilesInChannelCache(channelId) 248 a.Srv.Store.Channel().InvalidateMemberCount(channelId) 249 } 250 251 func (a *App) InvalidateCacheForChannelMembersNotifyProps(channelId string) { 252 a.InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId) 253 254 if a.Cluster != nil { 255 msg := &model.ClusterMessage{ 256 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS_NOTIFY_PROPS, 257 SendType: model.CLUSTER_SEND_BEST_EFFORT, 258 Data: channelId, 259 } 260 a.Cluster.SendClusterMessage(msg) 261 } 262 } 263 264 func (a *App) InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId string) { 265 a.Srv.Store.Channel().InvalidateCacheForChannelMembersNotifyProps(channelId) 266 } 267 268 func (a *App) InvalidateCacheForChannelByNameSkipClusterSend(teamId, name string) { 269 if teamId == "" { 270 teamId = "dm" 271 } 272 273 a.Srv.Store.Channel().InvalidateChannelByName(teamId, name) 274 } 275 276 func (a *App) InvalidateCacheForChannelPosts(channelId string) { 277 a.InvalidateCacheForChannelPostsSkipClusterSend(channelId) 278 279 if a.Cluster != nil { 280 msg := &model.ClusterMessage{ 281 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_POSTS, 282 SendType: model.CLUSTER_SEND_BEST_EFFORT, 283 Data: channelId, 284 } 285 a.Cluster.SendClusterMessage(msg) 286 } 287 } 288 289 func (a *App) InvalidateCacheForChannelPostsSkipClusterSend(channelId string) { 290 a.Srv.Store.Post().InvalidateLastPostTimeCache(channelId) 291 } 292 293 func (a *App) InvalidateCacheForUser(userId string) { 294 a.InvalidateCacheForUserSkipClusterSend(userId) 295 296 if a.Cluster != nil { 297 msg := &model.ClusterMessage{ 298 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_USER, 299 SendType: model.CLUSTER_SEND_BEST_EFFORT, 300 Data: userId, 301 } 302 a.Cluster.SendClusterMessage(msg) 303 } 304 } 305 306 func (a *App) InvalidateCacheForUserSkipClusterSend(userId string) { 307 a.Srv.Store.Channel().InvalidateAllChannelMembersForUser(userId) 308 a.Srv.Store.User().InvalidateProfilesInChannelCacheByUser(userId) 309 a.Srv.Store.User().InvalidatProfileCacheForUser(userId) 310 311 hub := a.GetHubForUserId(userId) 312 if hub != nil { 313 hub.InvalidateUser(userId) 314 } 315 } 316 317 func (a *App) InvalidateCacheForWebhook(webhookId string) { 318 a.InvalidateCacheForWebhookSkipClusterSend(webhookId) 319 320 if a.Cluster != nil { 321 msg := &model.ClusterMessage{ 322 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_WEBHOOK, 323 SendType: model.CLUSTER_SEND_BEST_EFFORT, 324 Data: webhookId, 325 } 326 a.Cluster.SendClusterMessage(msg) 327 } 328 } 329 330 func (a *App) InvalidateCacheForWebhookSkipClusterSend(webhookId string) { 331 a.Srv.Store.Webhook().InvalidateWebhookCache(webhookId) 332 } 333 334 func (a *App) InvalidateWebConnSessionCacheForUser(userId string) { 335 hub := a.GetHubForUserId(userId) 336 if hub != nil { 337 hub.InvalidateUser(userId) 338 } 339 } 340 341 func (a *App) UpdateWebConnUserActivity(session model.Session, activityAt int64) { 342 hub := a.GetHubForUserId(session.UserId) 343 if hub != nil { 344 hub.UpdateActivity(session.UserId, session.Token, activityAt) 345 } 346 } 347 348 func (h *Hub) Register(webConn *WebConn) { 349 select { 350 case h.register <- webConn: 351 case <-h.didStop: 352 } 353 354 if webConn.IsAuthenticated() { 355 webConn.SendHello() 356 } 357 } 358 359 func (h *Hub) Unregister(webConn *WebConn) { 360 select { 361 case h.unregister <- webConn: 362 case <-h.stop: 363 } 364 } 365 366 func (h *Hub) Broadcast(message *model.WebSocketEvent) { 367 if h != nil && h.broadcast != nil && message != nil { 368 select { 369 case h.broadcast <- message: 370 case <-h.didStop: 371 } 372 } 373 } 374 375 func (h *Hub) InvalidateUser(userId string) { 376 select { 377 case h.invalidateUser <- userId: 378 case <-h.didStop: 379 } 380 } 381 382 func (h *Hub) UpdateActivity(userId, sessionToken string, activityAt int64) { 383 select { 384 case h.activity <- &WebConnActivityMessage{UserId: userId, SessionToken: sessionToken, ActivityAt: activityAt}: 385 case <-h.didStop: 386 } 387 } 388 389 func getGoroutineId() int { 390 var buf [64]byte 391 n := runtime.Stack(buf[:], false) 392 idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0] 393 id, err := strconv.Atoi(idField) 394 if err != nil { 395 id = -1 396 } 397 return id 398 } 399 400 func (h *Hub) Stop() { 401 close(h.stop) 402 <-h.didStop 403 } 404 405 func (h *Hub) Start() { 406 var doStart func() 407 var doRecoverableStart func() 408 var doRecover func() 409 410 doStart = func() { 411 h.goroutineId = getGoroutineId() 412 mlog.Debug(fmt.Sprintf("Hub for index %v is starting with goroutine %v", h.connectionIndex, h.goroutineId)) 413 414 connections := newHubConnectionIndex() 415 416 for { 417 select { 418 case webCon := <-h.register: 419 connections.Add(webCon) 420 atomic.StoreInt64(&h.connectionCount, int64(len(connections.All()))) 421 case webCon := <-h.unregister: 422 connections.Remove(webCon) 423 atomic.StoreInt64(&h.connectionCount, int64(len(connections.All()))) 424 425 if len(webCon.UserId) == 0 { 426 continue 427 } 428 429 conns := connections.ForUser(webCon.UserId) 430 if len(conns) == 0 { 431 h.app.Srv.Go(func() { 432 h.app.SetStatusOffline(webCon.UserId, false) 433 }) 434 } else { 435 var latestActivity int64 = 0 436 for _, conn := range conns { 437 if conn.LastUserActivityAt > latestActivity { 438 latestActivity = conn.LastUserActivityAt 439 } 440 } 441 if h.app.IsUserAway(latestActivity) { 442 h.app.Srv.Go(func() { 443 h.app.SetStatusLastActivityAt(webCon.UserId, latestActivity) 444 }) 445 } 446 } 447 case userId := <-h.invalidateUser: 448 for _, webCon := range connections.ForUser(userId) { 449 webCon.InvalidateCache() 450 } 451 case activity := <-h.activity: 452 for _, webCon := range connections.ForUser(activity.UserId) { 453 if webCon.GetSessionToken() == activity.SessionToken { 454 webCon.LastUserActivityAt = activity.ActivityAt 455 } 456 } 457 case msg := <-h.broadcast: 458 candidates := connections.All() 459 if msg.Broadcast.UserId != "" { 460 candidates = connections.ForUser(msg.Broadcast.UserId) 461 } 462 msg.PrecomputeJSON() 463 for _, webCon := range candidates { 464 if webCon.ShouldSendEvent(msg) { 465 select { 466 case webCon.Send <- msg: 467 default: 468 mlog.Error(fmt.Sprintf("webhub.broadcast: cannot send, closing websocket for userId=%v", webCon.UserId)) 469 close(webCon.Send) 470 connections.Remove(webCon) 471 } 472 } 473 } 474 case <-h.stop: 475 userIds := make(map[string]bool) 476 477 for _, webCon := range connections.All() { 478 userIds[webCon.UserId] = true 479 webCon.Close() 480 } 481 482 for userId := range userIds { 483 h.app.SetStatusOffline(userId, false) 484 } 485 486 h.ExplicitStop = true 487 close(h.didStop) 488 489 return 490 } 491 } 492 } 493 494 doRecoverableStart = func() { 495 defer doRecover() 496 doStart() 497 } 498 499 doRecover = func() { 500 if !h.ExplicitStop { 501 if r := recover(); r != nil { 502 mlog.Error(fmt.Sprintf("Recovering from Hub panic. Panic was: %v", r)) 503 } else { 504 mlog.Error("Webhub stopped unexpectedly. Recovering.") 505 } 506 507 mlog.Error(string(debug.Stack())) 508 509 go doRecoverableStart() 510 } 511 } 512 513 go doRecoverableStart() 514 } 515 516 type hubConnectionIndexIndexes struct { 517 connections int 518 connectionsByUserId int 519 } 520 521 // hubConnectionIndex provides fast addition, removal, and iteration of web connections. 522 type hubConnectionIndex struct { 523 connections []*WebConn 524 connectionsByUserId map[string][]*WebConn 525 connectionIndexes map[*WebConn]*hubConnectionIndexIndexes 526 } 527 528 func newHubConnectionIndex() *hubConnectionIndex { 529 return &hubConnectionIndex{ 530 connections: make([]*WebConn, 0, model.SESSION_CACHE_SIZE), 531 connectionsByUserId: make(map[string][]*WebConn), 532 connectionIndexes: make(map[*WebConn]*hubConnectionIndexIndexes), 533 } 534 } 535 536 func (i *hubConnectionIndex) Add(wc *WebConn) { 537 i.connections = append(i.connections, wc) 538 i.connectionsByUserId[wc.UserId] = append(i.connectionsByUserId[wc.UserId], wc) 539 i.connectionIndexes[wc] = &hubConnectionIndexIndexes{ 540 connections: len(i.connections) - 1, 541 connectionsByUserId: len(i.connectionsByUserId[wc.UserId]) - 1, 542 } 543 } 544 545 func (i *hubConnectionIndex) Remove(wc *WebConn) { 546 indexes, ok := i.connectionIndexes[wc] 547 if !ok { 548 return 549 } 550 551 last := i.connections[len(i.connections)-1] 552 i.connections[indexes.connections] = last 553 i.connections = i.connections[:len(i.connections)-1] 554 i.connectionIndexes[last].connections = indexes.connections 555 556 userConnections := i.connectionsByUserId[wc.UserId] 557 last = userConnections[len(userConnections)-1] 558 userConnections[indexes.connectionsByUserId] = last 559 i.connectionsByUserId[wc.UserId] = userConnections[:len(userConnections)-1] 560 i.connectionIndexes[last].connectionsByUserId = indexes.connectionsByUserId 561 562 delete(i.connectionIndexes, wc) 563 } 564 565 func (i *hubConnectionIndex) ForUser(id string) []*WebConn { 566 return i.connectionsByUserId[id] 567 } 568 569 func (i *hubConnectionIndex) All() []*WebConn { 570 return i.connections 571 }