github.com/spline-fu/mattermost-server@v4.10.10+incompatible/app/web_hub.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "fmt" 8 "hash/fnv" 9 "runtime" 10 "runtime/debug" 11 "strconv" 12 "strings" 13 "sync/atomic" 14 "time" 15 16 "github.com/mattermost/mattermost-server/mlog" 17 "github.com/mattermost/mattermost-server/model" 18 ) 19 20 const ( 21 BROADCAST_QUEUE_SIZE = 4096 22 DEADLOCK_TICKER = 15 * time.Second // check every 15 seconds 23 DEADLOCK_WARN = (BROADCAST_QUEUE_SIZE * 99) / 100 // number of buffered messages before printing stack trace 24 ) 25 26 type Hub struct { 27 // connectionCount should be kept first. 28 // See https://github.com/mattermost/mattermost-server/pull/7281 29 connectionCount int64 30 app *App 31 connectionIndex int 32 register chan *WebConn 33 unregister chan *WebConn 34 broadcast chan *model.WebSocketEvent 35 stop chan struct{} 36 didStop chan struct{} 37 invalidateUser chan string 38 ExplicitStop bool 39 goroutineId int 40 } 41 42 func (a *App) NewWebHub() *Hub { 43 return &Hub{ 44 app: a, 45 register: make(chan *WebConn, 1), 46 unregister: make(chan *WebConn, 1), 47 broadcast: make(chan *model.WebSocketEvent, BROADCAST_QUEUE_SIZE), 48 stop: make(chan struct{}), 49 didStop: make(chan struct{}), 50 invalidateUser: make(chan string), 51 ExplicitStop: false, 52 } 53 } 54 55 func (a *App) TotalWebsocketConnections() int { 56 count := int64(0) 57 for _, hub := range a.Hubs { 58 count = count + atomic.LoadInt64(&hub.connectionCount) 59 } 60 61 return int(count) 62 } 63 64 func (a *App) HubStart() { 65 // Total number of hubs is twice the number of CPUs. 66 numberOfHubs := runtime.NumCPU() * 2 67 mlog.Info(fmt.Sprintf("Starting %v websocket hubs", numberOfHubs)) 68 69 a.Hubs = make([]*Hub, numberOfHubs) 70 a.HubsStopCheckingForDeadlock = make(chan bool, 1) 71 72 for i := 0; i < len(a.Hubs); i++ { 73 a.Hubs[i] = a.NewWebHub() 74 a.Hubs[i].connectionIndex = i 75 a.Hubs[i].Start() 76 } 77 78 go func() { 79 ticker := time.NewTicker(DEADLOCK_TICKER) 80 81 defer func() { 82 ticker.Stop() 83 }() 84 85 for { 86 select { 87 case <-ticker.C: 88 for _, hub := range a.Hubs { 89 if len(hub.broadcast) >= DEADLOCK_WARN { 90 mlog.Error(fmt.Sprintf("Hub processing might be deadlock on hub %v goroutine %v with %v events in the buffer", hub.connectionIndex, hub.goroutineId, len(hub.broadcast))) 91 buf := make([]byte, 1<<16) 92 runtime.Stack(buf, true) 93 output := fmt.Sprintf("%s", buf) 94 splits := strings.Split(output, "goroutine ") 95 96 for _, part := range splits { 97 if strings.Contains(part, fmt.Sprintf("%v", hub.goroutineId)) { 98 mlog.Error(fmt.Sprintf("Trace for possible deadlock goroutine %v", part)) 99 } 100 } 101 } 102 } 103 104 case <-a.HubsStopCheckingForDeadlock: 105 return 106 } 107 } 108 }() 109 } 110 111 func (a *App) HubStop() { 112 mlog.Info("stopping websocket hub connections") 113 114 select { 115 case a.HubsStopCheckingForDeadlock <- true: 116 default: 117 mlog.Warn("We appear to have already sent the stop checking for deadlocks command") 118 } 119 120 for _, hub := range a.Hubs { 121 hub.Stop() 122 } 123 124 a.Hubs = []*Hub{} 125 } 126 127 func (a *App) GetHubForUserId(userId string) *Hub { 128 if len(a.Hubs) == 0 { 129 return nil 130 } 131 132 hash := fnv.New32a() 133 hash.Write([]byte(userId)) 134 index := hash.Sum32() % uint32(len(a.Hubs)) 135 return a.Hubs[index] 136 } 137 138 func (a *App) HubRegister(webConn *WebConn) { 139 hub := a.GetHubForUserId(webConn.UserId) 140 if hub != nil { 141 hub.Register(webConn) 142 } 143 } 144 145 func (a *App) HubUnregister(webConn *WebConn) { 146 hub := a.GetHubForUserId(webConn.UserId) 147 if hub != nil { 148 hub.Unregister(webConn) 149 } 150 } 151 152 func (a *App) Publish(message *model.WebSocketEvent) { 153 if metrics := a.Metrics; metrics != nil { 154 metrics.IncrementWebsocketEvent(message.Event) 155 } 156 157 a.PublishSkipClusterSend(message) 158 159 if a.Cluster != nil { 160 cm := &model.ClusterMessage{ 161 Event: model.CLUSTER_EVENT_PUBLISH, 162 SendType: model.CLUSTER_SEND_BEST_EFFORT, 163 Data: message.ToJson(), 164 } 165 166 if message.Event == model.WEBSOCKET_EVENT_POSTED || 167 message.Event == model.WEBSOCKET_EVENT_POST_EDITED || 168 message.Event == model.WEBSOCKET_EVENT_DIRECT_ADDED || 169 message.Event == model.WEBSOCKET_EVENT_GROUP_ADDED || 170 message.Event == model.WEBSOCKET_EVENT_ADDED_TO_TEAM { 171 cm.SendType = model.CLUSTER_SEND_RELIABLE 172 } 173 174 a.Cluster.SendClusterMessage(cm) 175 } 176 } 177 178 func (a *App) PublishSkipClusterSend(message *model.WebSocketEvent) { 179 if message.Broadcast.UserId != "" { 180 hub := a.GetHubForUserId(message.Broadcast.UserId) 181 if hub != nil { 182 hub.Broadcast(message) 183 } 184 } else { 185 for _, hub := range a.Hubs { 186 hub.Broadcast(message) 187 } 188 } 189 } 190 191 func (a *App) InvalidateCacheForChannel(channel *model.Channel) { 192 a.InvalidateCacheForChannelSkipClusterSend(channel.Id) 193 a.InvalidateCacheForChannelByNameSkipClusterSend(channel.TeamId, channel.Name) 194 195 if a.Cluster != nil { 196 msg := &model.ClusterMessage{ 197 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL, 198 SendType: model.CLUSTER_SEND_BEST_EFFORT, 199 Data: channel.Id, 200 } 201 202 a.Cluster.SendClusterMessage(msg) 203 204 nameMsg := &model.ClusterMessage{ 205 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_BY_NAME, 206 SendType: model.CLUSTER_SEND_BEST_EFFORT, 207 Props: make(map[string]string), 208 } 209 210 nameMsg.Props["name"] = channel.Name 211 if channel.TeamId == "" { 212 nameMsg.Props["id"] = "dm" 213 } else { 214 nameMsg.Props["id"] = channel.TeamId 215 } 216 217 a.Cluster.SendClusterMessage(nameMsg) 218 } 219 } 220 221 func (a *App) InvalidateCacheForChannelSkipClusterSend(channelId string) { 222 a.Srv.Store.Channel().InvalidateChannel(channelId) 223 } 224 225 func (a *App) InvalidateCacheForChannelMembers(channelId string) { 226 a.InvalidateCacheForChannelMembersSkipClusterSend(channelId) 227 228 if a.Cluster != nil { 229 msg := &model.ClusterMessage{ 230 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS, 231 SendType: model.CLUSTER_SEND_BEST_EFFORT, 232 Data: channelId, 233 } 234 a.Cluster.SendClusterMessage(msg) 235 } 236 } 237 238 func (a *App) InvalidateCacheForChannelMembersSkipClusterSend(channelId string) { 239 a.Srv.Store.User().InvalidateProfilesInChannelCache(channelId) 240 a.Srv.Store.Channel().InvalidateMemberCount(channelId) 241 } 242 243 func (a *App) InvalidateCacheForChannelMembersNotifyProps(channelId string) { 244 a.InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId) 245 246 if a.Cluster != nil { 247 msg := &model.ClusterMessage{ 248 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS_NOTIFY_PROPS, 249 SendType: model.CLUSTER_SEND_BEST_EFFORT, 250 Data: channelId, 251 } 252 a.Cluster.SendClusterMessage(msg) 253 } 254 } 255 256 func (a *App) InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId string) { 257 a.Srv.Store.Channel().InvalidateCacheForChannelMembersNotifyProps(channelId) 258 } 259 260 func (a *App) InvalidateCacheForChannelByNameSkipClusterSend(teamId, name string) { 261 if teamId == "" { 262 teamId = "dm" 263 } 264 265 a.Srv.Store.Channel().InvalidateChannelByName(teamId, name) 266 } 267 268 func (a *App) InvalidateCacheForChannelPosts(channelId string) { 269 a.InvalidateCacheForChannelPostsSkipClusterSend(channelId) 270 271 if a.Cluster != nil { 272 msg := &model.ClusterMessage{ 273 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_POSTS, 274 SendType: model.CLUSTER_SEND_BEST_EFFORT, 275 Data: channelId, 276 } 277 a.Cluster.SendClusterMessage(msg) 278 } 279 } 280 281 func (a *App) InvalidateCacheForChannelPostsSkipClusterSend(channelId string) { 282 a.Srv.Store.Post().InvalidateLastPostTimeCache(channelId) 283 } 284 285 func (a *App) InvalidateCacheForUser(userId string) { 286 a.InvalidateCacheForUserSkipClusterSend(userId) 287 288 if a.Cluster != nil { 289 msg := &model.ClusterMessage{ 290 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_USER, 291 SendType: model.CLUSTER_SEND_BEST_EFFORT, 292 Data: userId, 293 } 294 a.Cluster.SendClusterMessage(msg) 295 } 296 } 297 298 func (a *App) InvalidateCacheForUserSkipClusterSend(userId string) { 299 a.Srv.Store.Channel().InvalidateAllChannelMembersForUser(userId) 300 a.Srv.Store.User().InvalidateProfilesInChannelCacheByUser(userId) 301 a.Srv.Store.User().InvalidatProfileCacheForUser(userId) 302 303 hub := a.GetHubForUserId(userId) 304 if hub != nil { 305 hub.InvalidateUser(userId) 306 } 307 } 308 309 func (a *App) InvalidateCacheForWebhook(webhookId string) { 310 a.InvalidateCacheForWebhookSkipClusterSend(webhookId) 311 312 if a.Cluster != nil { 313 msg := &model.ClusterMessage{ 314 Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_WEBHOOK, 315 SendType: model.CLUSTER_SEND_BEST_EFFORT, 316 Data: webhookId, 317 } 318 a.Cluster.SendClusterMessage(msg) 319 } 320 } 321 322 func (a *App) InvalidateCacheForWebhookSkipClusterSend(webhookId string) { 323 a.Srv.Store.Webhook().InvalidateWebhookCache(webhookId) 324 } 325 326 func (a *App) InvalidateWebConnSessionCacheForUser(userId string) { 327 hub := a.GetHubForUserId(userId) 328 if hub != nil { 329 hub.InvalidateUser(userId) 330 } 331 } 332 333 func (h *Hub) Register(webConn *WebConn) { 334 h.register <- webConn 335 336 if webConn.IsAuthenticated() { 337 webConn.SendHello() 338 } 339 } 340 341 func (h *Hub) Unregister(webConn *WebConn) { 342 select { 343 case h.unregister <- webConn: 344 case <-h.stop: 345 } 346 } 347 348 func (h *Hub) Broadcast(message *model.WebSocketEvent) { 349 if h != nil && h.broadcast != nil && message != nil { 350 h.broadcast <- message 351 } 352 } 353 354 func (h *Hub) InvalidateUser(userId string) { 355 h.invalidateUser <- userId 356 } 357 358 func getGoroutineId() int { 359 var buf [64]byte 360 n := runtime.Stack(buf[:], false) 361 idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0] 362 id, err := strconv.Atoi(idField) 363 if err != nil { 364 id = -1 365 } 366 return id 367 } 368 369 func (h *Hub) Stop() { 370 close(h.stop) 371 <-h.didStop 372 } 373 374 func (h *Hub) Start() { 375 var doStart func() 376 var doRecoverableStart func() 377 var doRecover func() 378 379 doStart = func() { 380 h.goroutineId = getGoroutineId() 381 mlog.Debug(fmt.Sprintf("Hub for index %v is starting with goroutine %v", h.connectionIndex, h.goroutineId)) 382 383 connections := newHubConnectionIndex() 384 385 for { 386 select { 387 case webCon := <-h.register: 388 connections.Add(webCon) 389 atomic.StoreInt64(&h.connectionCount, int64(len(connections.All()))) 390 case webCon := <-h.unregister: 391 connections.Remove(webCon) 392 atomic.StoreInt64(&h.connectionCount, int64(len(connections.All()))) 393 394 if len(webCon.UserId) == 0 { 395 continue 396 } 397 398 if len(connections.ForUser(webCon.UserId)) == 0 { 399 h.app.Go(func() { 400 h.app.SetStatusOffline(webCon.UserId, false) 401 }) 402 } 403 case userId := <-h.invalidateUser: 404 for _, webCon := range connections.ForUser(userId) { 405 webCon.InvalidateCache() 406 } 407 case msg := <-h.broadcast: 408 candidates := connections.All() 409 if msg.Broadcast.UserId != "" { 410 candidates = connections.ForUser(msg.Broadcast.UserId) 411 } 412 msg.PrecomputeJSON() 413 for _, webCon := range candidates { 414 if webCon.ShouldSendEvent(msg) { 415 select { 416 case webCon.Send <- msg: 417 default: 418 mlog.Error(fmt.Sprintf("webhub.broadcast: cannot send, closing websocket for userId=%v", webCon.UserId)) 419 close(webCon.Send) 420 connections.Remove(webCon) 421 } 422 } 423 } 424 case <-h.stop: 425 userIds := make(map[string]bool) 426 427 for _, webCon := range connections.All() { 428 userIds[webCon.UserId] = true 429 webCon.Close() 430 } 431 432 for userId := range userIds { 433 h.app.SetStatusOffline(userId, false) 434 } 435 436 h.ExplicitStop = true 437 close(h.didStop) 438 439 return 440 } 441 } 442 } 443 444 doRecoverableStart = func() { 445 defer doRecover() 446 doStart() 447 } 448 449 doRecover = func() { 450 if !h.ExplicitStop { 451 if r := recover(); r != nil { 452 mlog.Error(fmt.Sprintf("Recovering from Hub panic. Panic was: %v", r)) 453 } else { 454 mlog.Error("Webhub stopped unexpectedly. Recovering.") 455 } 456 457 mlog.Error(string(debug.Stack())) 458 459 go doRecoverableStart() 460 } 461 } 462 463 go doRecoverableStart() 464 } 465 466 type hubConnectionIndexIndexes struct { 467 connections int 468 connectionsByUserId int 469 } 470 471 // hubConnectionIndex provides fast addition, removal, and iteration of web connections. 472 type hubConnectionIndex struct { 473 connections []*WebConn 474 connectionsByUserId map[string][]*WebConn 475 connectionIndexes map[*WebConn]*hubConnectionIndexIndexes 476 } 477 478 func newHubConnectionIndex() *hubConnectionIndex { 479 return &hubConnectionIndex{ 480 connections: make([]*WebConn, 0, model.SESSION_CACHE_SIZE), 481 connectionsByUserId: make(map[string][]*WebConn), 482 connectionIndexes: make(map[*WebConn]*hubConnectionIndexIndexes), 483 } 484 } 485 486 func (i *hubConnectionIndex) Add(wc *WebConn) { 487 i.connections = append(i.connections, wc) 488 i.connectionsByUserId[wc.UserId] = append(i.connectionsByUserId[wc.UserId], wc) 489 i.connectionIndexes[wc] = &hubConnectionIndexIndexes{ 490 connections: len(i.connections) - 1, 491 connectionsByUserId: len(i.connectionsByUserId[wc.UserId]) - 1, 492 } 493 } 494 495 func (i *hubConnectionIndex) Remove(wc *WebConn) { 496 indexes, ok := i.connectionIndexes[wc] 497 if !ok { 498 return 499 } 500 501 last := i.connections[len(i.connections)-1] 502 i.connections[indexes.connections] = last 503 i.connections = i.connections[:len(i.connections)-1] 504 i.connectionIndexes[last].connections = indexes.connections 505 506 userConnections := i.connectionsByUserId[wc.UserId] 507 last = userConnections[len(userConnections)-1] 508 userConnections[indexes.connectionsByUserId] = last 509 i.connectionsByUserId[wc.UserId] = userConnections[:len(userConnections)-1] 510 i.connectionIndexes[last].connectionsByUserId = indexes.connectionsByUserId 511 512 delete(i.connectionIndexes, wc) 513 } 514 515 func (i *hubConnectionIndex) ForUser(id string) []*WebConn { 516 return i.connectionsByUserId[id] 517 } 518 519 func (i *hubConnectionIndex) All() []*WebConn { 520 return i.connections 521 }