github.com/spline-fu/mattermost-server@v4.10.10+incompatible/app/web_hub.go (about)

     1  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
     2  // See License.txt for license information.
     3  
     4  package app
     5  
     6  import (
     7  	"fmt"
     8  	"hash/fnv"
     9  	"runtime"
    10  	"runtime/debug"
    11  	"strconv"
    12  	"strings"
    13  	"sync/atomic"
    14  	"time"
    15  
    16  	"github.com/mattermost/mattermost-server/mlog"
    17  	"github.com/mattermost/mattermost-server/model"
    18  )
    19  
    20  const (
    21  	BROADCAST_QUEUE_SIZE = 4096
    22  	DEADLOCK_TICKER      = 15 * time.Second                  // check every 15 seconds
    23  	DEADLOCK_WARN        = (BROADCAST_QUEUE_SIZE * 99) / 100 // number of buffered messages before printing stack trace
    24  )
    25  
    26  type Hub struct {
    27  	// connectionCount should be kept first.
    28  	// See https://github.com/mattermost/mattermost-server/pull/7281
    29  	connectionCount int64
    30  	app             *App
    31  	connectionIndex int
    32  	register        chan *WebConn
    33  	unregister      chan *WebConn
    34  	broadcast       chan *model.WebSocketEvent
    35  	stop            chan struct{}
    36  	didStop         chan struct{}
    37  	invalidateUser  chan string
    38  	ExplicitStop    bool
    39  	goroutineId     int
    40  }
    41  
    42  func (a *App) NewWebHub() *Hub {
    43  	return &Hub{
    44  		app:            a,
    45  		register:       make(chan *WebConn, 1),
    46  		unregister:     make(chan *WebConn, 1),
    47  		broadcast:      make(chan *model.WebSocketEvent, BROADCAST_QUEUE_SIZE),
    48  		stop:           make(chan struct{}),
    49  		didStop:        make(chan struct{}),
    50  		invalidateUser: make(chan string),
    51  		ExplicitStop:   false,
    52  	}
    53  }
    54  
    55  func (a *App) TotalWebsocketConnections() int {
    56  	count := int64(0)
    57  	for _, hub := range a.Hubs {
    58  		count = count + atomic.LoadInt64(&hub.connectionCount)
    59  	}
    60  
    61  	return int(count)
    62  }
    63  
    64  func (a *App) HubStart() {
    65  	// Total number of hubs is twice the number of CPUs.
    66  	numberOfHubs := runtime.NumCPU() * 2
    67  	mlog.Info(fmt.Sprintf("Starting %v websocket hubs", numberOfHubs))
    68  
    69  	a.Hubs = make([]*Hub, numberOfHubs)
    70  	a.HubsStopCheckingForDeadlock = make(chan bool, 1)
    71  
    72  	for i := 0; i < len(a.Hubs); i++ {
    73  		a.Hubs[i] = a.NewWebHub()
    74  		a.Hubs[i].connectionIndex = i
    75  		a.Hubs[i].Start()
    76  	}
    77  
    78  	go func() {
    79  		ticker := time.NewTicker(DEADLOCK_TICKER)
    80  
    81  		defer func() {
    82  			ticker.Stop()
    83  		}()
    84  
    85  		for {
    86  			select {
    87  			case <-ticker.C:
    88  				for _, hub := range a.Hubs {
    89  					if len(hub.broadcast) >= DEADLOCK_WARN {
    90  						mlog.Error(fmt.Sprintf("Hub processing might be deadlock on hub %v goroutine %v with %v events in the buffer", hub.connectionIndex, hub.goroutineId, len(hub.broadcast)))
    91  						buf := make([]byte, 1<<16)
    92  						runtime.Stack(buf, true)
    93  						output := fmt.Sprintf("%s", buf)
    94  						splits := strings.Split(output, "goroutine ")
    95  
    96  						for _, part := range splits {
    97  							if strings.Contains(part, fmt.Sprintf("%v", hub.goroutineId)) {
    98  								mlog.Error(fmt.Sprintf("Trace for possible deadlock goroutine %v", part))
    99  							}
   100  						}
   101  					}
   102  				}
   103  
   104  			case <-a.HubsStopCheckingForDeadlock:
   105  				return
   106  			}
   107  		}
   108  	}()
   109  }
   110  
   111  func (a *App) HubStop() {
   112  	mlog.Info("stopping websocket hub connections")
   113  
   114  	select {
   115  	case a.HubsStopCheckingForDeadlock <- true:
   116  	default:
   117  		mlog.Warn("We appear to have already sent the stop checking for deadlocks command")
   118  	}
   119  
   120  	for _, hub := range a.Hubs {
   121  		hub.Stop()
   122  	}
   123  
   124  	a.Hubs = []*Hub{}
   125  }
   126  
   127  func (a *App) GetHubForUserId(userId string) *Hub {
   128  	if len(a.Hubs) == 0 {
   129  		return nil
   130  	}
   131  
   132  	hash := fnv.New32a()
   133  	hash.Write([]byte(userId))
   134  	index := hash.Sum32() % uint32(len(a.Hubs))
   135  	return a.Hubs[index]
   136  }
   137  
   138  func (a *App) HubRegister(webConn *WebConn) {
   139  	hub := a.GetHubForUserId(webConn.UserId)
   140  	if hub != nil {
   141  		hub.Register(webConn)
   142  	}
   143  }
   144  
   145  func (a *App) HubUnregister(webConn *WebConn) {
   146  	hub := a.GetHubForUserId(webConn.UserId)
   147  	if hub != nil {
   148  		hub.Unregister(webConn)
   149  	}
   150  }
   151  
   152  func (a *App) Publish(message *model.WebSocketEvent) {
   153  	if metrics := a.Metrics; metrics != nil {
   154  		metrics.IncrementWebsocketEvent(message.Event)
   155  	}
   156  
   157  	a.PublishSkipClusterSend(message)
   158  
   159  	if a.Cluster != nil {
   160  		cm := &model.ClusterMessage{
   161  			Event:    model.CLUSTER_EVENT_PUBLISH,
   162  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   163  			Data:     message.ToJson(),
   164  		}
   165  
   166  		if message.Event == model.WEBSOCKET_EVENT_POSTED ||
   167  			message.Event == model.WEBSOCKET_EVENT_POST_EDITED ||
   168  			message.Event == model.WEBSOCKET_EVENT_DIRECT_ADDED ||
   169  			message.Event == model.WEBSOCKET_EVENT_GROUP_ADDED ||
   170  			message.Event == model.WEBSOCKET_EVENT_ADDED_TO_TEAM {
   171  			cm.SendType = model.CLUSTER_SEND_RELIABLE
   172  		}
   173  
   174  		a.Cluster.SendClusterMessage(cm)
   175  	}
   176  }
   177  
   178  func (a *App) PublishSkipClusterSend(message *model.WebSocketEvent) {
   179  	if message.Broadcast.UserId != "" {
   180  		hub := a.GetHubForUserId(message.Broadcast.UserId)
   181  		if hub != nil {
   182  			hub.Broadcast(message)
   183  		}
   184  	} else {
   185  		for _, hub := range a.Hubs {
   186  			hub.Broadcast(message)
   187  		}
   188  	}
   189  }
   190  
   191  func (a *App) InvalidateCacheForChannel(channel *model.Channel) {
   192  	a.InvalidateCacheForChannelSkipClusterSend(channel.Id)
   193  	a.InvalidateCacheForChannelByNameSkipClusterSend(channel.TeamId, channel.Name)
   194  
   195  	if a.Cluster != nil {
   196  		msg := &model.ClusterMessage{
   197  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL,
   198  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   199  			Data:     channel.Id,
   200  		}
   201  
   202  		a.Cluster.SendClusterMessage(msg)
   203  
   204  		nameMsg := &model.ClusterMessage{
   205  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_BY_NAME,
   206  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   207  			Props:    make(map[string]string),
   208  		}
   209  
   210  		nameMsg.Props["name"] = channel.Name
   211  		if channel.TeamId == "" {
   212  			nameMsg.Props["id"] = "dm"
   213  		} else {
   214  			nameMsg.Props["id"] = channel.TeamId
   215  		}
   216  
   217  		a.Cluster.SendClusterMessage(nameMsg)
   218  	}
   219  }
   220  
   221  func (a *App) InvalidateCacheForChannelSkipClusterSend(channelId string) {
   222  	a.Srv.Store.Channel().InvalidateChannel(channelId)
   223  }
   224  
   225  func (a *App) InvalidateCacheForChannelMembers(channelId string) {
   226  	a.InvalidateCacheForChannelMembersSkipClusterSend(channelId)
   227  
   228  	if a.Cluster != nil {
   229  		msg := &model.ClusterMessage{
   230  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS,
   231  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   232  			Data:     channelId,
   233  		}
   234  		a.Cluster.SendClusterMessage(msg)
   235  	}
   236  }
   237  
   238  func (a *App) InvalidateCacheForChannelMembersSkipClusterSend(channelId string) {
   239  	a.Srv.Store.User().InvalidateProfilesInChannelCache(channelId)
   240  	a.Srv.Store.Channel().InvalidateMemberCount(channelId)
   241  }
   242  
   243  func (a *App) InvalidateCacheForChannelMembersNotifyProps(channelId string) {
   244  	a.InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId)
   245  
   246  	if a.Cluster != nil {
   247  		msg := &model.ClusterMessage{
   248  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBERS_NOTIFY_PROPS,
   249  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   250  			Data:     channelId,
   251  		}
   252  		a.Cluster.SendClusterMessage(msg)
   253  	}
   254  }
   255  
   256  func (a *App) InvalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelId string) {
   257  	a.Srv.Store.Channel().InvalidateCacheForChannelMembersNotifyProps(channelId)
   258  }
   259  
   260  func (a *App) InvalidateCacheForChannelByNameSkipClusterSend(teamId, name string) {
   261  	if teamId == "" {
   262  		teamId = "dm"
   263  	}
   264  
   265  	a.Srv.Store.Channel().InvalidateChannelByName(teamId, name)
   266  }
   267  
   268  func (a *App) InvalidateCacheForChannelPosts(channelId string) {
   269  	a.InvalidateCacheForChannelPostsSkipClusterSend(channelId)
   270  
   271  	if a.Cluster != nil {
   272  		msg := &model.ClusterMessage{
   273  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_POSTS,
   274  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   275  			Data:     channelId,
   276  		}
   277  		a.Cluster.SendClusterMessage(msg)
   278  	}
   279  }
   280  
   281  func (a *App) InvalidateCacheForChannelPostsSkipClusterSend(channelId string) {
   282  	a.Srv.Store.Post().InvalidateLastPostTimeCache(channelId)
   283  }
   284  
   285  func (a *App) InvalidateCacheForUser(userId string) {
   286  	a.InvalidateCacheForUserSkipClusterSend(userId)
   287  
   288  	if a.Cluster != nil {
   289  		msg := &model.ClusterMessage{
   290  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_USER,
   291  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   292  			Data:     userId,
   293  		}
   294  		a.Cluster.SendClusterMessage(msg)
   295  	}
   296  }
   297  
   298  func (a *App) InvalidateCacheForUserSkipClusterSend(userId string) {
   299  	a.Srv.Store.Channel().InvalidateAllChannelMembersForUser(userId)
   300  	a.Srv.Store.User().InvalidateProfilesInChannelCacheByUser(userId)
   301  	a.Srv.Store.User().InvalidatProfileCacheForUser(userId)
   302  
   303  	hub := a.GetHubForUserId(userId)
   304  	if hub != nil {
   305  		hub.InvalidateUser(userId)
   306  	}
   307  }
   308  
   309  func (a *App) InvalidateCacheForWebhook(webhookId string) {
   310  	a.InvalidateCacheForWebhookSkipClusterSend(webhookId)
   311  
   312  	if a.Cluster != nil {
   313  		msg := &model.ClusterMessage{
   314  			Event:    model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_WEBHOOK,
   315  			SendType: model.CLUSTER_SEND_BEST_EFFORT,
   316  			Data:     webhookId,
   317  		}
   318  		a.Cluster.SendClusterMessage(msg)
   319  	}
   320  }
   321  
   322  func (a *App) InvalidateCacheForWebhookSkipClusterSend(webhookId string) {
   323  	a.Srv.Store.Webhook().InvalidateWebhookCache(webhookId)
   324  }
   325  
   326  func (a *App) InvalidateWebConnSessionCacheForUser(userId string) {
   327  	hub := a.GetHubForUserId(userId)
   328  	if hub != nil {
   329  		hub.InvalidateUser(userId)
   330  	}
   331  }
   332  
   333  func (h *Hub) Register(webConn *WebConn) {
   334  	h.register <- webConn
   335  
   336  	if webConn.IsAuthenticated() {
   337  		webConn.SendHello()
   338  	}
   339  }
   340  
   341  func (h *Hub) Unregister(webConn *WebConn) {
   342  	select {
   343  	case h.unregister <- webConn:
   344  	case <-h.stop:
   345  	}
   346  }
   347  
   348  func (h *Hub) Broadcast(message *model.WebSocketEvent) {
   349  	if h != nil && h.broadcast != nil && message != nil {
   350  		h.broadcast <- message
   351  	}
   352  }
   353  
   354  func (h *Hub) InvalidateUser(userId string) {
   355  	h.invalidateUser <- userId
   356  }
   357  
   358  func getGoroutineId() int {
   359  	var buf [64]byte
   360  	n := runtime.Stack(buf[:], false)
   361  	idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0]
   362  	id, err := strconv.Atoi(idField)
   363  	if err != nil {
   364  		id = -1
   365  	}
   366  	return id
   367  }
   368  
   369  func (h *Hub) Stop() {
   370  	close(h.stop)
   371  	<-h.didStop
   372  }
   373  
   374  func (h *Hub) Start() {
   375  	var doStart func()
   376  	var doRecoverableStart func()
   377  	var doRecover func()
   378  
   379  	doStart = func() {
   380  		h.goroutineId = getGoroutineId()
   381  		mlog.Debug(fmt.Sprintf("Hub for index %v is starting with goroutine %v", h.connectionIndex, h.goroutineId))
   382  
   383  		connections := newHubConnectionIndex()
   384  
   385  		for {
   386  			select {
   387  			case webCon := <-h.register:
   388  				connections.Add(webCon)
   389  				atomic.StoreInt64(&h.connectionCount, int64(len(connections.All())))
   390  			case webCon := <-h.unregister:
   391  				connections.Remove(webCon)
   392  				atomic.StoreInt64(&h.connectionCount, int64(len(connections.All())))
   393  
   394  				if len(webCon.UserId) == 0 {
   395  					continue
   396  				}
   397  
   398  				if len(connections.ForUser(webCon.UserId)) == 0 {
   399  					h.app.Go(func() {
   400  						h.app.SetStatusOffline(webCon.UserId, false)
   401  					})
   402  				}
   403  			case userId := <-h.invalidateUser:
   404  				for _, webCon := range connections.ForUser(userId) {
   405  					webCon.InvalidateCache()
   406  				}
   407  			case msg := <-h.broadcast:
   408  				candidates := connections.All()
   409  				if msg.Broadcast.UserId != "" {
   410  					candidates = connections.ForUser(msg.Broadcast.UserId)
   411  				}
   412  				msg.PrecomputeJSON()
   413  				for _, webCon := range candidates {
   414  					if webCon.ShouldSendEvent(msg) {
   415  						select {
   416  						case webCon.Send <- msg:
   417  						default:
   418  							mlog.Error(fmt.Sprintf("webhub.broadcast: cannot send, closing websocket for userId=%v", webCon.UserId))
   419  							close(webCon.Send)
   420  							connections.Remove(webCon)
   421  						}
   422  					}
   423  				}
   424  			case <-h.stop:
   425  				userIds := make(map[string]bool)
   426  
   427  				for _, webCon := range connections.All() {
   428  					userIds[webCon.UserId] = true
   429  					webCon.Close()
   430  				}
   431  
   432  				for userId := range userIds {
   433  					h.app.SetStatusOffline(userId, false)
   434  				}
   435  
   436  				h.ExplicitStop = true
   437  				close(h.didStop)
   438  
   439  				return
   440  			}
   441  		}
   442  	}
   443  
   444  	doRecoverableStart = func() {
   445  		defer doRecover()
   446  		doStart()
   447  	}
   448  
   449  	doRecover = func() {
   450  		if !h.ExplicitStop {
   451  			if r := recover(); r != nil {
   452  				mlog.Error(fmt.Sprintf("Recovering from Hub panic. Panic was: %v", r))
   453  			} else {
   454  				mlog.Error("Webhub stopped unexpectedly. Recovering.")
   455  			}
   456  
   457  			mlog.Error(string(debug.Stack()))
   458  
   459  			go doRecoverableStart()
   460  		}
   461  	}
   462  
   463  	go doRecoverableStart()
   464  }
   465  
   466  type hubConnectionIndexIndexes struct {
   467  	connections         int
   468  	connectionsByUserId int
   469  }
   470  
   471  // hubConnectionIndex provides fast addition, removal, and iteration of web connections.
   472  type hubConnectionIndex struct {
   473  	connections         []*WebConn
   474  	connectionsByUserId map[string][]*WebConn
   475  	connectionIndexes   map[*WebConn]*hubConnectionIndexIndexes
   476  }
   477  
   478  func newHubConnectionIndex() *hubConnectionIndex {
   479  	return &hubConnectionIndex{
   480  		connections:         make([]*WebConn, 0, model.SESSION_CACHE_SIZE),
   481  		connectionsByUserId: make(map[string][]*WebConn),
   482  		connectionIndexes:   make(map[*WebConn]*hubConnectionIndexIndexes),
   483  	}
   484  }
   485  
   486  func (i *hubConnectionIndex) Add(wc *WebConn) {
   487  	i.connections = append(i.connections, wc)
   488  	i.connectionsByUserId[wc.UserId] = append(i.connectionsByUserId[wc.UserId], wc)
   489  	i.connectionIndexes[wc] = &hubConnectionIndexIndexes{
   490  		connections:         len(i.connections) - 1,
   491  		connectionsByUserId: len(i.connectionsByUserId[wc.UserId]) - 1,
   492  	}
   493  }
   494  
   495  func (i *hubConnectionIndex) Remove(wc *WebConn) {
   496  	indexes, ok := i.connectionIndexes[wc]
   497  	if !ok {
   498  		return
   499  	}
   500  
   501  	last := i.connections[len(i.connections)-1]
   502  	i.connections[indexes.connections] = last
   503  	i.connections = i.connections[:len(i.connections)-1]
   504  	i.connectionIndexes[last].connections = indexes.connections
   505  
   506  	userConnections := i.connectionsByUserId[wc.UserId]
   507  	last = userConnections[len(userConnections)-1]
   508  	userConnections[indexes.connectionsByUserId] = last
   509  	i.connectionsByUserId[wc.UserId] = userConnections[:len(userConnections)-1]
   510  	i.connectionIndexes[last].connectionsByUserId = indexes.connectionsByUserId
   511  
   512  	delete(i.connectionIndexes, wc)
   513  }
   514  
   515  func (i *hubConnectionIndex) ForUser(id string) []*WebConn {
   516  	return i.connectionsByUserId[id]
   517  }
   518  
   519  func (i *hubConnectionIndex) All() []*WebConn {
   520  	return i.connections
   521  }