github.com/adacta-ru/mattermost-server/v6@v6.0.0/services/searchengine/bleveengine/bleve.go (about)

     1  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
     2  // See LICENSE.txt for license information.
     3  
     4  package bleveengine
     5  
     6  import (
     7  	"net/http"
     8  	"os"
     9  	"path/filepath"
    10  	"sync"
    11  	"sync/atomic"
    12  	"time"
    13  
    14  	"github.com/adacta-ru/mattermost-server/v6/jobs"
    15  	"github.com/adacta-ru/mattermost-server/v6/mlog"
    16  	"github.com/adacta-ru/mattermost-server/v6/model"
    17  
    18  	"github.com/blevesearch/bleve"
    19  	"github.com/blevesearch/bleve/analysis/analyzer/keyword"
    20  	"github.com/blevesearch/bleve/analysis/analyzer/standard"
    21  	"github.com/blevesearch/bleve/mapping"
    22  )
    23  
    24  const (
    25  	ENGINE_NAME   = "bleve"
    26  	POST_INDEX    = "posts"
    27  	USER_INDEX    = "users"
    28  	CHANNEL_INDEX = "channels"
    29  )
    30  
    31  type BleveEngine struct {
    32  	PostIndex    bleve.Index
    33  	UserIndex    bleve.Index
    34  	ChannelIndex bleve.Index
    35  	Mutex        sync.RWMutex
    36  	ready        int32
    37  	cfg          *model.Config
    38  	jobServer    *jobs.JobServer
    39  	indexSync    bool
    40  }
    41  
    42  var keywordMapping *mapping.FieldMapping
    43  var standardMapping *mapping.FieldMapping
    44  var dateMapping *mapping.FieldMapping
    45  
    46  func init() {
    47  	keywordMapping = bleve.NewTextFieldMapping()
    48  	keywordMapping.Analyzer = keyword.Name
    49  
    50  	standardMapping = bleve.NewTextFieldMapping()
    51  	standardMapping.Analyzer = standard.Name
    52  
    53  	dateMapping = bleve.NewNumericFieldMapping()
    54  }
    55  
    56  func getChannelIndexMapping() *mapping.IndexMappingImpl {
    57  	channelMapping := bleve.NewDocumentMapping()
    58  	channelMapping.AddFieldMappingsAt("Id", keywordMapping)
    59  	channelMapping.AddFieldMappingsAt("TeamId", keywordMapping)
    60  	channelMapping.AddFieldMappingsAt("NameSuggest", keywordMapping)
    61  
    62  	indexMapping := bleve.NewIndexMapping()
    63  	indexMapping.AddDocumentMapping("_default", channelMapping)
    64  
    65  	return indexMapping
    66  }
    67  
    68  func getPostIndexMapping() *mapping.IndexMappingImpl {
    69  	postMapping := bleve.NewDocumentMapping()
    70  	postMapping.AddFieldMappingsAt("Id", keywordMapping)
    71  	postMapping.AddFieldMappingsAt("TeamId", keywordMapping)
    72  	postMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
    73  	postMapping.AddFieldMappingsAt("UserId", keywordMapping)
    74  	postMapping.AddFieldMappingsAt("CreateAt", dateMapping)
    75  	postMapping.AddFieldMappingsAt("Message", standardMapping)
    76  	postMapping.AddFieldMappingsAt("Type", keywordMapping)
    77  	postMapping.AddFieldMappingsAt("Hashtags", standardMapping)
    78  	postMapping.AddFieldMappingsAt("Attachments", standardMapping)
    79  
    80  	indexMapping := bleve.NewIndexMapping()
    81  	indexMapping.AddDocumentMapping("_default", postMapping)
    82  
    83  	return indexMapping
    84  }
    85  
    86  func getUserIndexMapping() *mapping.IndexMappingImpl {
    87  	userMapping := bleve.NewDocumentMapping()
    88  	userMapping.AddFieldMappingsAt("Id", keywordMapping)
    89  	userMapping.AddFieldMappingsAt("SuggestionsWithFullname", keywordMapping)
    90  	userMapping.AddFieldMappingsAt("SuggestionsWithoutFullname", keywordMapping)
    91  	userMapping.AddFieldMappingsAt("TeamsIds", keywordMapping)
    92  	userMapping.AddFieldMappingsAt("ChannelsIds", keywordMapping)
    93  
    94  	indexMapping := bleve.NewIndexMapping()
    95  	indexMapping.AddDocumentMapping("_default", userMapping)
    96  
    97  	return indexMapping
    98  }
    99  
   100  func NewBleveEngine(cfg *model.Config, jobServer *jobs.JobServer) *BleveEngine {
   101  	return &BleveEngine{
   102  		cfg:       cfg,
   103  		jobServer: jobServer,
   104  	}
   105  }
   106  
   107  func (b *BleveEngine) getIndexDir(indexName string) string {
   108  	return filepath.Join(*b.cfg.BleveSettings.IndexDir, indexName+".bleve")
   109  }
   110  
   111  func (b *BleveEngine) createOrOpenIndex(indexName string, mapping *mapping.IndexMappingImpl) (bleve.Index, error) {
   112  	indexPath := b.getIndexDir(indexName)
   113  	if index, err := bleve.Open(indexPath); err == nil {
   114  		return index, nil
   115  	}
   116  
   117  	index, err := bleve.New(indexPath, mapping)
   118  	if err != nil {
   119  		return nil, err
   120  	}
   121  	return index, nil
   122  }
   123  
   124  func (b *BleveEngine) openIndexes() *model.AppError {
   125  	if atomic.LoadInt32(&b.ready) != 0 {
   126  		return model.NewAppError("Bleveengine.Start", "bleveengine.already_started.error", nil, "", http.StatusInternalServerError)
   127  	}
   128  
   129  	var err error
   130  	b.PostIndex, err = b.createOrOpenIndex(POST_INDEX, getPostIndexMapping())
   131  	if err != nil {
   132  		return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, err.Error(), http.StatusInternalServerError)
   133  	}
   134  
   135  	b.UserIndex, err = b.createOrOpenIndex(USER_INDEX, getUserIndexMapping())
   136  	if err != nil {
   137  		return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, err.Error(), http.StatusInternalServerError)
   138  	}
   139  
   140  	b.ChannelIndex, err = b.createOrOpenIndex(CHANNEL_INDEX, getChannelIndexMapping())
   141  	if err != nil {
   142  		return model.NewAppError("Bleveengine.Start", "bleveengine.create_channel_index.error", nil, err.Error(), http.StatusInternalServerError)
   143  	}
   144  
   145  	atomic.StoreInt32(&b.ready, 1)
   146  	return nil
   147  }
   148  
   149  func (b *BleveEngine) Start() *model.AppError {
   150  	if !*b.cfg.BleveSettings.EnableIndexing || *b.cfg.BleveSettings.IndexDir == "" {
   151  		return nil
   152  	}
   153  
   154  	b.Mutex.Lock()
   155  	defer b.Mutex.Unlock()
   156  
   157  	mlog.Info("EXPERIMENTAL: Starting Bleve")
   158  
   159  	return b.openIndexes()
   160  }
   161  
   162  func (b *BleveEngine) closeIndexes() *model.AppError {
   163  	if b.IsActive() {
   164  		if err := b.PostIndex.Close(); err != nil {
   165  			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, err.Error(), http.StatusInternalServerError)
   166  		}
   167  
   168  		if err := b.UserIndex.Close(); err != nil {
   169  			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, err.Error(), http.StatusInternalServerError)
   170  		}
   171  
   172  		if err := b.ChannelIndex.Close(); err != nil {
   173  			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_channel_index.error", nil, err.Error(), http.StatusInternalServerError)
   174  		}
   175  	}
   176  
   177  	atomic.StoreInt32(&b.ready, 0)
   178  	return nil
   179  }
   180  
   181  func (b *BleveEngine) Stop() *model.AppError {
   182  	b.Mutex.Lock()
   183  	defer b.Mutex.Unlock()
   184  
   185  	mlog.Info("Stopping Bleve")
   186  
   187  	return b.closeIndexes()
   188  }
   189  
   190  func (b *BleveEngine) IsActive() bool {
   191  	return atomic.LoadInt32(&b.ready) == 1
   192  }
   193  
   194  func (b *BleveEngine) IsIndexingSync() bool {
   195  	return b.indexSync
   196  }
   197  
   198  func (b *BleveEngine) RefreshIndexes() *model.AppError {
   199  	return nil
   200  }
   201  
   202  func (b *BleveEngine) GetVersion() int {
   203  	return 0
   204  }
   205  
   206  func (b *BleveEngine) GetName() string {
   207  	return ENGINE_NAME
   208  }
   209  
   210  func (b *BleveEngine) TestConfig(cfg *model.Config) *model.AppError {
   211  	return nil
   212  }
   213  
   214  func (b *BleveEngine) deleteIndexes() *model.AppError {
   215  	if err := os.RemoveAll(b.getIndexDir(POST_INDEX)); err != nil {
   216  		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_post_index.error", nil, err.Error(), http.StatusInternalServerError)
   217  	}
   218  	if err := os.RemoveAll(b.getIndexDir(USER_INDEX)); err != nil {
   219  		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_user_index.error", nil, err.Error(), http.StatusInternalServerError)
   220  	}
   221  	if err := os.RemoveAll(b.getIndexDir(CHANNEL_INDEX)); err != nil {
   222  		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, err.Error(), http.StatusInternalServerError)
   223  	}
   224  	return nil
   225  }
   226  
   227  func (b *BleveEngine) PurgeIndexes() *model.AppError {
   228  	if *b.cfg.BleveSettings.IndexDir == "" {
   229  		return nil
   230  	}
   231  
   232  	b.Mutex.Lock()
   233  	defer b.Mutex.Unlock()
   234  
   235  	mlog.Info("PurgeIndexes Bleve")
   236  	if err := b.closeIndexes(); err != nil {
   237  		return err
   238  	}
   239  
   240  	if err := b.deleteIndexes(); err != nil {
   241  		return err
   242  	}
   243  
   244  	return b.openIndexes()
   245  }
   246  
   247  func (b *BleveEngine) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError {
   248  	return nil
   249  }
   250  
   251  func (b *BleveEngine) IsAutocompletionEnabled() bool {
   252  	return *b.cfg.BleveSettings.EnableAutocomplete
   253  }
   254  
   255  func (b *BleveEngine) IsIndexingEnabled() bool {
   256  	return *b.cfg.BleveSettings.EnableIndexing
   257  }
   258  
   259  func (b *BleveEngine) IsSearchEnabled() bool {
   260  	return *b.cfg.BleveSettings.EnableSearching
   261  }
   262  
   263  func (b *BleveEngine) UpdateConfig(cfg *model.Config) {
   264  	b.Mutex.Lock()
   265  	defer b.Mutex.Unlock()
   266  
   267  	mlog.Info("UpdateConf Bleve")
   268  
   269  	if *cfg.BleveSettings.EnableIndexing != *b.cfg.BleveSettings.EnableIndexing || *cfg.BleveSettings.IndexDir != *b.cfg.BleveSettings.IndexDir {
   270  		if err := b.closeIndexes(); err != nil {
   271  			mlog.Error("Error closing Bleve indexes to update the config", mlog.Err(err))
   272  			return
   273  		}
   274  		b.cfg = cfg
   275  		if err := b.openIndexes(); err != nil {
   276  			mlog.Error("Error opening Bleve indexes after updating the config", mlog.Err(err))
   277  		}
   278  		return
   279  	}
   280  	b.cfg = cfg
   281  }