github.com/vnforks/kid@v5.11.1+incompatible/app/server.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "context" 8 "crypto/ecdsa" 9 "crypto/tls" 10 "fmt" 11 "net" 12 "net/http" 13 "net/url" 14 "os" 15 "strings" 16 "sync" 17 "sync/atomic" 18 "time" 19 20 "github.com/gorilla/mux" 21 "github.com/pkg/errors" 22 "github.com/rs/cors" 23 "github.com/throttled/throttled" 24 "golang.org/x/crypto/acme/autocert" 25 26 "github.com/mattermost/mattermost-server/config" 27 "github.com/mattermost/mattermost-server/einterfaces" 28 "github.com/mattermost/mattermost-server/jobs" 29 "github.com/mattermost/mattermost-server/mlog" 30 "github.com/mattermost/mattermost-server/model" 31 "github.com/mattermost/mattermost-server/plugin" 32 "github.com/mattermost/mattermost-server/services/httpservice" 33 "github.com/mattermost/mattermost-server/services/imageproxy" 34 "github.com/mattermost/mattermost-server/services/timezones" 35 "github.com/mattermost/mattermost-server/store" 36 "github.com/mattermost/mattermost-server/utils" 37 ) 38 39 var MaxNotificationsPerChannelDefault int64 = 1000000 40 41 type Server struct { 42 Store store.Store 43 WebSocketRouter *WebSocketRouter 44 45 // RootRouter is the starting point for all HTTP requests to the server. 46 RootRouter *mux.Router 47 48 // Router is the starting point for all web, api4 and ws requests to the server. It differs 49 // from RootRouter only if the SiteURL contains a /subpath. 50 Router *mux.Router 51 52 Server *http.Server 53 ListenAddr *net.TCPAddr 54 RateLimiter *RateLimiter 55 56 didFinishListen chan struct{} 57 58 goroutineCount int32 59 goroutineExitSignal chan struct{} 60 61 PluginsEnvironment *plugin.Environment 62 PluginConfigListenerId string 63 PluginsLock sync.RWMutex 64 65 EmailBatching *EmailBatchingJob 66 EmailRateLimiter *throttled.GCRARateLimiter 67 68 Hubs []*Hub 69 HubsStopCheckingForDeadlock chan bool 70 71 PushNotificationsHub PushNotificationsHub 72 73 runjobs bool 74 Jobs *jobs.JobServer 75 76 clusterLeaderListeners sync.Map 77 78 licenseValue atomic.Value 79 clientLicenseValue atomic.Value 80 licenseListeners map[string]func() 81 82 timezones *timezones.Timezones 83 84 newStore func() store.Store 85 86 htmlTemplateWatcher *utils.HTMLTemplateWatcher 87 sessionCache *utils.Cache 88 seenPendingPostIdsCache *utils.Cache 89 configListenerId string 90 licenseListenerId string 91 logListenerId string 92 clusterLeaderListenerId string 93 configStore config.Store 94 asymmetricSigningKey *ecdsa.PrivateKey 95 postActionCookieSecret []byte 96 97 pluginCommands []*PluginCommand 98 pluginCommandsLock sync.RWMutex 99 100 clientConfig map[string]string 101 clientConfigHash string 102 limitedClientConfig map[string]string 103 diagnosticId string 104 105 phase2PermissionsMigrationComplete bool 106 107 HTTPService httpservice.HTTPService 108 109 ImageProxy *imageproxy.ImageProxy 110 111 Log *mlog.Logger 112 113 joinCluster bool 114 startMetrics bool 115 startElasticsearch bool 116 117 AccountMigration einterfaces.AccountMigrationInterface 118 Cluster einterfaces.ClusterInterface 119 Compliance einterfaces.ComplianceInterface 120 DataRetention einterfaces.DataRetentionInterface 121 Elasticsearch einterfaces.ElasticsearchInterface 122 Ldap einterfaces.LdapInterface 123 MessageExport einterfaces.MessageExportInterface 124 Metrics einterfaces.MetricsInterface 125 Saml einterfaces.SamlInterface 126 } 127 128 func NewServer(options ...Option) (*Server, error) { 129 rootRouter := mux.NewRouter() 130 131 s := &Server{ 132 goroutineExitSignal: make(chan struct{}, 1), 133 RootRouter: rootRouter, 134 licenseListeners: map[string]func(){}, 135 sessionCache: utils.NewLru(model.SESSION_CACHE_SIZE), 136 seenPendingPostIdsCache: utils.NewLru(PENDING_POST_IDS_CACHE_SIZE), 137 clientConfig: make(map[string]string), 138 } 139 for _, option := range options { 140 if err := option(s); err != nil { 141 return nil, errors.Wrap(err, "failed to apply option") 142 } 143 } 144 145 if s.configStore == nil { 146 configStore, err := config.NewFileStore("config.json", true) 147 if err != nil { 148 return nil, errors.Wrap(err, "failed to load config") 149 } 150 151 s.configStore = configStore 152 } 153 154 if s.Log == nil { 155 s.Log = mlog.NewLogger(utils.MloggerConfigFromLoggerConfig(&s.Config().LogSettings)) 156 } 157 158 // Redirect default golang logger to this logger 159 mlog.RedirectStdLog(s.Log) 160 161 // Use this app logger as the global logger (eventually remove all instances of global logging) 162 mlog.InitGlobalLogger(s.Log) 163 164 s.logListenerId = s.AddConfigListener(func(_, after *model.Config) { 165 s.Log.ChangeLevels(utils.MloggerConfigFromLoggerConfig(&after.LogSettings)) 166 }) 167 168 s.HTTPService = httpservice.MakeHTTPService(s.FakeApp()) 169 170 s.ImageProxy = imageproxy.MakeImageProxy(s, s.HTTPService) 171 172 if err := utils.TranslationsPreInit(); err != nil { 173 return nil, errors.Wrapf(err, "unable to load Mattermost translation files") 174 } 175 176 err := s.RunOldAppInitalization() 177 if err != nil { 178 return nil, err 179 } 180 181 model.AppErrorInit(utils.T) 182 183 s.timezones = timezones.New() 184 185 // Start email batching because it's not like the other jobs 186 s.InitEmailBatching() 187 s.AddConfigListener(func(_, _ *model.Config) { 188 s.InitEmailBatching() 189 }) 190 191 mlog.Info(fmt.Sprintf("Current version is %v (%v/%v/%v/%v)", model.CurrentVersion, model.BuildNumber, model.BuildDate, model.BuildHash, model.BuildHashEnterprise)) 192 mlog.Info(fmt.Sprintf("Enterprise Enabled: %v", model.BuildEnterpriseReady)) 193 pwd, _ := os.Getwd() 194 mlog.Info(fmt.Sprintf("Current working directory is %v", pwd)) 195 mlog.Info("Loaded config", mlog.String("source", s.configStore.String())) 196 197 license := s.License() 198 199 if license == nil && len(s.Config().SqlSettings.DataSourceReplicas) > 1 { 200 mlog.Warn("More than 1 read replica functionality disabled by current license. Please contact your system administrator about upgrading your enterprise license.") 201 s.UpdateConfig(func(cfg *model.Config) { 202 cfg.SqlSettings.DataSourceReplicas = cfg.SqlSettings.DataSourceReplicas[:1] 203 }) 204 } 205 206 if license == nil { 207 s.UpdateConfig(func(cfg *model.Config) { 208 cfg.TeamSettings.MaxNotificationsPerChannel = &MaxNotificationsPerChannelDefault 209 }) 210 } 211 212 s.ReloadConfig() 213 214 // Enable developer settings if this is a "dev" build 215 if model.BuildNumber == "dev" { 216 s.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.EnableDeveloper = true }) 217 } 218 219 if result := <-s.Store.Status().ResetAll(); result.Err != nil { 220 mlog.Error(fmt.Sprint("Error to reset the server status.", result.Err.Error())) 221 } 222 223 if s.joinCluster && s.Cluster != nil { 224 s.FakeApp().RegisterAllClusterMessageHandlers() 225 s.Cluster.StartInterNodeCommunication() 226 } 227 228 if s.startMetrics && s.Metrics != nil { 229 s.Metrics.StartServer() 230 } 231 232 if s.startElasticsearch && s.Elasticsearch != nil { 233 s.StartElasticsearch() 234 } 235 236 s.initJobs() 237 238 if s.runjobs { 239 s.Go(func() { 240 runSecurityJob(s) 241 }) 242 s.Go(func() { 243 runDiagnosticsJob(s) 244 }) 245 s.Go(func() { 246 runSessionCleanupJob(s) 247 }) 248 s.Go(func() { 249 runTokenCleanupJob(s) 250 }) 251 s.Go(func() { 252 runCommandWebhookCleanupJob(s) 253 }) 254 255 if complianceI := s.Compliance; complianceI != nil { 256 complianceI.StartComplianceDailyJob() 257 } 258 259 if *s.Config().JobSettings.RunJobs && s.Jobs != nil { 260 s.Jobs.StartWorkers() 261 } 262 if *s.Config().JobSettings.RunScheduler && s.Jobs != nil { 263 s.Jobs.StartSchedulers() 264 } 265 } 266 267 return s, nil 268 } 269 270 // Global app opptions that should be applied to apps created by this server 271 func (s *Server) AppOptions() []AppOption { 272 return []AppOption{ 273 ServerConnector(s), 274 } 275 } 276 277 const TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN = time.Second 278 279 func (s *Server) StopHTTPServer() { 280 if s.Server != nil { 281 ctx, cancel := context.WithTimeout(context.Background(), TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN) 282 defer cancel() 283 didShutdown := false 284 for s.didFinishListen != nil && !didShutdown { 285 if err := s.Server.Shutdown(ctx); err != nil { 286 mlog.Warn(err.Error()) 287 } 288 timer := time.NewTimer(time.Millisecond * 50) 289 select { 290 case <-s.didFinishListen: 291 didShutdown = true 292 case <-timer.C: 293 } 294 timer.Stop() 295 } 296 s.Server.Close() 297 s.Server = nil 298 } 299 } 300 301 func (s *Server) Shutdown() error { 302 mlog.Info("Stopping Server...") 303 304 s.RunOldAppShutdown() 305 306 s.StopHTTPServer() 307 s.WaitForGoroutines() 308 309 if s.Store != nil { 310 s.Store.Close() 311 } 312 313 if s.htmlTemplateWatcher != nil { 314 s.htmlTemplateWatcher.Close() 315 } 316 317 s.RemoveConfigListener(s.configListenerId) 318 s.RemoveConfigListener(s.logListenerId) 319 320 s.configStore.Close() 321 322 if s.Cluster != nil { 323 s.Cluster.StopInterNodeCommunication() 324 } 325 326 if s.Metrics != nil { 327 s.Metrics.StopServer() 328 } 329 330 if s.Jobs != nil && s.runjobs { 331 s.Jobs.StopWorkers() 332 s.Jobs.StopSchedulers() 333 } 334 335 mlog.Info("Server stopped") 336 return nil 337 } 338 339 // Go creates a goroutine, but maintains a record of it to ensure that execution completes before 340 // the server is shutdown. 341 func (s *Server) Go(f func()) { 342 atomic.AddInt32(&s.goroutineCount, 1) 343 344 go func() { 345 f() 346 347 atomic.AddInt32(&s.goroutineCount, -1) 348 select { 349 case s.goroutineExitSignal <- struct{}{}: 350 default: 351 } 352 }() 353 } 354 355 // WaitForGoroutines blocks until all goroutines created by App.Go exit. 356 func (s *Server) WaitForGoroutines() { 357 for atomic.LoadInt32(&s.goroutineCount) != 0 { 358 <-s.goroutineExitSignal 359 } 360 } 361 362 var corsAllowedMethods = []string{ 363 "POST", 364 "GET", 365 "OPTIONS", 366 "PUT", 367 "PATCH", 368 "DELETE", 369 } 370 371 // golang.org/x/crypto/acme/autocert/autocert.go 372 func handleHTTPRedirect(w http.ResponseWriter, r *http.Request) { 373 if r.Method != "GET" && r.Method != "HEAD" { 374 http.Error(w, "Use HTTPS", http.StatusBadRequest) 375 return 376 } 377 target := "https://" + stripPort(r.Host) + r.URL.RequestURI() 378 http.Redirect(w, r, target, http.StatusFound) 379 } 380 381 // golang.org/x/crypto/acme/autocert/autocert.go 382 func stripPort(hostport string) string { 383 host, _, err := net.SplitHostPort(hostport) 384 if err != nil { 385 return hostport 386 } 387 return net.JoinHostPort(host, "443") 388 } 389 390 func (s *Server) Start() error { 391 mlog.Info("Starting Server...") 392 393 var handler http.Handler = s.RootRouter 394 if allowedOrigins := *s.Config().ServiceSettings.AllowCorsFrom; allowedOrigins != "" { 395 exposedCorsHeaders := *s.Config().ServiceSettings.CorsExposedHeaders 396 allowCredentials := *s.Config().ServiceSettings.CorsAllowCredentials 397 debug := *s.Config().ServiceSettings.CorsDebug 398 corsWrapper := cors.New(cors.Options{ 399 AllowedOrigins: strings.Fields(allowedOrigins), 400 AllowedMethods: corsAllowedMethods, 401 AllowedHeaders: []string{"*"}, 402 ExposedHeaders: strings.Fields(exposedCorsHeaders), 403 MaxAge: 86400, 404 AllowCredentials: allowCredentials, 405 Debug: debug, 406 }) 407 408 // If we have debugging of CORS turned on then forward messages to logs 409 if debug { 410 corsWrapper.Log = s.Log.StdLog(mlog.String("source", "cors")) 411 } 412 413 handler = corsWrapper.Handler(handler) 414 } 415 416 if *s.Config().RateLimitSettings.Enable { 417 mlog.Info("RateLimiter is enabled") 418 419 rateLimiter, err := NewRateLimiter(&s.Config().RateLimitSettings) 420 if err != nil { 421 return err 422 } 423 424 s.RateLimiter = rateLimiter 425 handler = rateLimiter.RateLimitHandler(handler) 426 } 427 428 // Creating a logger for logging errors from http.Server at error level 429 errStdLog, err := s.Log.StdLogAt(mlog.LevelError, mlog.String("source", "httpserver")) 430 if err != nil { 431 return err 432 } 433 434 s.Server = &http.Server{ 435 Handler: handler, 436 ReadTimeout: time.Duration(*s.Config().ServiceSettings.ReadTimeout) * time.Second, 437 WriteTimeout: time.Duration(*s.Config().ServiceSettings.WriteTimeout) * time.Second, 438 ErrorLog: errStdLog, 439 } 440 441 addr := *s.Config().ServiceSettings.ListenAddress 442 if addr == "" { 443 if *s.Config().ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { 444 addr = ":https" 445 } else { 446 addr = ":http" 447 } 448 } 449 450 listener, err := net.Listen("tcp", addr) 451 if err != nil { 452 errors.Wrapf(err, utils.T("api.server.start_server.starting.critical"), err) 453 return err 454 } 455 s.ListenAddr = listener.Addr().(*net.TCPAddr) 456 457 mlog.Info(fmt.Sprintf("Server is listening on %v", listener.Addr().String())) 458 459 // Migration from old let's encrypt library 460 if *s.Config().ServiceSettings.UseLetsEncrypt { 461 if stat, err := os.Stat(*s.Config().ServiceSettings.LetsEncryptCertificateCacheFile); err == nil && !stat.IsDir() { 462 os.Remove(*s.Config().ServiceSettings.LetsEncryptCertificateCacheFile) 463 } 464 } 465 466 m := &autocert.Manager{ 467 Cache: autocert.DirCache(*s.Config().ServiceSettings.LetsEncryptCertificateCacheFile), 468 Prompt: autocert.AcceptTOS, 469 } 470 471 if *s.Config().ServiceSettings.Forward80To443 { 472 if host, port, err := net.SplitHostPort(addr); err != nil { 473 mlog.Error("Unable to setup forwarding: " + err.Error()) 474 } else if port != "443" { 475 return fmt.Errorf(utils.T("api.server.start_server.forward80to443.enabled_but_listening_on_wrong_port"), port) 476 } else { 477 httpListenAddress := net.JoinHostPort(host, "http") 478 479 if *s.Config().ServiceSettings.UseLetsEncrypt { 480 server := &http.Server{ 481 Addr: httpListenAddress, 482 Handler: m.HTTPHandler(nil), 483 ErrorLog: s.Log.StdLog(mlog.String("source", "le_forwarder_server")), 484 } 485 go server.ListenAndServe() 486 } else { 487 go func() { 488 redirectListener, err := net.Listen("tcp", httpListenAddress) 489 if err != nil { 490 mlog.Error("Unable to setup forwarding: " + err.Error()) 491 return 492 } 493 defer redirectListener.Close() 494 495 server := &http.Server{ 496 Handler: http.HandlerFunc(handleHTTPRedirect), 497 ErrorLog: s.Log.StdLog(mlog.String("source", "forwarder_server")), 498 } 499 server.Serve(redirectListener) 500 }() 501 } 502 } 503 } else if *s.Config().ServiceSettings.UseLetsEncrypt { 504 return errors.New(utils.T("api.server.start_server.forward80to443.disabled_while_using_lets_encrypt")) 505 } 506 507 s.didFinishListen = make(chan struct{}) 508 go func() { 509 var err error 510 if *s.Config().ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { 511 512 tlsConfig := &tls.Config{ 513 PreferServerCipherSuites: true, 514 CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256}, 515 } 516 517 switch *s.Config().ServiceSettings.TLSMinVer { 518 case "1.0": 519 tlsConfig.MinVersion = tls.VersionTLS10 520 case "1.1": 521 tlsConfig.MinVersion = tls.VersionTLS11 522 default: 523 tlsConfig.MinVersion = tls.VersionTLS12 524 } 525 526 defaultCiphers := []uint16{ 527 tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 528 tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, 529 tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 530 tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 531 tls.TLS_RSA_WITH_AES_128_GCM_SHA256, 532 tls.TLS_RSA_WITH_AES_256_GCM_SHA384, 533 } 534 535 if len(s.Config().ServiceSettings.TLSOverwriteCiphers) == 0 { 536 tlsConfig.CipherSuites = defaultCiphers 537 } else { 538 var cipherSuites []uint16 539 for _, cipher := range s.Config().ServiceSettings.TLSOverwriteCiphers { 540 value, ok := model.ServerTLSSupportedCiphers[cipher] 541 542 if !ok { 543 mlog.Warn("Unsupported cipher passed", mlog.String("cipher", cipher)) 544 continue 545 } 546 547 cipherSuites = append(cipherSuites, value) 548 } 549 550 if len(cipherSuites) == 0 { 551 mlog.Warn("No supported ciphers passed, fallback to default cipher suite") 552 cipherSuites = defaultCiphers 553 } 554 555 tlsConfig.CipherSuites = cipherSuites 556 } 557 558 certFile := "" 559 keyFile := "" 560 561 if *s.Config().ServiceSettings.UseLetsEncrypt { 562 tlsConfig.GetCertificate = m.GetCertificate 563 tlsConfig.NextProtos = append(tlsConfig.NextProtos, "h2") 564 } else { 565 certFile = *s.Config().ServiceSettings.TLSCertFile 566 keyFile = *s.Config().ServiceSettings.TLSKeyFile 567 } 568 569 s.Server.TLSConfig = tlsConfig 570 err = s.Server.ServeTLS(listener, certFile, keyFile) 571 } else { 572 err = s.Server.Serve(listener) 573 } 574 575 if err != nil && err != http.ErrServerClosed { 576 mlog.Critical(fmt.Sprintf("Error starting server, err:%v", err)) 577 time.Sleep(time.Second) 578 } 579 580 close(s.didFinishListen) 581 }() 582 583 return nil 584 } 585 586 func (a *App) OriginChecker() func(*http.Request) bool { 587 if allowed := *a.Config().ServiceSettings.AllowCorsFrom; allowed != "" { 588 if allowed != "*" { 589 siteURL, err := url.Parse(*a.Config().ServiceSettings.SiteURL) 590 if err == nil { 591 siteURL.Path = "" 592 allowed += " " + siteURL.String() 593 } 594 } 595 596 return utils.OriginChecker(allowed) 597 } 598 return nil 599 } 600 601 func runSecurityJob(s *Server) { 602 doSecurity(s) 603 model.CreateRecurringTask("Security", func() { 604 doSecurity(s) 605 }, time.Hour*4) 606 } 607 608 func runDiagnosticsJob(s *Server) { 609 doDiagnostics(s) 610 model.CreateRecurringTask("Diagnostics", func() { 611 doDiagnostics(s) 612 }, time.Hour*24) 613 } 614 615 func runTokenCleanupJob(s *Server) { 616 doTokenCleanup(s) 617 model.CreateRecurringTask("Token Cleanup", func() { 618 doTokenCleanup(s) 619 }, time.Hour*1) 620 } 621 622 func runCommandWebhookCleanupJob(s *Server) { 623 doCommandWebhookCleanup(s) 624 model.CreateRecurringTask("Command Hook Cleanup", func() { 625 doCommandWebhookCleanup(s) 626 }, time.Hour*1) 627 } 628 629 func runSessionCleanupJob(s *Server) { 630 doSessionCleanup(s) 631 model.CreateRecurringTask("Session Cleanup", func() { 632 doSessionCleanup(s) 633 }, time.Hour*24) 634 } 635 636 func doSecurity(s *Server) { 637 s.DoSecurityUpdateCheck() 638 } 639 640 func doDiagnostics(s *Server) { 641 if *s.Config().LogSettings.EnableDiagnostics { 642 s.FakeApp().SendDailyDiagnostics() 643 } 644 } 645 646 func doTokenCleanup(s *Server) { 647 s.Store.Token().Cleanup() 648 } 649 650 func doCommandWebhookCleanup(s *Server) { 651 s.Store.CommandWebhook().Cleanup() 652 } 653 654 const ( 655 SESSIONS_CLEANUP_BATCH_SIZE = 1000 656 ) 657 658 func doSessionCleanup(s *Server) { 659 s.Store.Session().Cleanup(model.GetMillis(), SESSIONS_CLEANUP_BATCH_SIZE) 660 } 661 662 func (s *Server) StartElasticsearch() { 663 s.Go(func() { 664 if err := s.Elasticsearch.Start(); err != nil { 665 s.Log.Error(err.Error()) 666 } 667 }) 668 669 s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) { 670 if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { 671 s.Go(func() { 672 if err := s.Elasticsearch.Start(); err != nil { 673 mlog.Error(err.Error()) 674 } 675 }) 676 } else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { 677 s.Go(func() { 678 if err := s.Elasticsearch.Stop(); err != nil { 679 mlog.Error(err.Error()) 680 } 681 }) 682 } else if *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff { 683 s.Go(func() { 684 if *oldConfig.ElasticsearchSettings.EnableIndexing { 685 if err := s.Elasticsearch.Stop(); err != nil { 686 mlog.Error(err.Error()) 687 } 688 if err := s.Elasticsearch.Start(); err != nil { 689 mlog.Error(err.Error()) 690 } 691 } 692 }) 693 } 694 }) 695 696 s.AddLicenseListener(func() { 697 if s.License() != nil { 698 s.Go(func() { 699 if err := s.Elasticsearch.Start(); err != nil { 700 mlog.Error(err.Error()) 701 } 702 }) 703 } else { 704 s.Go(func() { 705 if err := s.Elasticsearch.Stop(); err != nil { 706 mlog.Error(err.Error()) 707 } 708 }) 709 } 710 }) 711 }