github.com/matrixorigin/matrixone@v1.2.0/pkg/util/debug/goroutine/leak.go (about)

     1  // Copyright 2023 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package goroutine
    16  
    17  import (
    18  	"context"
    19  	"sync"
    20  	"time"
    21  
    22  	"github.com/matrixorigin/matrixone/pkg/common/stopper"
    23  	"go.uber.org/zap"
    24  )
    25  
    26  var (
    27  	creatorGroupTop = 5
    28  	methodGroupTop  = 5
    29  	leakOnce        sync.Once
    30  )
    31  
    32  func (c *Config) adjust() {
    33  	if c.CheckDuration.Duration == 0 {
    34  		c.CheckDuration.Duration = time.Hour
    35  	}
    36  	if c.SuspectLeakCount == 0 {
    37  		c.SuspectLeakCount = 10000
    38  	}
    39  }
    40  
    41  func StartLeakCheck(
    42  	stopper *stopper.Stopper,
    43  	cfg Config) {
    44  	if !cfg.EnableLeakCheck {
    45  		return
    46  	}
    47  
    48  	leakOnce.Do(func() {
    49  		stopper.RunTask(
    50  			func(ctx context.Context) {
    51  				startLeakCheckTask(ctx, cfg)
    52  			})
    53  	})
    54  }
    55  
    56  func startLeakCheckTask(
    57  	ctx context.Context,
    58  	cfg Config) {
    59  	cfg.adjust()
    60  	ticker := time.NewTicker(time.Duration(cfg.CheckDuration.Duration))
    61  	defer ticker.Stop()
    62  
    63  	for {
    64  		select {
    65  		case <-ctx.Done():
    66  			return
    67  		case <-ticker.C:
    68  			doLeakCheck(cfg)
    69  		}
    70  	}
    71  }
    72  
    73  func doLeakCheck(cfg Config) {
    74  	values := GetAnalyzer().ParseSystem()
    75  	if len(values) < cfg.SuspectLeakCount {
    76  		return
    77  	}
    78  
    79  	getLogger().Warn("goroutine suspect leak detected",
    80  		zap.Int("total", len(values)))
    81  	res := GetAnalyzer().GroupAnalyze(values)
    82  	res.read(
    83  		func(
    84  			group int,
    85  			g Goroutine,
    86  			count int) {
    87  			if group < creatorGroupTop {
    88  				m, f := g.CreateBy()
    89  				getLogger().Warn("goroutine suspect leak group by creator",
    90  					zap.Int("group", group),
    91  					zap.String("create-method", m),
    92  					zap.String("create-file", f),
    93  					zap.Int("group-count", count))
    94  			}
    95  		},
    96  		func(
    97  			group int,
    98  			methodGroup int,
    99  			g Goroutine,
   100  			count int) {
   101  			if group < creatorGroupTop &&
   102  				methodGroup < methodGroupTop {
   103  				getLogger().Warn("goroutine suspect leak group by current method",
   104  					zap.Int("group", group),
   105  					zap.String("goroutine state", g.rawState),
   106  					zap.String("last-method", g.methods[0]),
   107  					zap.String("last-file", g.files[0]),
   108  					zap.Int("count", count))
   109  			}
   110  		})
   111  
   112  }