github.com/kubewharf/katalyst-core@v0.5.3/pkg/util/syntax/lock.go (about)

     1  /*
     2  Copyright 2022 The Katalyst Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18   native locking functions in package-sync doesn't contain deadlock-detecting logic,
    19   to avoid deadlock during runtime processes, it's advised to use locking functions
    20   here to provide deadlock-detecting when it exceeds timeout
    21  */
    22  
    23  package syntax
    24  
    25  import (
    26  	"runtime"
    27  	"sync"
    28  	"time"
    29  
    30  	"k8s.io/klog/v2"
    31  
    32  	"github.com/kubewharf/katalyst-core/pkg/metrics"
    33  )
    34  
    35  const defaultDeadlockPeriod = time.Minute * 5
    36  
    37  const (
    38  	metricsNameLockTimeout  = "lock_timeout"
    39  	metricsNameRLockTimeout = "rlock_timeout"
    40  )
    41  
    42  type Mutex struct {
    43  	sync.Mutex
    44  
    45  	emitter  metrics.MetricEmitter
    46  	deadlock time.Duration
    47  }
    48  
    49  func NewMutex(emitter metrics.MetricEmitter) *Mutex {
    50  	return &Mutex{
    51  		emitter:  emitter,
    52  		deadlock: defaultDeadlockPeriod,
    53  	}
    54  }
    55  
    56  func NewMutexWithPeriod(emitter metrics.MetricEmitter, period time.Duration) *Mutex {
    57  	return &Mutex{
    58  		emitter:  emitter,
    59  		deadlock: period,
    60  	}
    61  }
    62  
    63  func (m *Mutex) Lock() int {
    64  	return runWithTimeoutDetect(m.deadlock, m.Mutex.Lock, func() {
    65  		_ = m.emitter.StoreInt64(metricsNameLockTimeout, 1, metrics.MetricTypeNameRaw)
    66  		klog.Infof("potential deadlock Mutex.Lock for caller: %v", getCaller())
    67  	})
    68  }
    69  
    70  func (m *Mutex) Unlock() { m.Mutex.Unlock() }
    71  
    72  type RWMutex struct {
    73  	sync.RWMutex
    74  
    75  	emitter  metrics.MetricEmitter
    76  	deadlock time.Duration
    77  }
    78  
    79  func NewRWMutex(emitter metrics.MetricEmitter) *RWMutex {
    80  	return &RWMutex{
    81  		emitter:  emitter,
    82  		deadlock: defaultDeadlockPeriod,
    83  	}
    84  }
    85  
    86  func NewRWMutexWithPeriod(emitter metrics.MetricEmitter, period time.Duration) *RWMutex {
    87  	return &RWMutex{
    88  		emitter:  emitter,
    89  		deadlock: period,
    90  	}
    91  }
    92  
    93  func (rm *RWMutex) Lock() int {
    94  	return runWithTimeoutDetect(rm.deadlock, rm.RWMutex.Lock, func() {
    95  		_ = rm.emitter.StoreInt64(metricsNameLockTimeout, 1, metrics.MetricTypeNameRaw)
    96  		klog.Infof("potential deadlock RWMutex.Lock for caller: %v", getCaller())
    97  	})
    98  }
    99  
   100  func (rm *RWMutex) RLock() int {
   101  	return runWithTimeoutDetect(rm.deadlock, rm.RWMutex.RLock, func() {
   102  		_ = rm.emitter.StoreInt64(metricsNameRLockTimeout, 1, metrics.MetricTypeNameRaw)
   103  		klog.Infof("potential deadlock RWMutex.RLock for caller: %v", getCaller())
   104  	})
   105  }
   106  
   107  func (rm *RWMutex) Unlock()  { rm.RWMutex.Unlock() }
   108  func (rm *RWMutex) RUnlock() { rm.RWMutex.RUnlock() }
   109  
   110  // getCaller returns the caller info for logging
   111  func getCaller() string {
   112  	pc, _, _, ok := runtime.Caller(4)
   113  	if !ok {
   114  		return ""
   115  	}
   116  	return runtime.FuncForPC(pc).Name()
   117  }
   118  
   119  // runWithTimeoutDetect runs the given command, and exec the onTimeout function if timeout
   120  func runWithTimeoutDetect(timeout time.Duration, command, onTimeout func()) int {
   121  	c := make(chan interface{})
   122  	cnt := 0
   123  
   124  	go func() {
   125  		for {
   126  			select {
   127  			case <-c:
   128  				return
   129  			case <-time.After(timeout):
   130  				onTimeout()
   131  				cnt++
   132  			}
   133  		}
   134  	}()
   135  
   136  	command()
   137  	c <- struct{}{}
   138  	return cnt
   139  }