github.com/kubewharf/katalyst-core@v0.5.3/pkg/util/syntax/lock.go (about) 1 /* 2 Copyright 2022 The Katalyst Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 /* 18 native locking functions in package-sync doesn't contain deadlock-detecting logic, 19 to avoid deadlock during runtime processes, it's advised to use locking functions 20 here to provide deadlock-detecting when it exceeds timeout 21 */ 22 23 package syntax 24 25 import ( 26 "runtime" 27 "sync" 28 "time" 29 30 "k8s.io/klog/v2" 31 32 "github.com/kubewharf/katalyst-core/pkg/metrics" 33 ) 34 35 const defaultDeadlockPeriod = time.Minute * 5 36 37 const ( 38 metricsNameLockTimeout = "lock_timeout" 39 metricsNameRLockTimeout = "rlock_timeout" 40 ) 41 42 type Mutex struct { 43 sync.Mutex 44 45 emitter metrics.MetricEmitter 46 deadlock time.Duration 47 } 48 49 func NewMutex(emitter metrics.MetricEmitter) *Mutex { 50 return &Mutex{ 51 emitter: emitter, 52 deadlock: defaultDeadlockPeriod, 53 } 54 } 55 56 func NewMutexWithPeriod(emitter metrics.MetricEmitter, period time.Duration) *Mutex { 57 return &Mutex{ 58 emitter: emitter, 59 deadlock: period, 60 } 61 } 62 63 func (m *Mutex) Lock() int { 64 return runWithTimeoutDetect(m.deadlock, m.Mutex.Lock, func() { 65 _ = m.emitter.StoreInt64(metricsNameLockTimeout, 1, metrics.MetricTypeNameRaw) 66 klog.Infof("potential deadlock Mutex.Lock for caller: %v", getCaller()) 67 }) 68 } 69 70 func (m *Mutex) Unlock() { m.Mutex.Unlock() } 71 72 type RWMutex struct { 73 sync.RWMutex 74 75 emitter metrics.MetricEmitter 76 deadlock time.Duration 77 } 78 79 func NewRWMutex(emitter metrics.MetricEmitter) *RWMutex { 80 return &RWMutex{ 81 emitter: emitter, 82 deadlock: defaultDeadlockPeriod, 83 } 84 } 85 86 func NewRWMutexWithPeriod(emitter metrics.MetricEmitter, period time.Duration) *RWMutex { 87 return &RWMutex{ 88 emitter: emitter, 89 deadlock: period, 90 } 91 } 92 93 func (rm *RWMutex) Lock() int { 94 return runWithTimeoutDetect(rm.deadlock, rm.RWMutex.Lock, func() { 95 _ = rm.emitter.StoreInt64(metricsNameLockTimeout, 1, metrics.MetricTypeNameRaw) 96 klog.Infof("potential deadlock RWMutex.Lock for caller: %v", getCaller()) 97 }) 98 } 99 100 func (rm *RWMutex) RLock() int { 101 return runWithTimeoutDetect(rm.deadlock, rm.RWMutex.RLock, func() { 102 _ = rm.emitter.StoreInt64(metricsNameRLockTimeout, 1, metrics.MetricTypeNameRaw) 103 klog.Infof("potential deadlock RWMutex.RLock for caller: %v", getCaller()) 104 }) 105 } 106 107 func (rm *RWMutex) Unlock() { rm.RWMutex.Unlock() } 108 func (rm *RWMutex) RUnlock() { rm.RWMutex.RUnlock() } 109 110 // getCaller returns the caller info for logging 111 func getCaller() string { 112 pc, _, _, ok := runtime.Caller(4) 113 if !ok { 114 return "" 115 } 116 return runtime.FuncForPC(pc).Name() 117 } 118 119 // runWithTimeoutDetect runs the given command, and exec the onTimeout function if timeout 120 func runWithTimeoutDetect(timeout time.Duration, command, onTimeout func()) int { 121 c := make(chan interface{}) 122 cnt := 0 123 124 go func() { 125 for { 126 select { 127 case <-c: 128 return 129 case <-time.After(timeout): 130 onTimeout() 131 cnt++ 132 } 133 } 134 }() 135 136 command() 137 c <- struct{}{} 138 return cnt 139 }