github.com/alphadose/zenq/v2@v2.8.4/lib_runtime_linkage.go (about)

     1  package zenq
     2  
     3  import (
     4  	"runtime"
     5  	"unsafe"
     6  	_ "unsafe"
     7  
     8  	"github.com/alphadose/zenq/v2/constants"
     9  )
    10  
    11  type cacheLinePadding struct {
    12  	_ [constants.CacheLinePadSize]byte
    13  }
    14  
    15  // Linking ZenQ with golang internal runtime library to allow usage of scheduling primitives
    16  // like goready(), mcall() etc to allow low-level scheduling of goroutines
    17  
    18  type mutex struct {
    19  	// Futex-based impl treats it as uint32 key,
    20  	// while sema-based impl as M* waitm.
    21  	// Used to be a union, but unions break precise GC.
    22  	key uintptr
    23  }
    24  
    25  // The functions below are used for scheduling goroutines with exclusive control
    26  // Shifting to the below flow will remove the spinning and mutex lock implementations
    27  
    28  //go:linkname lock runtime.lock
    29  func lock(l *mutex)
    30  
    31  //go:linkname nanotime runtime.nanotime
    32  func nanotime() int64
    33  
    34  //go:linkname unlock runtime.unlock
    35  func unlock(l *mutex)
    36  
    37  //go:linkname goparkunlock runtime.goparkunlock
    38  func goparkunlock(lock *mutex, reason waitReason, traceEv byte, traceskip int)
    39  
    40  // GetG returns the pointer to the current goroutine
    41  // defined in the asm files
    42  func GetG() unsafe.Pointer
    43  
    44  //go:linkname Fastlog2 runtime.fastlog2
    45  func Fastlog2(x float64) float64
    46  
    47  //go:linkname goready runtime.goready
    48  func goready(goroutinePtr unsafe.Pointer, traceskip int)
    49  
    50  //go:linkname gopark runtime.gopark
    51  func gopark(unlockf func(unsafe.Pointer, unsafe.Pointer) bool, lock unsafe.Pointer, reason waitReason, traceEv byte, traceskip int)
    52  
    53  // Active spinning runtime support.
    54  // runtime_canSpin reports whether spinning makes sense at the moment.
    55  //go:linkname runtime_canSpin sync.runtime_canSpin
    56  func runtime_canSpin(i int) bool
    57  
    58  // runtime_doSpin does active spinning.
    59  // //go:linkname runtime_doSpin sync.runtime_doSpin
    60  // func runtime_doSpin()
    61  
    62  func runtime_doSpin() {
    63  	spin(30)
    64  }
    65  
    66  //go:linkname osyield runtime.osyield
    67  func osyield()
    68  
    69  //go:linkname runtime_nanotime sync.runtime_nanotime
    70  func runtime_nanotime() int64
    71  
    72  // Semacquire waits until *s > 0 and then atomically decrements it.
    73  // It is intended as a simple sleep primitive for use by the synchronization
    74  // library and should not be used directly.
    75  //go:linkname runtime_Semacquire sync.runtime_Semacquire
    76  func runtime_Semacquire(s *uint32)
    77  
    78  // SemacquireMutex is like Semacquire, but for profiling contended Mutexes.
    79  // If lifo is true, queue waiter at the head of wait queue.
    80  // skipframes is the number of frames to omit during tracing, counting from
    81  // runtime_SemacquireMutex's caller.
    82  //go:linkname runtime_SemacquireMutex sync.runtime_SemacquireMutex
    83  func runtime_SemacquireMutex(s *uint32, lifo bool, skipframes int)
    84  
    85  // Semrelease atomically increments *s and notifies a waiting goroutine
    86  // if one is blocked in Semacquire.
    87  // It is intended as a simple wakeup primitive for use by the synchronization
    88  // library and should not be used directly.
    89  // If handoff is true, pass count directly to the first waiter.
    90  // skipframes is the number of frames to omit during tracing, counting from
    91  // runtime_Semrelease's caller.
    92  //go:linkname runtime_Semrelease sync.runtime_Semrelease
    93  func runtime_Semrelease(s *uint32, handoff bool, skipframes int)
    94  
    95  //go:linkname goyield runtime.goyield
    96  func goyield()
    97  
    98  //go:linkname mcall runtime.mcall
    99  func mcall(fn func(unsafe.Pointer))
   100  
   101  //go:linkname park_m runtime.park_m
   102  func park_m(gp unsafe.Pointer)
   103  
   104  //go:linkname fastrandn runtime.fastrandn
   105  func fastrandn(n uint32) uint32
   106  
   107  //go:linkname throw runtime.throw
   108  func throw(s string)
   109  
   110  //go:linkname Readgstatus runtime.readgstatus
   111  func Readgstatus(gp unsafe.Pointer) uint32
   112  
   113  //go:linkname casgstatus runtime.casgstatus
   114  func casgstatus(gp unsafe.Pointer, oldval, newval uint32)
   115  
   116  //go:linkname dropg runtime.dropg
   117  func dropg()
   118  
   119  //go:linkname schedule runtime.schedule
   120  func schedule()
   121  
   122  //go:linkname mallocgc runtime.mallocgc
   123  func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer
   124  
   125  //go:linkname sysFree runtime.sysFree
   126  func sysFree(v unsafe.Pointer, n uintptr, sysStat unsafe.Pointer)
   127  
   128  //go:linkname sysFreeOS runtime.sysFreeOS
   129  func sysFreeOS(v unsafe.Pointer, n uintptr)
   130  
   131  //go:linkname gosched_m runtime.gosched_m
   132  func gosched_m(gp unsafe.Pointer)
   133  
   134  //go:linkname spin runtime.procyield
   135  func spin(cycles uint32)
   136  
   137  //go:linkname noescape runtime.noescape
   138  func noescape(p unsafe.Pointer) unsafe.Pointer
   139  
   140  // ProcPin and ProcUnpin disable pre-emption for any calling goroutine
   141  // can be used to guarantee consistent latency
   142  //go:linkname ProcPin runtime.procPin
   143  func ProcPin() int
   144  
   145  //go:linkname ProcUnpin runtime.procUnpin
   146  func ProcUnpin()
   147  
   148  //go:linkname memequal runtime.memequal
   149  func memequal(a, b unsafe.Pointer, size uintptr) bool
   150  
   151  //go:linkname Load8 runtime/internal/atomic.Load8
   152  func Load8(ptr *uint8) uint8
   153  
   154  //go:linkname And8 runtime/internal/atomic.And8
   155  func And8(ptr *uint8, val uint8)
   156  
   157  //go:linkname Or8 runtime/internal/atomic.Or8
   158  func Or8(ptr *uint8, val uint8)
   159  
   160  //go:linkname Store8 runtime/internal/atomic.Store8
   161  func Store8(ptr *uint8, val uint8)
   162  
   163  // custom parking function
   164  func fast_park(gp unsafe.Pointer) {
   165  	dropg()
   166  	casgstatus(gp, _Grunning, _Gwaiting)
   167  	schedule()
   168  }
   169  
   170  // whether the system has multiple cores or a single core
   171  var multicore = runtime.NumCPU() > 1
   172  
   173  // call ready after ensuring the goroutine is parked
   174  func safe_ready(gp unsafe.Pointer) {
   175  	// for better microprocessor branch prediction
   176  	if multicore {
   177  		for Readgstatus(gp)&^_Gscan != _Gwaiting {
   178  			spin(20)
   179  		}
   180  	} else {
   181  		for Readgstatus(gp)&^_Gscan != _Gwaiting {
   182  			mcall(gosched_m)
   183  		}
   184  	}
   185  	goready(gp, 1)
   186  }
   187  
   188  // simple wait
   189  func wait() {
   190  	if multicore {
   191  		spin(20)
   192  	} else {
   193  		mcall(gosched_m)
   194  	}
   195  }
   196  
   197  type waitReason uint8
   198  
   199  const (
   200  	waitReasonZero                  waitReason = iota // ""
   201  	waitReasonGCAssistMarking                         // "GC assist marking"
   202  	waitReasonIOWait                                  // "IO wait"
   203  	waitReasonChanReceiveNilChan                      // "chan receive (nil chan)"
   204  	waitReasonChanSendNilChan                         // "chan send (nil chan)"
   205  	waitReasonDumpingHeap                             // "dumping heap"
   206  	waitReasonGarbageCollection                       // "garbage collection"
   207  	waitReasonGarbageCollectionScan                   // "garbage collection scan"
   208  	waitReasonPanicWait                               // "panicwait"
   209  	waitReasonSelect                                  // "select"
   210  	waitReasonSelectNoCases                           // "select (no cases)"
   211  	waitReasonGCAssistWait                            // "GC assist wait"
   212  	waitReasonGCSweepWait                             // "GC sweep wait"
   213  	waitReasonGCScavengeWait                          // "GC scavenge wait"
   214  	waitReasonChanReceive                             // "chan receive"
   215  	waitReasonChanSend                                // "chan send"
   216  	waitReasonFinalizerWait                           // "finalizer wait"
   217  	waitReasonForceGCIdle                             // "force gc (idle)"
   218  	waitReasonSemacquire                              // "semacquire"
   219  	waitReasonSleep                                   // "sleep"
   220  	waitReasonSyncCondWait                            // "sync.Cond.Wait"
   221  	waitReasonTimerGoroutineIdle                      // "timer goroutine (idle)"
   222  	waitReasonTraceReaderBlocked                      // "trace reader (blocked)"
   223  	waitReasonWaitForGCCycle                          // "wait for GC cycle"
   224  	waitReasonGCWorkerIdle                            // "GC worker (idle)"
   225  	waitReasonPreempted                               // "preempted"
   226  	waitReasonDebugCall                               // "debug call"
   227  )
   228  
   229  // Event types in the trace, args are given in square brackets.
   230  const (
   231  	traceEvNone              = 0  // unused
   232  	traceEvBatch             = 1  // start of per-P batch of events [pid, timestamp]
   233  	traceEvFrequency         = 2  // contains tracer timer frequency [frequency (ticks per second)]
   234  	traceEvStack             = 3  // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}]
   235  	traceEvGomaxprocs        = 4  // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id]
   236  	traceEvProcStart         = 5  // start of P [timestamp, thread id]
   237  	traceEvProcStop          = 6  // stop of P [timestamp]
   238  	traceEvGCStart           = 7  // GC start [timestamp, seq, stack id]
   239  	traceEvGCDone            = 8  // GC done [timestamp]
   240  	traceEvGCSTWStart        = 9  // GC STW start [timestamp, kind]
   241  	traceEvGCSTWDone         = 10 // GC STW done [timestamp]
   242  	traceEvGCSweepStart      = 11 // GC sweep start [timestamp, stack id]
   243  	traceEvGCSweepDone       = 12 // GC sweep done [timestamp, swept, reclaimed]
   244  	traceEvGoCreate          = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id]
   245  	traceEvGoStart           = 14 // goroutine starts running [timestamp, goroutine id, seq]
   246  	traceEvGoEnd             = 15 // goroutine ends [timestamp]
   247  	traceEvGoStop            = 16 // goroutine stops (like in select{}) [timestamp, stack]
   248  	traceEvGoSched           = 17 // goroutine calls Gosched [timestamp, stack]
   249  	traceEvGoPreempt         = 18 // goroutine is preempted [timestamp, stack]
   250  	traceEvGoSleep           = 19 // goroutine calls Sleep [timestamp, stack]
   251  	traceEvGoBlock           = 20 // goroutine blocks [timestamp, stack]
   252  	traceEvGoUnblock         = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack]
   253  	traceEvGoBlockSend       = 22 // goroutine blocks on chan send [timestamp, stack]
   254  	traceEvGoBlockRecv       = 23 // goroutine blocks on chan recv [timestamp, stack]
   255  	traceEvGoBlockSelect     = 24 // goroutine blocks on select [timestamp, stack]
   256  	traceEvGoBlockSync       = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack]
   257  	traceEvGoBlockCond       = 26 // goroutine blocks on Cond [timestamp, stack]
   258  	traceEvGoBlockNet        = 27 // goroutine blocks on network [timestamp, stack]
   259  	traceEvGoSysCall         = 28 // syscall enter [timestamp, stack]
   260  	traceEvGoSysExit         = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp]
   261  	traceEvGoSysBlock        = 30 // syscall blocks [timestamp]
   262  	traceEvGoWaiting         = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id]
   263  	traceEvGoInSyscall       = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id]
   264  	traceEvHeapAlloc         = 33 // gcController.heapLive change [timestamp, heap_alloc]
   265  	traceEvHeapGoal          = 34 // gcController.heapGoal (formerly next_gc) change [timestamp, heap goal in bytes]
   266  	traceEvTimerGoroutine    = 35 // not currently used; previously denoted timer goroutine [timer goroutine id]
   267  	traceEvFutileWakeup      = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp]
   268  	traceEvString            = 37 // string dictionary entry [ID, length, string]
   269  	traceEvGoStartLocal      = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id]
   270  	traceEvGoUnblockLocal    = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack]
   271  	traceEvGoSysExitLocal    = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp]
   272  	traceEvGoStartLabel      = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id]
   273  	traceEvGoBlockGC         = 42 // goroutine blocks on GC assist [timestamp, stack]
   274  	traceEvGCMarkAssistStart = 43 // GC mark assist start [timestamp, stack]
   275  	traceEvGCMarkAssistDone  = 44 // GC mark assist done [timestamp]
   276  	traceEvUserTaskCreate    = 45 // trace.NewContext [timestamp, internal task id, internal parent task id, stack, name string]
   277  	traceEvUserTaskEnd       = 46 // end of a task [timestamp, internal task id, stack]
   278  	traceEvUserRegion        = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), stack, name string]
   279  	traceEvUserLog           = 48 // trace.Log [timestamp, internal task id, key string id, stack, value string]
   280  	traceEvCount             = 49
   281  	// Byte is used but only 6 bits are available for event type.
   282  	// The remaining 2 bits are used to specify the number of arguments.
   283  	// That means, the max event type value is 63.
   284  )
   285  
   286  // defined constants
   287  const (
   288  	// G status
   289  	//
   290  	// Beyond indicating the general state of a G, the G status
   291  	// acts like a lock on the goroutine's stack (and hence its
   292  	// ability to execute user code).
   293  	//
   294  	// If you add to this list, add to the list
   295  	// of "okay during garbage collection" status
   296  	// in mgcmark.go too.
   297  	//
   298  	// TODO(austin): The _Gscan bit could be much lighter-weight.
   299  	// For example, we could choose not to run _Gscanrunnable
   300  	// goroutines found in the run queue, rather than CAS-looping
   301  	// until they become _Grunnable. And transitions like
   302  	// _Gscanwaiting -> _Gscanrunnable are actually okay because
   303  	// they don't affect stack ownership.
   304  
   305  	// _Gidle means this goroutine was just allocated and has not
   306  	// yet been initialized.
   307  	_Gidle = iota // 0
   308  
   309  	// _Grunnable means this goroutine is on a run queue. It is
   310  	// not currently executing user code. The stack is not owned.
   311  	_Grunnable // 1
   312  
   313  	// _Grunning means this goroutine may execute user code. The
   314  	// stack is owned by this goroutine. It is not on a run queue.
   315  	// It is assigned an M and a P (g.m and g.m.p are valid).
   316  	_Grunning // 2
   317  
   318  	// _Gsyscall means this goroutine is executing a system call.
   319  	// It is not executing user code. The stack is owned by this
   320  	// goroutine. It is not on a run queue. It is assigned an M.
   321  	_Gsyscall // 3
   322  
   323  	// _Gwaiting means this goroutine is blocked in the runtime.
   324  	// It is not executing user code. It is not on a run queue,
   325  	// but should be recorded somewhere (e.g., a channel wait
   326  	// queue) so it can be ready()d when necessary. The stack is
   327  	// not owned *except* that a channel operation may read or
   328  	// write parts of the stack under the appropriate channel
   329  	// lock. Otherwise, it is not safe to access the stack after a
   330  	// goroutine enters _Gwaiting (e.g., it may get moved).
   331  	_Gwaiting // 4
   332  
   333  	// _Gmoribund_unused is currently unused, but hardcoded in gdb
   334  	// scripts.
   335  	_Gmoribund_unused // 5
   336  
   337  	// _Gdead means this goroutine is currently unused. It may be
   338  	// just exited, on a free list, or just being initialized. It
   339  	// is not executing user code. It may or may not have a stack
   340  	// allocated. The G and its stack (if any) are owned by the M
   341  	// that is exiting the G or that obtained the G from the free
   342  	// list.
   343  	_Gdead // 6
   344  
   345  	// _Genqueue_unused is currently unused.
   346  	_Genqueue_unused // 7
   347  
   348  	// _Gcopystack means this goroutine's stack is being moved. It
   349  	// is not executing user code and is not on a run queue. The
   350  	// stack is owned by the goroutine that put it in _Gcopystack.
   351  	_Gcopystack // 8
   352  
   353  	// _Gpreempted means this goroutine stopped itself for a
   354  	// suspendG preemption. It is like _Gwaiting, but nothing is
   355  	// yet responsible for ready()ing it. Some suspendG must CAS
   356  	// the status to _Gwaiting to take responsibility for
   357  	// ready()ing this G.
   358  	_Gpreempted // 9
   359  
   360  	// _Gscan combined with one of the above states other than
   361  	// _Grunning indicates that GC is scanning the stack. The
   362  	// goroutine is not executing user code and the stack is owned
   363  	// by the goroutine that set the _Gscan bit.
   364  	//
   365  	// _Gscanrunning is different: it is used to briefly block
   366  	// state transitions while GC signals the G to scan its own
   367  	// stack. This is otherwise like _Grunning.
   368  	//
   369  	// atomicstatus&~Gscan gives the state the goroutine will
   370  	// return to when the scan completes.
   371  	_Gscan          = 0x1000
   372  	_Gscanrunnable  = _Gscan + _Grunnable  // 0x1001
   373  	_Gscanrunning   = _Gscan + _Grunning   // 0x1002
   374  	_Gscansyscall   = _Gscan + _Gsyscall   // 0x1003
   375  	_Gscanwaiting   = _Gscan + _Gwaiting   // 0x1004
   376  	_Gscanpreempted = _Gscan + _Gpreempted // 0x1009
   377  )