github.com/haraldrudell/parl@v0.4.176/echo-moderator.go (about)

     1  /*
     2  © 2022–present Harald Rudell <harald.rudell@gmail.com> (https://haraldrudell.github.io/haraldrudell/)
     3  ISC License
     4  */
     5  
     6  package parl
     7  
     8  import (
     9  	"math"
    10  	"strconv"
    11  	"sync/atomic"
    12  	"time"
    13  
    14  	"github.com/haraldrudell/parl/ptime"
    15  )
    16  
    17  const (
    18  	// invocations of less invocation that 10 ms are not reported
    19  	minLatencyWarningPoint = 10 * time.Millisecond
    20  )
    21  
    22  var echoModeratorID atomic.Uint64
    23  
    24  type mcReturnTicket func()
    25  
    26  // EchoModerator is a parallelism-limiting Moderator that:
    27  //   - prints any increase in parallelism over the concurrency value
    28  //   - prints exhibited invocation slowness exceeding latencyWarningPoint
    29  //   - prints progressive slowness exceeding latencyWarningPoint for
    30  //     non-returning invocations in progress on schedule timerPeriod
    31  //   - EchoModerator can be used in production for ongoing diagnose of apis and
    32  //     libraries
    33  //   - cost is one thread, one timer, and a locked linked-list of invocations
    34  //   - —
    35  //   - EchoModerator is intended to control and diagnose [exec.Command] invocations
    36  //   - problems include:
    37  //   - — too many parallel invocations
    38  //   - — invocations that do not return or are long running
    39  //   - — too many threads held waiting to invoke
    40  //   - — unexpected behavior under load
    41  //   - — deviating behavior when operated for extended periods of time
    42  type EchoModerator struct {
    43  	// moderator limits parallelism
    44  	moderator ModeratorCore
    45  	// label preceds all printouts, default is “echoModerator1”
    46  	label string
    47  	// waiting causes printout if too many threads are waiting at the moderator
    48  	waiting AtomicMax[uint64]
    49  	log     PrintfFunc
    50  	// examines individual invocations
    51  	invocationTimer InvocationTimer[mcReturnTicket]
    52  }
    53  
    54  // NewEchoModerator returns a parallelism-limiting moderator with printouts for
    55  // excessive slowness or parallelism
    56  //   - concurrency is the highest number of executions that can take place in parallel
    57  //   - printout on:
    58  //   - — too many threads waiting at the moderator
    59  //   - — too slow or hung invocations
    60  //   - stores self-referencing pointers
    61  func NewEchoModerator(
    62  	concurrency uint64,
    63  	latencyWarningPoint time.Duration,
    64  	waitingWarningPoint uint64,
    65  	timerPeriod time.Duration,
    66  	label string, goGen GoGen, log PrintfFunc,
    67  ) (echoModerator *EchoModerator) {
    68  	if latencyWarningPoint < minLatencyWarningPoint {
    69  		latencyWarningPoint = minLatencyWarningPoint
    70  	}
    71  	if label == "" {
    72  		label = "echoModerator" + strconv.Itoa(int(echoModeratorID.Add(1)))
    73  	}
    74  	m := EchoModerator{
    75  		moderator: *NewModeratorCore(concurrency),
    76  		label:     label,
    77  		log:       log,
    78  		waiting:   *NewAtomicMax(waitingWarningPoint),
    79  	}
    80  	m.invocationTimer = *NewInvocationTimer[mcReturnTicket](
    81  		m.loggingCallback, m.returnMcTicket,
    82  		latencyWarningPoint,
    83  		// no parallelism warnings
    84  		//	- instead warning on too many threads waiting at moderator
    85  		math.MaxUint64,
    86  		timerPeriod, goGen,
    87  	)
    88  	return &m
    89  }
    90  
    91  // Ticket waits for a EchoModerator ticket and provides a function to return it
    92  //
    93  //	func moderatedFunc() {
    94  //	  defer echoModerator.Ticket()()
    95  func (m *EchoModerator) Ticket() (returnTicket func()) {
    96  
    97  	// if highest pending request, log that
    98  	if _, _, waiting := m.moderator.Status(); m.waiting.Value(waiting) {
    99  		age, threadID := m.invocationTimer.Oldest()
   100  		var threadStr string
   101  		if threadID.IsValid() {
   102  			threadStr = "oldest thread ID: " + threadID.String()
   103  		}
   104  		m.log("%s new waiting threads max: %d slowest operation: %s%s",
   105  			m.label, waiting+1, ptime.Duration(age), threadStr)
   106  	}
   107  
   108  	// blocks here
   109  	var ticketReturn mcReturnTicket = m.moderator.Ticket()
   110  
   111  	// hand the ticket return to invocation
   112  	//	- to avoid additional object creation, invocation
   113  	//		will safekeep the ticket return and provide it
   114  	//		via callback at end of invocation
   115  	//	- invocation will invoke returnMcTicket with it
   116  	returnTicket = m.invocationTimer.Invocation(ticketReturn)
   117  	return
   118  }
   119  
   120  // returnMcTicket receives tickets to be returned from an ending Invocation
   121  func (m *EchoModerator) returnMcTicket(ticketReturn mcReturnTicket) {
   122  	ticketReturn()
   123  }
   124  
   125  // loggingCallback logs output from invocationTimer
   126  //   - there is also logging in Ticket
   127  func (m *EchoModerator) loggingCallback(
   128  	reason CBReason,
   129  	maxParallelism uint64,
   130  	maxLatency time.Duration,
   131  	threadID ThreadID) {
   132  	m.log("%s %s: max parallelism: %d max latency: %s goroutine-ID: %s",
   133  		m.label, reason, maxParallelism, maxLatency, threadID)
   134  }