github.com/Jeffail/benthos/v3@v3.65.0/internal/checkpoint/type.go (about)

     1  package checkpoint
     2  
     3  import (
     4  	"sync"
     5  )
     6  
     7  // Type keeps track of a sequence of pending checkpoint payloads, and as pending
     8  // checkpoints are resolved it retains the latest fully resolved payload in the
     9  // sequence where all prior sequence checkpoints are also resolved.
    10  //
    11  // Also keeps track of the logical size of the unresolved sequence, which allows
    12  // for limiting the number of pending checkpoints.
    13  type Type struct {
    14  	positionOffset int64
    15  	checkpoint     interface{}
    16  
    17  	latest, earliest *node
    18  }
    19  
    20  // New returns a new checkpointer.
    21  func New() *Type {
    22  	return &Type{}
    23  }
    24  
    25  // Track a new unresolved payload. This payload will be cached until it is
    26  // marked as resolved. While it is cached no more recent payload will ever be
    27  // committed.
    28  //
    29  // While the returned resolve funcs can be called from any goroutine, it
    30  // is assumed that Track is called from a single goroutine.
    31  func (t *Type) Track(payload interface{}, batchSize int64) func() interface{} {
    32  	newNode := getNode()
    33  	newNode.payload = payload
    34  	newNode.position = batchSize
    35  
    36  	if t.earliest == nil {
    37  		t.earliest = newNode
    38  	}
    39  
    40  	if t.latest != nil {
    41  		newNode.prev = t.latest
    42  		newNode.position += t.latest.position
    43  		t.latest.next = newNode
    44  	}
    45  
    46  	t.latest = newNode
    47  
    48  	return func() interface{} {
    49  		if newNode.prev != nil {
    50  			newNode.prev.position = newNode.position
    51  			newNode.prev.payload = newNode.payload
    52  			newNode.prev.next = newNode.next
    53  		} else {
    54  			t.checkpoint = newNode.payload
    55  			t.positionOffset = newNode.position
    56  			t.earliest = newNode.next
    57  		}
    58  
    59  		if newNode.next != nil {
    60  			newNode.next.prev = newNode.prev
    61  		} else {
    62  			t.latest = newNode.prev
    63  			if t.latest == nil {
    64  				t.positionOffset = 0
    65  			}
    66  		}
    67  
    68  		putNode(newNode)
    69  		return t.checkpoint
    70  	}
    71  }
    72  
    73  // Pending returns the gap between the earliest and latests unresolved messages.
    74  func (t *Type) Pending() int64 {
    75  	if t.latest == nil {
    76  		return 0
    77  	}
    78  	return t.latest.position - t.positionOffset
    79  }
    80  
    81  // Highest returns the payload of the highest resolved checkpoint.
    82  func (t *Type) Highest() interface{} {
    83  	return t.checkpoint
    84  }
    85  
    86  type node struct {
    87  	position   int64
    88  	payload    interface{}
    89  	prev, next *node
    90  }
    91  
    92  var nodePool = &sync.Pool{
    93  	New: func() interface{} {
    94  		return &node{}
    95  	},
    96  }
    97  
    98  func getNode() *node {
    99  	return nodePool.Get().(*node)
   100  }
   101  
   102  func putNode(node *node) {
   103  	node.position = 0
   104  	node.payload = nil
   105  	node.prev = nil
   106  	node.next = nil
   107  	nodePool.Put(node)
   108  }