github.com/unicornultrafoundation/go-u2u@v1.0.0-rc1.0.20240205080301-e74a83d3fadc/gossip/protocols/epochpacks/epprocessor/processor.go (about)

     1  package epprocessor
     2  
     3  import (
     4  	"errors"
     5  	"sync"
     6  
     7  	"github.com/unicornultrafoundation/go-helios/native/dag"
     8  	"github.com/unicornultrafoundation/go-helios/native/idx"
     9  	"github.com/unicornultrafoundation/go-helios/utils/datasemaphore"
    10  	"github.com/unicornultrafoundation/go-helios/utils/workers"
    11  
    12  	"github.com/unicornultrafoundation/go-u2u/native"
    13  	"github.com/unicornultrafoundation/go-u2u/native/iep"
    14  	"github.com/unicornultrafoundation/go-u2u/native/ier"
    15  )
    16  
    17  var (
    18  	ErrBusy = errors.New("failed to acquire events semaphore")
    19  )
    20  
    21  // Processor is responsible for processing incoming events
    22  type Processor struct {
    23  	cfg Config
    24  
    25  	quit chan struct{}
    26  	wg   sync.WaitGroup
    27  
    28  	callback Callback
    29  
    30  	inserter *workers.Workers
    31  	checker  *workers.Workers
    32  
    33  	itemsSemaphore *datasemaphore.DataSemaphore
    34  }
    35  
    36  type ItemCallback struct {
    37  	ProcessEV  func(ev native.LlrSignedEpochVote) error
    38  	ProcessER  func(er ier.LlrIdxFullEpochRecord) error
    39  	ReleasedEV func(ev native.LlrSignedEpochVote, peer string, err error)
    40  	ReleasedER func(er ier.LlrIdxFullEpochRecord, peer string, err error)
    41  	CheckEV    func(ev native.LlrSignedEpochVote, checked func(error))
    42  }
    43  
    44  type Callback struct {
    45  	Item ItemCallback
    46  }
    47  
    48  // New creates an event processor
    49  func New(itemsSemaphore *datasemaphore.DataSemaphore, cfg Config, callback Callback) *Processor {
    50  	f := &Processor{
    51  		cfg:            cfg,
    52  		quit:           make(chan struct{}),
    53  		itemsSemaphore: itemsSemaphore,
    54  		callback:       callback,
    55  	}
    56  	f.callback = callback
    57  	f.inserter = workers.New(&f.wg, f.quit, cfg.MaxTasks)
    58  	f.checker = workers.New(&f.wg, f.quit, cfg.MaxTasks)
    59  	return f
    60  }
    61  
    62  // Start boots up the items processor.
    63  func (f *Processor) Start() {
    64  	f.inserter.Start(1)
    65  	f.checker.Start(1)
    66  }
    67  
    68  // Stop interrupts the processor, canceling all the pending operations.
    69  // Stop waits until all the internal goroutines have finished.
    70  func (f *Processor) Stop() {
    71  	close(f.quit)
    72  	f.itemsSemaphore.Terminate()
    73  	f.wg.Wait()
    74  }
    75  
    76  // Overloaded returns true if too much items are being processed
    77  func (f *Processor) Overloaded() bool {
    78  	return f.TasksCount() > f.cfg.MaxTasks*3/4
    79  }
    80  
    81  type checkRes struct {
    82  	ev  native.LlrSignedEpochVote
    83  	err error
    84  	pos idx.Event
    85  }
    86  
    87  func (f *Processor) Enqueue(peer string, eps []iep.LlrEpochPack, totalSize uint64, done func()) error {
    88  	if len(eps) == 0 {
    89  		if done != nil {
    90  			done()
    91  		}
    92  		return nil
    93  	}
    94  	metric := dag.Metric{Num: idx.Event(len(eps)), Size: totalSize}
    95  	if !f.itemsSemaphore.Acquire(metric, f.cfg.SemaphoreTimeout) {
    96  		return ErrBusy
    97  	}
    98  
    99  	// process each EpochPack as a separate task
   100  	return f.inserter.Enqueue(func() {
   101  		if done != nil {
   102  			defer done()
   103  		}
   104  		defer f.itemsSemaphore.Release(metric)
   105  		for _, ep := range eps {
   106  			items := ep.Votes
   107  			record := ep.Record
   108  			checkedC := make(chan *checkRes, len(items))
   109  			err := f.checker.Enqueue(func() {
   110  				for i, v := range items {
   111  					pos := idx.Event(i)
   112  					ev := v
   113  					f.callback.Item.CheckEV(ev, func(err error) {
   114  						checkedC <- &checkRes{
   115  							ev:  ev,
   116  							err: err,
   117  							pos: pos,
   118  						}
   119  					})
   120  				}
   121  			})
   122  			if err != nil {
   123  				return
   124  			}
   125  			itemsLen := len(items)
   126  			var orderedResults = make([]*checkRes, itemsLen)
   127  			var processed int
   128  			for processed < itemsLen {
   129  				select {
   130  				case res := <-checkedC:
   131  					orderedResults[res.pos] = res
   132  
   133  					for i := processed; processed < len(orderedResults) && orderedResults[i] != nil; i++ {
   134  						f.processEV(peer, orderedResults[i].ev, orderedResults[i].err)
   135  						orderedResults[i] = nil // free the memory
   136  						processed++
   137  					}
   138  
   139  				case <-f.quit:
   140  					return
   141  				}
   142  			}
   143  			f.processER(peer, record)
   144  		}
   145  	})
   146  }
   147  
   148  func (f *Processor) processEV(peer string, ev native.LlrSignedEpochVote, resErr error) {
   149  	// release item if failed validation
   150  	if resErr != nil {
   151  		f.callback.Item.ReleasedEV(ev, peer, resErr)
   152  		return
   153  	}
   154  	// process item
   155  	err := f.callback.Item.ProcessEV(ev)
   156  	f.callback.Item.ReleasedEV(ev, peer, err)
   157  }
   158  
   159  func (f *Processor) processER(peer string, er ier.LlrIdxFullEpochRecord) {
   160  	// process item
   161  	err := f.callback.Item.ProcessER(er)
   162  	f.callback.Item.ReleasedER(er, peer, err)
   163  }
   164  
   165  func (f *Processor) TasksCount() int {
   166  	return f.inserter.TasksCount() + f.checker.TasksCount()
   167  }