github.com/weedge/lib@v0.0.0-20230424045628-a36dcc1d90e4/asyncbuffer/batch.go (about)

     1  package asyncbuffer
     2  
     3  //@todo: add send obj sequence id
     4  
     5  import (
     6  	"fmt"
     7  	"runtime/debug"
     8  	"sync/atomic"
     9  	"unsafe"
    10  
    11  	"time"
    12  
    13  	"github.com/weedge/lib/log"
    14  )
    15  
    16  func (sd *SendData) AddBufferItem(item *InputBufferItem) (err error) {
    17  	ch, ok := sd.MapDataCh[item.ChName]
    18  	if ok == false {
    19  		err = fmt.Errorf("bufferName: %s chName: %s don't exist", sd.BufferName, item.ChName)
    20  		return
    21  	}
    22  	if item == nil {
    23  		err = fmt.Errorf("bufferName: %s AddBufferItem InputBufferItem is nil", sd.BufferName)
    24  		return
    25  	}
    26  	sd.ISendObj = item.Data
    27  	err, res := sd.ISendObj.FormatInput()
    28  	if err != nil {
    29  		return
    30  	}
    31  	ch <- res
    32  
    33  	return
    34  }
    35  
    36  func (sd *SendData) InitAsyncSub(chName string, workerNum int) {
    37  	for i := 1; i <= workerNum; i++ {
    38  		sd.asyncSubFromOneCh(chName)
    39  	}
    40  }
    41  
    42  func (sd *SendData) initFlushTicker(buffName string) {
    43  	go func() {
    44  		for {
    45  			now := time.Now()
    46  			beginTs := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).Unix()
    47  			endTs := beginTs + 86400 - 1
    48  			//0:0:10
    49  			afterTs := endTs - now.Unix() + 10
    50  			afterTs = afterTs % 86400
    51  			if afterTs <= 0 {
    52  				afterTs = 86400
    53  			}
    54  
    55  			select {
    56  			case <-time.After(3 * time.Second):
    57  				//println(buffName, "flushTicker")
    58  				log.Debugf("%s flushTicker", buffName)
    59  				sd.BufferSend([]byte{})
    60  			case <-time.After(time.Duration(afterTs) * time.Second):
    61  				//println(buffName, "counterTicker dayBufferCounter", sd.BufferDayCounter, "clear")
    62  				log.Infof("%s dayBufferCounter:%d clear~!", buffName, sd.BufferDayCounter)
    63  				sd.counterClear()
    64  			}
    65  		}
    66  	}()
    67  }
    68  
    69  func (sd *SendData) asyncSubFromOneCh(chName string) {
    70  	go func(name string) {
    71  		defer func() {
    72  			if err := recover(); err != nil {
    73  				panicBuffer := ""
    74  				if sd != nil && sd.BufferData != nil {
    75  					for _, item := range sd.BufferData {
    76  						panicBuffer += *(*string)(unsafe.Pointer(&item))
    77  					}
    78  				}
    79  				if sd != nil {
    80  					sd.BufferIndex = 0
    81  				}
    82  
    83  				log.Errorf("bufferName: %s AsyncSubFromOneCh chName[%s] panicBuffer[%s] panic recovered err[%s] stack[%s]", sd.BufferName, chName, panicBuffer, err, string(debug.Stack()))
    84  				//println("panicBuffer-->:", panicBuffer)
    85  				//println("stack:", string(debug.Stack()))
    86  				time.Sleep(200 * time.Millisecond)
    87  				sd.asyncSubFromOneCh(name)
    88  			}
    89  		}()
    90  		sd.subFromOneCh(name)
    91  	}(chName)
    92  }
    93  
    94  func (sd *SendData) counter() {
    95  	atomic.AddUint64(&sd.BufferDayCounter, 1)
    96  }
    97  func (sd *SendData) counterClear() {
    98  	atomic.StoreUint64(&sd.BufferDayCounter, 0)
    99  }
   100  
   101  func (sd *SendData) subFromOneCh(chName string) {
   102  	log.Infof("bufferName: %s subFromOneCh select ch chName:%s bufferWindowSize: %d delaySendTimeMS: %d", sd.BufferName, chName, sd.BufferWindowSize, sd.DelaySendTime)
   103  	ch, ok := sd.MapDataCh[chName]
   104  	if ok == false {
   105  		return
   106  	}
   107  	for {
   108  		select {
   109  		case data, ok := <-ch:
   110  			if ok == false { //close
   111  				log.Infof("chName: %s close continue", chName)
   112  				time.Sleep(3 * time.Second)
   113  				continue
   114  			}
   115  
   116  			//println("getSendDataFrom data:", *(*string)(unsafe.Pointer(&data)))
   117  			log.Infof("getSendDataFrom ch:%s data: %s", chName, *(*string)(unsafe.Pointer(&data)))
   118  			sd.counter()
   119  			sd.BufferSend(data)
   120  		case isFlush, ok := <-sd.IsFlushCh:
   121  			if ok == false { //close
   122  				log.Infof("IsFlushCh close continue")
   123  				time.Sleep(3 * time.Second)
   124  				continue
   125  			}
   126  			log.Debugf("bufferName: %s IsFlushCh send isFlush: %v start flush~!", sd.BufferName, isFlush)
   127  			if isFlush {
   128  				sd.BufferSend([]byte{})
   129  			}
   130  		}
   131  	}
   132  }
   133  
   134  // one sendData -> multi pub and batch sub
   135  func (sd *SendData) batchSubFromCh() {
   136  	log.Debugf("batchSubFromCh")
   137  	for chName := range sd.MapDataCh {
   138  		sd.subFromOneCh(chName)
   139  	}
   140  }
   141  
   142  func (sd *SendData) BufferSend(data []byte) {
   143  	sd.OpLock.Lock()
   144  	defer sd.OpLock.Unlock()
   145  
   146  	if len(data) > 0 {
   147  		sd.BufferData[atomic.LoadInt64(&sd.BufferIndex)] = data
   148  		atomic.AddInt64(&sd.BufferIndex, 1)
   149  	}
   150  
   151  	if atomic.LoadInt64(&sd.BufferIndex) == int64(sd.BufferWindowSize) || len(data) == 0 {
   152  		sd.flush()
   153  	}
   154  
   155  	return
   156  }
   157  
   158  //notice: if close ch panic
   159  func (sd *SendData) FlushBuffer() {
   160  	sd.IsFlushCh <- true
   161  }
   162  
   163  func (sd *SendData) flush() {
   164  	if atomic.LoadInt64(&sd.BufferIndex) <= 0 {
   165  		//log.Infof("bufferName: %s un flush~!", sd.BufferName)
   166  		return
   167  	}
   168  	if sd.ISendObj == nil {
   169  		log.Errorf("bufferName: %s sd.ISendObj is nil, flush fail~!", sd.BufferName)
   170  		return
   171  	}
   172  
   173  	time.Sleep(time.Duration(sd.DelaySendTime) * time.Millisecond)
   174  	sd.ISendObj.BatchDo(sd.BufferData[0:atomic.LoadInt64(&sd.BufferIndex)])
   175  	atomic.StoreInt64(&sd.BufferIndex, 0)
   176  }