github.com/futurehomeno/fimpgo@v1.14.0/transport/streams.go (about)

     1  package transport
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"github.com/futurehomeno/fimpgo"
     7  	log "github.com/sirupsen/logrus"
     8  	"io/ioutil"
     9  	"strings"
    10  	"sync"
    11  	"time"
    12  )
    13  //BufferedStream - Implements in memory buffered stream. Underlying  buffer is flushed by reaching its max size or based on interval , depends what comes first.
    14  // Content of the buffer is flushed either into file or into sink channel.
    15  //-----------------------------------------------------------------------
    16  // Source ---> buffer -> sink ---> File
    17  //                         |-----> Channel
    18  //-----------------------------------------------------------------------
    19  type BufferedStream struct {
    20  	bufferMaxSize       int // number of messages
    21  	bufferInterval      time.Duration
    22  	buffer              []fimpgo.FimpMessage
    23  	lock                sync.Mutex
    24  	flushToFile         bool
    25  	compressBeforeFlush bool
    26  	filePrefix          string
    27  	flushToSinkChannel  bool
    28  	sinkChannel         chan []byte
    29  	compressor          *fimpgo.MsgCompressor
    30  	ticker              *time.Ticker
    31  	fileSinkDir         string
    32  }
    33  
    34  func (su *BufferedStream) SinkChannel() chan []byte {
    35  	return su.sinkChannel
    36  }
    37  
    38  func NewBufferedStream(bufferSizeLimit int, bufferInterval time.Duration,compressBeforeFlush bool) *BufferedStream {
    39  	su := &BufferedStream{bufferMaxSize: bufferSizeLimit, bufferInterval: bufferInterval,compressBeforeFlush: compressBeforeFlush}
    40  	if su.compressBeforeFlush {
    41  		su.compressor = fimpgo.NewMsgCompressor("","")
    42  	}
    43  	if su.bufferInterval != 0 {
    44  		su.ticker = time.NewTicker(time.Second * su.bufferInterval)
    45  		go func() {
    46  			for _ = range su.ticker.C {
    47  				if su.Size()>0 {
    48  					su.FlushBuffer()
    49  				}
    50  			}
    51  		}()
    52  	}
    53  
    54  	return su
    55  }
    56  
    57  //SetSourceStream - Configures source channel and starts message processing. Internal loop can be aborted by closing channel
    58  func (su *BufferedStream) SetSourceStream(msgCh fimpgo.MessageCh) {
    59  	go func() {
    60  		for msg := range msgCh {
    61  			su.EnqueueMessage(msg.Topic,msg.Payload)
    62  		}
    63  	}()
    64  }
    65  
    66  
    67  //EnqueueMessage - must be used to enqueue new message into the stream
    68  func (su *BufferedStream) EnqueueMessage(topic string, msg *fimpgo.FimpMessage) {
    69  	topic = strings.ReplaceAll(topic,"pt:j1/mt:evt","")
    70  	topic = strings.ReplaceAll(topic,"pt:j1/mt:cmd","")
    71  	msg.Topic = topic
    72  	if len(su.buffer) >= su.bufferMaxSize {
    73  		su.FlushBuffer()
    74  	}
    75  	su.lock.Lock()
    76  	su.buffer = append(su.buffer, *msg)
    77  	su.lock.Unlock()
    78  	log.Debugf("Msg queued. Buffer size=%d , maxSize = %d",len(su.buffer),su.bufferMaxSize)
    79  }
    80  
    81  func (su *BufferedStream) Size()int {
    82  	return len(su.buffer)
    83  }
    84  
    85  func (su *BufferedStream) FlushBuffer() {
    86  	//var payload []byte
    87  	su.lock.Lock()
    88  	su.serializeBuffer()
    89  	su.buffer = su.buffer[:0] // setting size to 0 without allocation
    90  	su.lock.Unlock()
    91  }
    92  
    93  func (su *BufferedStream) ConfigureFileSink(filePrefix,path string) {
    94  	su.flushToFile = true
    95  	su.filePrefix = filePrefix
    96  	su.fileSinkDir = path
    97  }
    98  
    99  func (su *BufferedStream) ConfigureChanelSink(size int) chan []byte {
   100  	su.flushToSinkChannel = true
   101  	su.sinkChannel = make( chan []byte,size)
   102  	return su.sinkChannel
   103  }
   104  
   105  func (su *BufferedStream) serializeBuffer() error {
   106  	log.Debugf("Serializing stream buffer.size=%d , maxSize = %d",len(su.buffer),su.bufferMaxSize)
   107  	for i , _ := range su.buffer {
   108  		if su.buffer[i].ValueType == fimpgo.VTypeObject {
   109  			su.buffer[i].GetObjectValue(&su.buffer[i].Value)
   110  		}
   111  	}
   112  	bPayload, err := json.Marshal(su.buffer)
   113  	if err != nil {
   114  		return err
   115  	}
   116  	if su.compressBeforeFlush {
   117  		bPayload,err = su.compressor.CompressBinMsg(bPayload)
   118  		if err != nil {
   119  			log.Error("Compressor error : ",err.Error())
   120  			return err
   121  		}
   122  		log.Debug("Compressed")
   123  	}
   124  
   125  	if su.flushToFile {
   126  		fextension := "json"
   127  		if su.compressBeforeFlush {
   128  			fextension = "gz"
   129  		}
   130  		fname := fmt.Sprintf("%s/%s_%s.%s",su.fileSinkDir,su.filePrefix,time.Now().Format(time.RFC3339),fextension)
   131  		err := ioutil.WriteFile(fname,bPayload,0777)
   132  		if err != nil {
   133  			return err
   134  		}
   135  		log.Debug("Serialized to file")
   136  	}
   137  
   138  	if su.flushToSinkChannel {
   139  		su.sinkChannel <- bPayload
   140  	}
   141  
   142  	return err
   143  }