github.com/futurehomeno/fimpgo@v1.14.0/transport/streams.go (about) 1 package transport 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "github.com/futurehomeno/fimpgo" 7 log "github.com/sirupsen/logrus" 8 "io/ioutil" 9 "strings" 10 "sync" 11 "time" 12 ) 13 //BufferedStream - Implements in memory buffered stream. Underlying buffer is flushed by reaching its max size or based on interval , depends what comes first. 14 // Content of the buffer is flushed either into file or into sink channel. 15 //----------------------------------------------------------------------- 16 // Source ---> buffer -> sink ---> File 17 // |-----> Channel 18 //----------------------------------------------------------------------- 19 type BufferedStream struct { 20 bufferMaxSize int // number of messages 21 bufferInterval time.Duration 22 buffer []fimpgo.FimpMessage 23 lock sync.Mutex 24 flushToFile bool 25 compressBeforeFlush bool 26 filePrefix string 27 flushToSinkChannel bool 28 sinkChannel chan []byte 29 compressor *fimpgo.MsgCompressor 30 ticker *time.Ticker 31 fileSinkDir string 32 } 33 34 func (su *BufferedStream) SinkChannel() chan []byte { 35 return su.sinkChannel 36 } 37 38 func NewBufferedStream(bufferSizeLimit int, bufferInterval time.Duration,compressBeforeFlush bool) *BufferedStream { 39 su := &BufferedStream{bufferMaxSize: bufferSizeLimit, bufferInterval: bufferInterval,compressBeforeFlush: compressBeforeFlush} 40 if su.compressBeforeFlush { 41 su.compressor = fimpgo.NewMsgCompressor("","") 42 } 43 if su.bufferInterval != 0 { 44 su.ticker = time.NewTicker(time.Second * su.bufferInterval) 45 go func() { 46 for _ = range su.ticker.C { 47 if su.Size()>0 { 48 su.FlushBuffer() 49 } 50 } 51 }() 52 } 53 54 return su 55 } 56 57 //SetSourceStream - Configures source channel and starts message processing. Internal loop can be aborted by closing channel 58 func (su *BufferedStream) SetSourceStream(msgCh fimpgo.MessageCh) { 59 go func() { 60 for msg := range msgCh { 61 su.EnqueueMessage(msg.Topic,msg.Payload) 62 } 63 }() 64 } 65 66 67 //EnqueueMessage - must be used to enqueue new message into the stream 68 func (su *BufferedStream) EnqueueMessage(topic string, msg *fimpgo.FimpMessage) { 69 topic = strings.ReplaceAll(topic,"pt:j1/mt:evt","") 70 topic = strings.ReplaceAll(topic,"pt:j1/mt:cmd","") 71 msg.Topic = topic 72 if len(su.buffer) >= su.bufferMaxSize { 73 su.FlushBuffer() 74 } 75 su.lock.Lock() 76 su.buffer = append(su.buffer, *msg) 77 su.lock.Unlock() 78 log.Debugf("Msg queued. Buffer size=%d , maxSize = %d",len(su.buffer),su.bufferMaxSize) 79 } 80 81 func (su *BufferedStream) Size()int { 82 return len(su.buffer) 83 } 84 85 func (su *BufferedStream) FlushBuffer() { 86 //var payload []byte 87 su.lock.Lock() 88 su.serializeBuffer() 89 su.buffer = su.buffer[:0] // setting size to 0 without allocation 90 su.lock.Unlock() 91 } 92 93 func (su *BufferedStream) ConfigureFileSink(filePrefix,path string) { 94 su.flushToFile = true 95 su.filePrefix = filePrefix 96 su.fileSinkDir = path 97 } 98 99 func (su *BufferedStream) ConfigureChanelSink(size int) chan []byte { 100 su.flushToSinkChannel = true 101 su.sinkChannel = make( chan []byte,size) 102 return su.sinkChannel 103 } 104 105 func (su *BufferedStream) serializeBuffer() error { 106 log.Debugf("Serializing stream buffer.size=%d , maxSize = %d",len(su.buffer),su.bufferMaxSize) 107 for i , _ := range su.buffer { 108 if su.buffer[i].ValueType == fimpgo.VTypeObject { 109 su.buffer[i].GetObjectValue(&su.buffer[i].Value) 110 } 111 } 112 bPayload, err := json.Marshal(su.buffer) 113 if err != nil { 114 return err 115 } 116 if su.compressBeforeFlush { 117 bPayload,err = su.compressor.CompressBinMsg(bPayload) 118 if err != nil { 119 log.Error("Compressor error : ",err.Error()) 120 return err 121 } 122 log.Debug("Compressed") 123 } 124 125 if su.flushToFile { 126 fextension := "json" 127 if su.compressBeforeFlush { 128 fextension = "gz" 129 } 130 fname := fmt.Sprintf("%s/%s_%s.%s",su.fileSinkDir,su.filePrefix,time.Now().Format(time.RFC3339),fextension) 131 err := ioutil.WriteFile(fname,bPayload,0777) 132 if err != nil { 133 return err 134 } 135 log.Debug("Serialized to file") 136 } 137 138 if su.flushToSinkChannel { 139 su.sinkChannel <- bPayload 140 } 141 142 return err 143 }