github.com/zartbot/goflow@v0.0.0-20221001080520-456f7dfad288/service/output/elastic.go (about) 1 package output 2 3 import ( 4 "context" 5 "fmt" 6 "log" 7 "math/rand" 8 "time" 9 10 "github.com/olivere/elastic" 11 "github.com/zartbot/goflow/datarecord" 12 ) 13 14 //NewElasticClient is a construct function for Client 15 func NewElasticClient(url string) (*elastic.Client, error) { 16 var ( 17 err error 18 elasticClient *elastic.Client 19 ) 20 for { 21 elasticClient, err = elastic.NewClient( 22 elastic.SetURL(url), 23 elastic.SetSniff(false), 24 ) 25 if err != nil { 26 log.Println(err) 27 time.Sleep(3 * time.Second) 28 } else { 29 break 30 } 31 } 32 return elasticClient, err 33 } 34 35 //ElasticBulkProcessor elastic bulk import processor 36 type ElasticBulkProcessor struct { 37 C *elastic.Client 38 P *elastic.BulkProcessor 39 Workers int 40 IndexPrefix string 41 FlushInterval int 42 StopC chan struct{} // stop channel for caller 43 CreateIdxFlag chan struct{} 44 } 45 46 // Run starts the ElasticBulkProcessor. 47 func (b *ElasticBulkProcessor) Run() error { 48 // Start bulk processor 49 p, err := b.C.BulkProcessor(). 50 Workers(b.Workers). // # of workers 51 BulkActions(5000). // # of queued requests before committed// 52 //BulkSize(1024000000). // # of bytes in requests before committed 53 FlushInterval(time.Duration(b.FlushInterval) * time.Millisecond). // autocommit every interval milliseconds 54 Do(context.Background()) 55 if err != nil { 56 return err 57 } 58 b.P = p 59 // Start indexer that pushes data into bulk processor 60 b.StopC = make(chan struct{}) 61 b.CreateIdxFlag = make(chan struct{}) 62 go b.ensureIndex() 63 <-b.CreateIdxFlag 64 return nil 65 } 66 67 // Close the bulker. 68 func (b *ElasticBulkProcessor) Close() error { 69 b.StopC <- struct{}{} 70 <-b.StopC 71 close(b.StopC) 72 return nil 73 } 74 75 //ElasticRecordProcessor is used bulkupload Record to ElasticSearch Server 76 func (b *ElasticBulkProcessor) ElasticRecordProcessor(dfchan chan *datarecord.DataFrame) { 77 var ( 78 stop bool 79 //rt string 80 //valid bool 81 ) 82 83 for !stop { 84 select { 85 case <-b.StopC: 86 stop = true 87 case d := <-dfchan: 88 exporttime := time.Unix(int64(d.ExportTime), 0).Format("2006.01.02") 89 indexName := b.IndexPrefix + "-" + d.Type + "-" + exporttime 90 //fmt.Println("DEBUG:::", indexName, exporttime) 91 rl := d.RecordList() 92 for _, item := range rl { 93 //rt, valid = item["Type"].(string) 94 //if valid { 95 r := elastic.NewBulkIndexRequest().Index(indexName).Type("log").Doc(item) 96 b.P.Add(r) 97 //fmt.Printf("\n\n\nDEBUG[%s]::BULK-RECORD%+v\n\n\n", indexName, item) 98 //} 99 } 100 // Sleep for a short time. 101 time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) 102 } 103 } 104 b.StopC <- struct{}{} // ack stopping 105 } 106 107 //ElasticImport :Bulk import record to ElasticSearch 108 func ElasticBulkImportProcessor(url string, numWorkers int, indexPrefix string, FlushInterval int) (*ElasticBulkProcessor, error) { 109 elasticClient, err := NewElasticClient(url) 110 if err != nil { 111 log.Fatal(err) 112 } 113 log.Println("ElasticSearch Server connected.....") 114 BulkProcessor := &ElasticBulkProcessor{ 115 C: elasticClient, 116 Workers: numWorkers, 117 IndexPrefix: indexPrefix, 118 FlushInterval: FlushInterval, 119 } 120 err = BulkProcessor.Run() 121 if err != nil { 122 log.Fatal(err) 123 } 124 log.Println("ElasticSearch Bulk Processor is running...") 125 return BulkProcessor, err 126 } 127 128 func (b *ElasticBulkProcessor) ensureIndex() { 129 const date = "2006.01.02" 130 var indexname, nextindexname string 131 startflag := true 132 133 mapping := `{ 134 "settings":{ 135 "index.refresh_interval":"30s", 136 "number_of_shards":3, 137 "number_of_replicas":0 138 }, 139 "mappings":{ 140 "log":{ 141 "properties":{ 142 "Location_A":{ 143 "properties" : { 144 "location": { 145 "type":"geo_point" 146 } 147 } 148 }, 149 "Location_B":{ 150 "properties" : { 151 "location": { 152 "type":"geo_point" 153 } 154 } 155 }, 156 "conn_delay_app_mean" : { 157 "type":"float" 158 }, 159 "conn_delay_client_to_server_mean" : { 160 "type":"float" 161 }, 162 "conn_delay_to_server_mean" : { 163 "type":"float" 164 }, 165 "conn_duration_mean" : { 166 "type":"float" 167 }, 168 "conn_delay_network_to_client_mean" : { 169 "type":"float" 170 }, 171 "conn_delay_network_to_server_mean" : { 172 "type":"float" 173 }, 174 "conn_delay_network_mean" : { 175 "type":"float" 176 }, 177 "conn_delay_network_client_to_server_mean" : { 178 "type":"float" 179 }, 180 "conn_transaction_duration_mean" : { 181 "type":"float" 182 }, 183 "conn_client_pkts_retransmit_rate" : { 184 "type":"float" 185 }, 186 "conn_server_pkts_retransmit_rate" : { 187 "type":"float" 188 }, 189 "conn_client_bytes_retransmit_rate" : { 190 "type":"float" 191 }, 192 "conn_server_bytes_retransmit_rate" : { 193 "type":"float" 194 } 195 } 196 } 197 } 198 }` 199 200 if b.IndexPrefix == "" { 201 log.Fatal("No indexprefix name.") 202 return 203 } 204 for { 205 currentTick := time.Now().Format(date) 206 nextTick := time.Now().Add(time.Duration(3600*24) * time.Second).Format(date) 207 for _, item := range datarecord.RecordTypeList { 208 indexname = fmt.Sprintf("%s-%s-%s", b.IndexPrefix, item, currentTick) 209 nextindexname = fmt.Sprintf("%s-%s-%s", b.IndexPrefix, item, nextTick) 210 //fmt.Println(indexname, nextindexname) 211 212 exists, _ := b.C.IndexExists(indexname).Do(context.Background()) 213 if !exists { 214 log.Println("Creating index:", indexname) 215 _, err := b.C.CreateIndex(indexname).BodyString(mapping).Do(context.Background()) 216 if err != nil { 217 log.Println(err) 218 } 219 } 220 221 nextExists, _ := b.C.IndexExists(nextindexname).Do(context.Background()) 222 if !nextExists { 223 log.Println("Preparing nextday index:", nextindexname) 224 _, err := b.C.CreateIndex(nextindexname).BodyString(mapping).Do(context.Background()) 225 if err != nil { 226 log.Println(err) 227 } 228 } 229 230 } 231 232 if startflag { 233 b.CreateIdxFlag <- struct{}{} 234 startflag = false 235 log.Println("Index Created... ") 236 } 237 238 time.Sleep(120 * time.Second) 239 240 } 241 242 }