github.com/zartbot/goflow@v0.0.0-20221001080520-456f7dfad288/service/output/elastic.go (about)

     1  package output
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"log"
     7  	"math/rand"
     8  	"time"
     9  
    10  	"github.com/olivere/elastic"
    11  	"github.com/zartbot/goflow/datarecord"
    12  )
    13  
    14  //NewElasticClient is a construct function for Client
    15  func NewElasticClient(url string) (*elastic.Client, error) {
    16  	var (
    17  		err           error
    18  		elasticClient *elastic.Client
    19  	)
    20  	for {
    21  		elasticClient, err = elastic.NewClient(
    22  			elastic.SetURL(url),
    23  			elastic.SetSniff(false),
    24  		)
    25  		if err != nil {
    26  			log.Println(err)
    27  			time.Sleep(3 * time.Second)
    28  		} else {
    29  			break
    30  		}
    31  	}
    32  	return elasticClient, err
    33  }
    34  
    35  //ElasticBulkProcessor elastic bulk import processor
    36  type ElasticBulkProcessor struct {
    37  	C             *elastic.Client
    38  	P             *elastic.BulkProcessor
    39  	Workers       int
    40  	IndexPrefix   string
    41  	FlushInterval int
    42  	StopC         chan struct{} // stop channel for caller
    43  	CreateIdxFlag chan struct{}
    44  }
    45  
    46  // Run starts the ElasticBulkProcessor.
    47  func (b *ElasticBulkProcessor) Run() error {
    48  	// Start bulk processor
    49  	p, err := b.C.BulkProcessor().
    50  		Workers(b.Workers). // # of workers
    51  		BulkActions(5000).  // # of queued requests before committed//
    52  		//BulkSize(1024000000).                                             // # of bytes in requests before committed
    53  		FlushInterval(time.Duration(b.FlushInterval) * time.Millisecond). // autocommit every interval milliseconds
    54  		Do(context.Background())
    55  	if err != nil {
    56  		return err
    57  	}
    58  	b.P = p
    59  	// Start indexer that pushes data into bulk processor
    60  	b.StopC = make(chan struct{})
    61  	b.CreateIdxFlag = make(chan struct{})
    62  	go b.ensureIndex()
    63  	<-b.CreateIdxFlag
    64  	return nil
    65  }
    66  
    67  // Close the bulker.
    68  func (b *ElasticBulkProcessor) Close() error {
    69  	b.StopC <- struct{}{}
    70  	<-b.StopC
    71  	close(b.StopC)
    72  	return nil
    73  }
    74  
    75  //ElasticRecordProcessor is used bulkupload Record to ElasticSearch Server
    76  func (b *ElasticBulkProcessor) ElasticRecordProcessor(dfchan chan *datarecord.DataFrame) {
    77  	var (
    78  		stop bool
    79  		//rt    string
    80  		//valid bool
    81  	)
    82  
    83  	for !stop {
    84  		select {
    85  		case <-b.StopC:
    86  			stop = true
    87  		case d := <-dfchan:
    88  			exporttime := time.Unix(int64(d.ExportTime), 0).Format("2006.01.02")
    89  			indexName := b.IndexPrefix + "-" + d.Type + "-" + exporttime
    90  			//fmt.Println("DEBUG:::", indexName, exporttime)
    91  			rl := d.RecordList()
    92  			for _, item := range rl {
    93  				//rt, valid = item["Type"].(string)
    94  				//if valid {
    95  				r := elastic.NewBulkIndexRequest().Index(indexName).Type("log").Doc(item)
    96  				b.P.Add(r)
    97  				//fmt.Printf("\n\n\nDEBUG[%s]::BULK-RECORD%+v\n\n\n", indexName, item)
    98  				//}
    99  			}
   100  			// Sleep for a short time.
   101  			time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond)
   102  		}
   103  	}
   104  	b.StopC <- struct{}{} // ack stopping
   105  }
   106  
   107  //ElasticImport :Bulk import record to ElasticSearch
   108  func ElasticBulkImportProcessor(url string, numWorkers int, indexPrefix string, FlushInterval int) (*ElasticBulkProcessor, error) {
   109  	elasticClient, err := NewElasticClient(url)
   110  	if err != nil {
   111  		log.Fatal(err)
   112  	}
   113  	log.Println("ElasticSearch Server connected.....")
   114  	BulkProcessor := &ElasticBulkProcessor{
   115  		C:             elasticClient,
   116  		Workers:       numWorkers,
   117  		IndexPrefix:   indexPrefix,
   118  		FlushInterval: FlushInterval,
   119  	}
   120  	err = BulkProcessor.Run()
   121  	if err != nil {
   122  		log.Fatal(err)
   123  	}
   124  	log.Println("ElasticSearch Bulk Processor is running...")
   125  	return BulkProcessor, err
   126  }
   127  
   128  func (b *ElasticBulkProcessor) ensureIndex() {
   129  	const date = "2006.01.02"
   130  	var indexname, nextindexname string
   131  	startflag := true
   132  
   133  	mapping := `{
   134  		"settings":{
   135              "index.refresh_interval":"30s",
   136  			"number_of_shards":3,
   137  			"number_of_replicas":0
   138  		},
   139          "mappings":{
   140              "log":{
   141                  "properties":{
   142                      "Location_A":{
   143                          "properties" : {
   144                              "location": {
   145                                  "type":"geo_point"
   146                              }
   147                          }
   148                      },
   149                      "Location_B":{
   150                          "properties" : {
   151                              "location": {
   152                                  "type":"geo_point"
   153                              }
   154                          }
   155  					},
   156  					"conn_delay_app_mean" : {
   157  						"type":"float"
   158  					},
   159  					"conn_delay_client_to_server_mean" : {
   160  						"type":"float"
   161  					},
   162  					"conn_delay_to_server_mean" : {
   163  						"type":"float"
   164  					},
   165  					"conn_duration_mean" : {
   166  						"type":"float"
   167  					},
   168  					"conn_delay_network_to_client_mean" : {
   169  						"type":"float"
   170  					},
   171  					"conn_delay_network_to_server_mean" : {
   172  						"type":"float"
   173  					},
   174  					"conn_delay_network_mean" : {
   175  						"type":"float"
   176  					},
   177  					"conn_delay_network_client_to_server_mean" : {
   178  						"type":"float"
   179  					},
   180  					"conn_transaction_duration_mean" : {
   181  						"type":"float"
   182  					},
   183  					"conn_client_pkts_retransmit_rate" : {
   184  						"type":"float"
   185  					},
   186  					"conn_server_pkts_retransmit_rate" : {
   187  						"type":"float"
   188  					},
   189  					"conn_client_bytes_retransmit_rate" : {
   190  						"type":"float"
   191  					},
   192  					"conn_server_bytes_retransmit_rate" : {
   193  						"type":"float"
   194  					}
   195                  }
   196              }
   197  		}
   198      }`
   199  
   200  	if b.IndexPrefix == "" {
   201  		log.Fatal("No indexprefix name.")
   202  		return
   203  	}
   204  	for {
   205  		currentTick := time.Now().Format(date)
   206  		nextTick := time.Now().Add(time.Duration(3600*24) * time.Second).Format(date)
   207  		for _, item := range datarecord.RecordTypeList {
   208  			indexname = fmt.Sprintf("%s-%s-%s", b.IndexPrefix, item, currentTick)
   209  			nextindexname = fmt.Sprintf("%s-%s-%s", b.IndexPrefix, item, nextTick)
   210  			//fmt.Println(indexname, nextindexname)
   211  
   212  			exists, _ := b.C.IndexExists(indexname).Do(context.Background())
   213  			if !exists {
   214  				log.Println("Creating index:", indexname)
   215  				_, err := b.C.CreateIndex(indexname).BodyString(mapping).Do(context.Background())
   216  				if err != nil {
   217  					log.Println(err)
   218  				}
   219  			}
   220  
   221  			nextExists, _ := b.C.IndexExists(nextindexname).Do(context.Background())
   222  			if !nextExists {
   223  				log.Println("Preparing nextday index:", nextindexname)
   224  				_, err := b.C.CreateIndex(nextindexname).BodyString(mapping).Do(context.Background())
   225  				if err != nil {
   226  					log.Println(err)
   227  				}
   228  			}
   229  
   230  		}
   231  
   232  		if startflag {
   233  			b.CreateIdxFlag <- struct{}{}
   234  			startflag = false
   235  			log.Println("Index Created... ")
   236  		}
   237  
   238  		time.Sleep(120 * time.Second)
   239  
   240  	}
   241  
   242  }