github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/sampledataset/sampledataset.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package sampledataset
    18  
    19  import (
    20  	"bufio"
    21  	"bytes"
    22  	"fmt"
    23  
    24  	writer "github.com/siglens/siglens/pkg/es/writer"
    25  	"github.com/siglens/siglens/pkg/usageStats"
    26  	"github.com/siglens/siglens/pkg/utils"
    27  	log "github.com/sirupsen/logrus"
    28  	"github.com/valyala/bytebufferpool"
    29  	"github.com/valyala/fasthttp"
    30  )
    31  
    32  func generateESBody(recs int, actionLine string, rdr Generator,
    33  	bb *bytebufferpool.ByteBuffer) ([]byte, error) {
    34  
    35  	for i := 0; i < recs; i++ {
    36  		_, _ = bb.WriteString(actionLine)
    37  		logline, err := rdr.GetLogLine()
    38  		if err != nil {
    39  			return nil, err
    40  		}
    41  		_, _ = bb.Write(logline)
    42  		_, _ = bb.WriteString("\n")
    43  	}
    44  	payLoad := bb.Bytes()
    45  	return payLoad, nil
    46  }
    47  
    48  func populateActionLines(idxPrefix string, indexName string, numIndices int) []string {
    49  	if numIndices == 0 {
    50  		log.Fatalf("number of indices cannot be zero!")
    51  	}
    52  	actionLines := make([]string, numIndices)
    53  	for i := 0; i < numIndices; i++ {
    54  		var idx string
    55  		if indexName != "" {
    56  			idx = indexName
    57  		} else {
    58  			idx = fmt.Sprintf("%s-%d", idxPrefix, i)
    59  		}
    60  		actionLine := "{\"index\": {\"_index\": \"" + idx + "\", \"_type\": \"_doc\"}}\n"
    61  		actionLines[i] = actionLine
    62  	}
    63  	return actionLines
    64  }
    65  
    66  func ProcessSyntheicDataRequest(ctx *fasthttp.RequestCtx, myid uint64) {
    67  
    68  	actLines := populateActionLines("", "test-data", 1)
    69  
    70  	tsNow := utils.GetCurrentTimeInMs()
    71  	// TODO: init based on dataset selected later
    72  	rdr := InitDynamicUserGenerator(true, 2)
    73  	err := rdr.Init()
    74  	if err != nil {
    75  		log.Errorf("ProcessSyntheicDataRequest: Error in rdr Init: %v", err)
    76  		return
    77  	}
    78  	bb := bytebufferpool.Get()
    79  	payload, err := generateESBody(20000, actLines[0], rdr, bb)
    80  	if err != nil {
    81  		log.Errorf("Error generating bulk body!: %v", err)
    82  		bytebufferpool.Put(bb)
    83  		return
    84  	}
    85  
    86  	r := bytes.NewReader(payload)
    87  	scanner := bufio.NewScanner(r)
    88  	scanner.Split(bufio.ScanLines)
    89  	localIndexMap := make(map[string]string)
    90  
    91  	responsebody := make(map[string]interface{})
    92  	for scanner.Scan() {
    93  		scanner.Scan()
    94  		rawJson := scanner.Bytes()
    95  		numBytes := len(rawJson)
    96  		err = writer.ProcessIndexRequest(rawJson, tsNow, "test-data", uint64(numBytes), false, localIndexMap, myid)
    97  		if err != nil {
    98  			ctx.SetStatusCode(fasthttp.StatusBadRequest)
    99  			responsebody["error"] = err.Error()
   100  			utils.WriteJsonResponse(ctx, responsebody)
   101  			return
   102  		}
   103  		usageStats.UpdateStats(uint64(numBytes), 20000, myid)
   104  	}
   105  
   106  	ctx.SetStatusCode(fasthttp.StatusOK)
   107  	responsebody["message"] = "Successfully ingested 20k lines of logs!"
   108  	utils.WriteJsonResponse(ctx, responsebody)
   109  
   110  }