github.com/kubeshop/testkube@v1.17.23/pkg/logs/repository/minio.go (about)

     1  package repository
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"context"
     7  	"encoding/json"
     8  	"errors"
     9  	"io"
    10  	"strings"
    11  	"time"
    12  
    13  	"go.uber.org/zap"
    14  
    15  	"github.com/kubeshop/testkube/pkg/log"
    16  	"github.com/kubeshop/testkube/pkg/logs/events"
    17  	"github.com/kubeshop/testkube/pkg/repository/result"
    18  	"github.com/kubeshop/testkube/pkg/storage"
    19  	"github.com/kubeshop/testkube/pkg/utils"
    20  )
    21  
    22  const (
    23  	defaultBufferSize = 100
    24  	logsV1Prefix      = "{\"id\""
    25  )
    26  
    27  func NewMinioRepository(storageClient storage.ClientBucket, bucket string) LogsRepository {
    28  	return MinioLogsRepository{
    29  		storageClient: storageClient,
    30  		log:           log.DefaultLogger,
    31  		bucket:        bucket,
    32  	}
    33  }
    34  
    35  type MinioLogsRepository struct {
    36  	storageClient storage.ClientBucket
    37  	log           *zap.SugaredLogger
    38  	bucket        string
    39  }
    40  
    41  func (r MinioLogsRepository) Get(ctx context.Context, id string) (chan events.LogResponse, error) {
    42  	file, info, err := r.storageClient.DownloadFileFromBucket(ctx, r.bucket, "", id)
    43  	if err != nil {
    44  		r.log.Errorw("error downloading log file from bucket", "error", err, "bucket", r.bucket, "id", id)
    45  		return nil, err
    46  	}
    47  
    48  	ch := make(chan events.LogResponse, defaultBufferSize)
    49  
    50  	go func() {
    51  		defer close(ch)
    52  
    53  		buffer, version, err := r.readLineLogsV2(file, ch)
    54  		if err != nil {
    55  			ch <- events.LogResponse{Error: err}
    56  			return
    57  		}
    58  
    59  		if version == events.LogVersionV1 {
    60  			if err = r.readLineLogsV1(ch, buffer, info.LastModified); err != nil {
    61  				ch <- events.LogResponse{Error: err}
    62  				return
    63  			}
    64  		}
    65  	}()
    66  
    67  	return ch, nil
    68  }
    69  
    70  func (r MinioLogsRepository) readLineLogsV2(file io.Reader, ch chan events.LogResponse) ([]byte, events.LogVersion, error) {
    71  	var buffer []byte
    72  	reader := bufio.NewReader(file)
    73  	firstLine := false
    74  	version := events.LogVersionV2
    75  	for {
    76  		b, err := utils.ReadLongLine(reader)
    77  		if err != nil {
    78  			if errors.Is(err, io.EOF) {
    79  				break
    80  			}
    81  
    82  			r.log.Errorw("error getting log line", "error", err)
    83  			return nil, "", err
    84  		}
    85  
    86  		if !firstLine {
    87  			firstLine = true
    88  			if strings.HasPrefix(string(b), logsV1Prefix) {
    89  				version = events.LogVersionV1
    90  			}
    91  		}
    92  
    93  		if version == events.LogVersionV1 {
    94  			buffer = append(buffer, b...)
    95  		}
    96  
    97  		if version == events.LogVersionV2 {
    98  			var log events.Log
    99  			err = json.Unmarshal(b, &log)
   100  			if err != nil {
   101  				r.log.Errorw("error unmarshalling log line", "error", err)
   102  				ch <- events.LogResponse{Error: err}
   103  				continue
   104  			}
   105  
   106  			ch <- events.LogResponse{Log: log}
   107  		}
   108  	}
   109  
   110  	return buffer, version, nil
   111  }
   112  
   113  func (r MinioLogsRepository) readLineLogsV1(ch chan events.LogResponse, buffer []byte, logTime time.Time) error {
   114  	var output result.ExecutionOutput
   115  	decoder := json.NewDecoder(bytes.NewBuffer(buffer))
   116  	err := decoder.Decode(&output)
   117  	if err != nil {
   118  		r.log.Errorw("error decoding logs", "error", err)
   119  		return err
   120  	}
   121  
   122  	reader := bufio.NewReader(bytes.NewBuffer([]byte(output.Output)))
   123  	for {
   124  		b, err := utils.ReadLongLine(reader)
   125  		if err != nil {
   126  			if errors.Is(err, io.EOF) {
   127  				break
   128  			}
   129  
   130  			r.log.Errorw("error getting log line", "error", err)
   131  			return err
   132  		}
   133  
   134  		ch <- events.LogResponse{Log: events.Log{
   135  			Time:    logTime,
   136  			Content: string(b),
   137  			Version: string(events.LogVersionV1),
   138  		}}
   139  	}
   140  
   141  	return nil
   142  }