github.com/kubeshop/testkube@v1.17.23/pkg/logs/adapter/minio_v2.go (about)

     1  package adapter
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"os"
     7  	"path/filepath"
     8  	"sync"
     9  
    10  	"github.com/minio/minio-go/v7"
    11  	"go.uber.org/zap"
    12  
    13  	"github.com/kubeshop/testkube/pkg/log"
    14  	"github.com/kubeshop/testkube/pkg/logs/events"
    15  	minioconnecter "github.com/kubeshop/testkube/pkg/storage/minio"
    16  )
    17  
    18  // DefaultDataDir is a default directory where logs are stored (logs-service Dockerfile creates this directory)
    19  const DefaultDataDir = "/data"
    20  
    21  var _ Adapter = &MinioV2Adapter{}
    22  
    23  // NewMinioV2Adapter creates new MinioV2Adapter which will send data to local MinIO bucket
    24  func NewMinioV2Adapter(endpoint, accessKeyID, secretAccessKey, region, token, bucket string, ssl, skipVerify bool, certFile, keyFile, caFile string) (*MinioV2Adapter, error) {
    25  	ctx := context.Background()
    26  	opts := minioconnecter.GetTLSOptions(ssl, skipVerify, certFile, keyFile, caFile)
    27  	c := &MinioV2Adapter{
    28  		minioConnecter: minioconnecter.NewConnecter(endpoint, accessKeyID, secretAccessKey, region, token, bucket, log.DefaultLogger, opts...),
    29  		log:            log.DefaultLogger,
    30  		bucket:         bucket,
    31  		region:         region,
    32  		files:          make(map[string]*os.File),
    33  		path:           DefaultDataDir,
    34  	}
    35  	minioClient, err := c.minioConnecter.GetClient()
    36  	if err != nil {
    37  		c.log.Errorw("error connecting to minio", "err", err)
    38  		return c, err
    39  	}
    40  
    41  	c.minioClient = minioClient
    42  	exists, err := c.minioClient.BucketExists(ctx, c.bucket)
    43  	if err != nil {
    44  		c.log.Errorw("error checking if bucket exists", "err", err)
    45  		return c, err
    46  	}
    47  
    48  	if !exists {
    49  		err = c.minioClient.MakeBucket(ctx, c.bucket,
    50  			minio.MakeBucketOptions{Region: c.region})
    51  		if err != nil {
    52  			c.log.Errorw("error creating bucket", "err", err)
    53  			return c, err
    54  		}
    55  	}
    56  	return c, nil
    57  }
    58  
    59  type MinioV2Adapter struct {
    60  	minioConnecter *minioconnecter.Connecter
    61  	minioClient    *minio.Client
    62  	bucket         string
    63  	region         string
    64  	log            *zap.SugaredLogger
    65  	traceMessages  bool
    66  	lock           sync.RWMutex
    67  	path           string
    68  	files          map[string]*os.File
    69  }
    70  
    71  func (s *MinioV2Adapter) Init(ctx context.Context, id string) error {
    72  	filePath := filepath.Join(s.path, id)
    73  
    74  	file, err := os.Create(filePath)
    75  	if err != nil {
    76  		return err
    77  	}
    78  
    79  	s.putFile(id, file)
    80  
    81  	return nil
    82  }
    83  
    84  func (s *MinioV2Adapter) putFile(id string, f *os.File) {
    85  	s.lock.Lock()
    86  	defer s.lock.Unlock()
    87  	s.files[id] = f
    88  }
    89  
    90  func (s *MinioV2Adapter) countFiles() int {
    91  	s.lock.RLock()
    92  	defer s.lock.RUnlock()
    93  	return len(s.files)
    94  }
    95  
    96  func (s *MinioV2Adapter) getFile(id string) (f *os.File, err error) {
    97  	s.lock.RLock()
    98  	defer s.lock.RUnlock()
    99  	file, ok := s.files[id]
   100  	if !ok {
   101  		return nil, os.ErrNotExist
   102  	}
   103  
   104  	return file, nil
   105  }
   106  
   107  func (s *MinioV2Adapter) deleteFile(id string) {
   108  	s.lock.Lock()
   109  	defer s.lock.Unlock()
   110  	delete(s.files, id)
   111  }
   112  
   113  func (s *MinioV2Adapter) WithTraceMessages(enabled bool) *MinioV2Adapter {
   114  	s.traceMessages = enabled
   115  	return s
   116  }
   117  
   118  func (s *MinioV2Adapter) WithPath(path string) {
   119  	s.path = path
   120  }
   121  
   122  func (s *MinioV2Adapter) Notify(ctx context.Context, id string, e events.Log) error {
   123  	if s.traceMessages {
   124  		s.log.Debugw("minio consumer notify", "id", id, "event", e)
   125  	}
   126  
   127  	chunk, err := json.Marshal(e)
   128  	if err != nil {
   129  		return err
   130  	}
   131  
   132  	file, err := s.getFile(id)
   133  	if err != nil {
   134  		return err
   135  	}
   136  
   137  	_, err = file.Write(append(chunk, []byte("\n")...))
   138  
   139  	return err
   140  }
   141  
   142  func (s *MinioV2Adapter) Stop(ctx context.Context, id string) error {
   143  	log := s.log.With("id", id)
   144  
   145  	log.Debugw("stopping minio consumer")
   146  
   147  	file, err := s.getFile(id)
   148  	if err != nil {
   149  		return err
   150  	}
   151  
   152  	// rewind file to the beginning
   153  	_, err = file.Seek(0, 0)
   154  	if err != nil {
   155  		return err
   156  	}
   157  
   158  	stat, err := file.Stat()
   159  	if err != nil {
   160  		return err
   161  	}
   162  
   163  	info, err := s.minioClient.PutObject(ctx, s.bucket, id, file, stat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
   164  	if err != nil {
   165  		log.Errorw("error putting object", "err", err)
   166  		return err
   167  	}
   168  
   169  	log.Debugw("put object successfully", "id", id, "s.bucket", s.bucket, "uploadInfo", info)
   170  
   171  	// clean memory
   172  	err = file.Close()
   173  
   174  	if err != nil {
   175  		return err
   176  	}
   177  
   178  	s.deleteFile(id)
   179  
   180  	err = os.Remove(filepath.Join(s.path, id))
   181  	if err != nil {
   182  		return err
   183  	}
   184  
   185  	log.Debugw("minio consumer stopped, tmp file removed", "filesInUse", s.countFiles())
   186  
   187  	return nil
   188  }
   189  
   190  func (s *MinioV2Adapter) Name() string {
   191  	return "minio-v2"
   192  }