github.com/kubeshop/testkube@v1.17.23/pkg/repository/result/minio_output.go (about)

     1  package result
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/json"
     7  	"fmt"
     8  	"io"
     9  
    10  	"go.mongodb.org/mongo-driver/bson"
    11  	"go.mongodb.org/mongo-driver/mongo"
    12  
    13  	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
    14  	"github.com/kubeshop/testkube/pkg/log"
    15  	"github.com/kubeshop/testkube/pkg/storage"
    16  	"github.com/kubeshop/testkube/pkg/storage/minio"
    17  )
    18  
    19  var _ OutputRepository = (*MinioRepository)(nil)
    20  
    21  type MinioRepository struct {
    22  	storage             storage.Client
    23  	executionCollection *mongo.Collection
    24  	bucket              string
    25  }
    26  
    27  func NewMinioOutputRepository(storageClient storage.Client, executionCollection *mongo.Collection, bucket string) *MinioRepository {
    28  	log.DefaultLogger.Debugw("creating minio output repository", "bucket", bucket)
    29  	return &MinioRepository{
    30  		storage:             storageClient,
    31  		executionCollection: executionCollection,
    32  		bucket:              bucket,
    33  	}
    34  }
    35  
    36  func (m *MinioRepository) GetOutput(ctx context.Context, id, testName, testSuiteName string) (output string, err error) {
    37  	eOutput, err := m.getOutput(ctx, id)
    38  	if err != nil {
    39  		return "", err
    40  	}
    41  	return eOutput.Output, err
    42  }
    43  
    44  func (m *MinioRepository) getOutput(ctx context.Context, id string) (ExecutionOutput, error) {
    45  	file, _, err := m.storage.DownloadFileFromBucket(ctx, m.bucket, "", id)
    46  	if err != nil && err == minio.ErrArtifactsNotFound {
    47  		log.DefaultLogger.Infow("output not found in minio", "id", id)
    48  		return ExecutionOutput{}, nil
    49  	}
    50  	if err != nil {
    51  		return ExecutionOutput{}, fmt.Errorf("error downloading output logs from minio: %w", err)
    52  	}
    53  	var eOutput ExecutionOutput
    54  	decoder := json.NewDecoder(file)
    55  	err = decoder.Decode(&eOutput)
    56  	if err != nil {
    57  		return ExecutionOutput{}, err
    58  	}
    59  	return eOutput, err
    60  }
    61  
    62  func (m *MinioRepository) saveOutput(ctx context.Context, eOutput ExecutionOutput) error {
    63  	data, err := json.Marshal(eOutput)
    64  	if err != nil {
    65  		return err
    66  	}
    67  	reader := bytes.NewReader(data)
    68  	err = m.storage.UploadFileToBucket(ctx, m.bucket, "", eOutput.Id, reader, reader.Size())
    69  	return err
    70  }
    71  
    72  func (m *MinioRepository) InsertOutput(ctx context.Context, id, testName, testSuiteName, output string) error {
    73  	log.DefaultLogger.Debugw("inserting output", "id", id, "testName", testName, "testSuiteName", testSuiteName)
    74  	eOutput := ExecutionOutput{Id: id, Name: id, TestName: testName, TestSuiteName: testSuiteName, Output: output}
    75  	return m.saveOutput(ctx, eOutput)
    76  }
    77  
    78  func (m *MinioRepository) UpdateOutput(ctx context.Context, id, testName, testSuiteName, output string) error {
    79  	log.DefaultLogger.Debugw("updating output", "id", id)
    80  	eOutput, err := m.getOutput(ctx, id)
    81  	if err != nil {
    82  		return err
    83  	}
    84  	eOutput.Output = output
    85  	return m.saveOutput(ctx, eOutput)
    86  }
    87  
    88  func (m *MinioRepository) DeleteOutput(ctx context.Context, id, testName, testSuiteName string) error {
    89  	log.DefaultLogger.Debugw("deleting output", "id", id)
    90  	return m.storage.DeleteFileFromBucket(ctx, m.bucket, "", id)
    91  }
    92  
    93  func (m *MinioRepository) DeleteOutputByTest(ctx context.Context, testName string) error {
    94  	log.DefaultLogger.Debugw("deleting output by test", "testName", testName)
    95  	var executions []testkube.Execution
    96  	cursor, err := m.executionCollection.Find(ctx, bson.M{"testname": testName})
    97  	if err != nil {
    98  		return err
    99  	}
   100  	err = cursor.All(ctx, &executions)
   101  	if err != nil {
   102  		return err
   103  	}
   104  	for _, execution := range executions {
   105  		log.DefaultLogger.Debugw("deleting output for execution", "execution", execution)
   106  		err = m.DeleteOutput(ctx, execution.Id, testName, "")
   107  		if err != nil {
   108  			return err
   109  		}
   110  	}
   111  	return nil
   112  }
   113  
   114  func (m *MinioRepository) DeleteOutputForTests(ctx context.Context, testNames []string) error {
   115  	log.DefaultLogger.Debugw("deleting output for tests", "testNames", testNames)
   116  	for _, testName := range testNames {
   117  		err := m.DeleteOutputByTest(ctx, testName)
   118  		if err != nil {
   119  			return err
   120  		}
   121  	}
   122  	return nil
   123  }
   124  
   125  func (m *MinioRepository) DeleteOutputByTestSuite(ctx context.Context, testSuiteName string) error {
   126  	var executions []testkube.Execution
   127  	cursor, err := m.executionCollection.Find(ctx, bson.M{"testsuitename": testSuiteName})
   128  	if err != nil {
   129  		return err
   130  	}
   131  	err = cursor.All(ctx, &executions)
   132  	if err != nil {
   133  		return err
   134  	}
   135  	for _, execution := range executions {
   136  		err = m.DeleteOutput(ctx, execution.Id, "", testSuiteName)
   137  		if err != nil {
   138  			return err
   139  		}
   140  	}
   141  	return nil
   142  }
   143  
   144  func (m *MinioRepository) DeleteOutputForTestSuites(ctx context.Context, testSuiteNames []string) error {
   145  	for _, testSuiteName := range testSuiteNames {
   146  		err := m.DeleteOutputByTestSuite(ctx, testSuiteName)
   147  		if err != nil {
   148  			return err
   149  		}
   150  	}
   151  
   152  	return nil
   153  }
   154  
   155  func (m *MinioRepository) DeleteOutputForAllTestSuite(ctx context.Context) error {
   156  	var executions []testkube.Execution
   157  	cursor, err := m.executionCollection.Find(ctx, bson.M{"testsuitename": bson.M{"$ne": ""}})
   158  	if err != nil {
   159  		return err
   160  	}
   161  	err = cursor.All(ctx, &executions)
   162  	if err != nil {
   163  		return err
   164  	}
   165  	for _, execution := range executions {
   166  		err = m.DeleteOutput(ctx, execution.Id, "", "")
   167  		if err != nil {
   168  			return err
   169  		}
   170  	}
   171  	return nil
   172  }
   173  
   174  func (m *MinioRepository) DeleteAllOutput(ctx context.Context) error {
   175  	err := m.storage.DeleteBucket(ctx, m.bucket, true)
   176  	if err != nil {
   177  		return err
   178  	}
   179  	return m.storage.CreateBucket(ctx, m.bucket)
   180  }
   181  
   182  func (m *MinioRepository) StreamOutput(ctx context.Context, executionID, testName, testSuiteName string) (reader io.Reader, err error) {
   183  	file, _, err := m.storage.DownloadFileFromBucket(ctx, m.bucket, "", executionID)
   184  	if err != nil {
   185  		return nil, err
   186  	}
   187  	return file, nil
   188  }
   189  
   190  func (m *MinioRepository) GetOutputSize(ctx context.Context, executionID, testName, testSuiteName string) (size int, err error) {
   191  	//TODO: improve with minio client
   192  	stream, err := m.StreamOutput(ctx, executionID, testName, testSuiteName)
   193  	if err != nil {
   194  		return 0, err
   195  	}
   196  	const bufferSize = 1024
   197  	buf := make([]byte, bufferSize)
   198  	for {
   199  		n, err := stream.Read(buf)
   200  		if err != nil {
   201  			if err == io.EOF {
   202  				break
   203  			}
   204  			return 0, err
   205  		}
   206  		size += n
   207  	}
   208  	return size, nil
   209  }