github.com/kubeshop/testkube@v1.17.23/pkg/repository/result/minio_output.go (about) 1 package result 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/json" 7 "fmt" 8 "io" 9 10 "go.mongodb.org/mongo-driver/bson" 11 "go.mongodb.org/mongo-driver/mongo" 12 13 "github.com/kubeshop/testkube/pkg/api/v1/testkube" 14 "github.com/kubeshop/testkube/pkg/log" 15 "github.com/kubeshop/testkube/pkg/storage" 16 "github.com/kubeshop/testkube/pkg/storage/minio" 17 ) 18 19 var _ OutputRepository = (*MinioRepository)(nil) 20 21 type MinioRepository struct { 22 storage storage.Client 23 executionCollection *mongo.Collection 24 bucket string 25 } 26 27 func NewMinioOutputRepository(storageClient storage.Client, executionCollection *mongo.Collection, bucket string) *MinioRepository { 28 log.DefaultLogger.Debugw("creating minio output repository", "bucket", bucket) 29 return &MinioRepository{ 30 storage: storageClient, 31 executionCollection: executionCollection, 32 bucket: bucket, 33 } 34 } 35 36 func (m *MinioRepository) GetOutput(ctx context.Context, id, testName, testSuiteName string) (output string, err error) { 37 eOutput, err := m.getOutput(ctx, id) 38 if err != nil { 39 return "", err 40 } 41 return eOutput.Output, err 42 } 43 44 func (m *MinioRepository) getOutput(ctx context.Context, id string) (ExecutionOutput, error) { 45 file, _, err := m.storage.DownloadFileFromBucket(ctx, m.bucket, "", id) 46 if err != nil && err == minio.ErrArtifactsNotFound { 47 log.DefaultLogger.Infow("output not found in minio", "id", id) 48 return ExecutionOutput{}, nil 49 } 50 if err != nil { 51 return ExecutionOutput{}, fmt.Errorf("error downloading output logs from minio: %w", err) 52 } 53 var eOutput ExecutionOutput 54 decoder := json.NewDecoder(file) 55 err = decoder.Decode(&eOutput) 56 if err != nil { 57 return ExecutionOutput{}, err 58 } 59 return eOutput, err 60 } 61 62 func (m *MinioRepository) saveOutput(ctx context.Context, eOutput ExecutionOutput) error { 63 data, err := json.Marshal(eOutput) 64 if err != nil { 65 return err 66 } 67 reader := bytes.NewReader(data) 68 err = m.storage.UploadFileToBucket(ctx, m.bucket, "", eOutput.Id, reader, reader.Size()) 69 return err 70 } 71 72 func (m *MinioRepository) InsertOutput(ctx context.Context, id, testName, testSuiteName, output string) error { 73 log.DefaultLogger.Debugw("inserting output", "id", id, "testName", testName, "testSuiteName", testSuiteName) 74 eOutput := ExecutionOutput{Id: id, Name: id, TestName: testName, TestSuiteName: testSuiteName, Output: output} 75 return m.saveOutput(ctx, eOutput) 76 } 77 78 func (m *MinioRepository) UpdateOutput(ctx context.Context, id, testName, testSuiteName, output string) error { 79 log.DefaultLogger.Debugw("updating output", "id", id) 80 eOutput, err := m.getOutput(ctx, id) 81 if err != nil { 82 return err 83 } 84 eOutput.Output = output 85 return m.saveOutput(ctx, eOutput) 86 } 87 88 func (m *MinioRepository) DeleteOutput(ctx context.Context, id, testName, testSuiteName string) error { 89 log.DefaultLogger.Debugw("deleting output", "id", id) 90 return m.storage.DeleteFileFromBucket(ctx, m.bucket, "", id) 91 } 92 93 func (m *MinioRepository) DeleteOutputByTest(ctx context.Context, testName string) error { 94 log.DefaultLogger.Debugw("deleting output by test", "testName", testName) 95 var executions []testkube.Execution 96 cursor, err := m.executionCollection.Find(ctx, bson.M{"testname": testName}) 97 if err != nil { 98 return err 99 } 100 err = cursor.All(ctx, &executions) 101 if err != nil { 102 return err 103 } 104 for _, execution := range executions { 105 log.DefaultLogger.Debugw("deleting output for execution", "execution", execution) 106 err = m.DeleteOutput(ctx, execution.Id, testName, "") 107 if err != nil { 108 return err 109 } 110 } 111 return nil 112 } 113 114 func (m *MinioRepository) DeleteOutputForTests(ctx context.Context, testNames []string) error { 115 log.DefaultLogger.Debugw("deleting output for tests", "testNames", testNames) 116 for _, testName := range testNames { 117 err := m.DeleteOutputByTest(ctx, testName) 118 if err != nil { 119 return err 120 } 121 } 122 return nil 123 } 124 125 func (m *MinioRepository) DeleteOutputByTestSuite(ctx context.Context, testSuiteName string) error { 126 var executions []testkube.Execution 127 cursor, err := m.executionCollection.Find(ctx, bson.M{"testsuitename": testSuiteName}) 128 if err != nil { 129 return err 130 } 131 err = cursor.All(ctx, &executions) 132 if err != nil { 133 return err 134 } 135 for _, execution := range executions { 136 err = m.DeleteOutput(ctx, execution.Id, "", testSuiteName) 137 if err != nil { 138 return err 139 } 140 } 141 return nil 142 } 143 144 func (m *MinioRepository) DeleteOutputForTestSuites(ctx context.Context, testSuiteNames []string) error { 145 for _, testSuiteName := range testSuiteNames { 146 err := m.DeleteOutputByTestSuite(ctx, testSuiteName) 147 if err != nil { 148 return err 149 } 150 } 151 152 return nil 153 } 154 155 func (m *MinioRepository) DeleteOutputForAllTestSuite(ctx context.Context) error { 156 var executions []testkube.Execution 157 cursor, err := m.executionCollection.Find(ctx, bson.M{"testsuitename": bson.M{"$ne": ""}}) 158 if err != nil { 159 return err 160 } 161 err = cursor.All(ctx, &executions) 162 if err != nil { 163 return err 164 } 165 for _, execution := range executions { 166 err = m.DeleteOutput(ctx, execution.Id, "", "") 167 if err != nil { 168 return err 169 } 170 } 171 return nil 172 } 173 174 func (m *MinioRepository) DeleteAllOutput(ctx context.Context) error { 175 err := m.storage.DeleteBucket(ctx, m.bucket, true) 176 if err != nil { 177 return err 178 } 179 return m.storage.CreateBucket(ctx, m.bucket) 180 } 181 182 func (m *MinioRepository) StreamOutput(ctx context.Context, executionID, testName, testSuiteName string) (reader io.Reader, err error) { 183 file, _, err := m.storage.DownloadFileFromBucket(ctx, m.bucket, "", executionID) 184 if err != nil { 185 return nil, err 186 } 187 return file, nil 188 } 189 190 func (m *MinioRepository) GetOutputSize(ctx context.Context, executionID, testName, testSuiteName string) (size int, err error) { 191 //TODO: improve with minio client 192 stream, err := m.StreamOutput(ctx, executionID, testName, testSuiteName) 193 if err != nil { 194 return 0, err 195 } 196 const bufferSize = 1024 197 buf := make([]byte, bufferSize) 198 for { 199 n, err := stream.Read(buf) 200 if err != nil { 201 if err == io.EOF { 202 break 203 } 204 return 0, err 205 } 206 size += n 207 } 208 return size, nil 209 }