github.com/kubeshop/testkube@v1.17.23/pkg/executor/scraper/minio_scraper_integration_test.go (about)

     1  package scraper_test
     2  
     3  import (
     4  	"context"
     5  	"net/http"
     6  	"net/http/httptest"
     7  	"os"
     8  	"path/filepath"
     9  	"testing"
    10  
    11  	"github.com/kubeshop/testkube/pkg/utils/test"
    12  
    13  	cloudevents "github.com/cloudevents/sdk-go/v2"
    14  	"github.com/golang/mock/gomock"
    15  	"github.com/stretchr/testify/assert"
    16  	"github.com/stretchr/testify/require"
    17  
    18  	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
    19  	"github.com/kubeshop/testkube/pkg/executor/scraper"
    20  	"github.com/kubeshop/testkube/pkg/filesystem"
    21  	"github.com/kubeshop/testkube/pkg/storage/minio"
    22  )
    23  
    24  func TestMinIOScraper_Archive_Integration(t *testing.T) {
    25  	test.IntegrationTest(t)
    26  	t.Parallel()
    27  
    28  	mockCtrl := gomock.NewController(t)
    29  	defer mockCtrl.Finish()
    30  
    31  	tempDir, err := os.MkdirTemp("", "test")
    32  	require.NoError(t, err)
    33  
    34  	defer os.RemoveAll(tempDir)
    35  
    36  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
    37  	require.NoError(t, err)
    38  
    39  	file1 := filepath.Join(tempDir, "file1.txt")
    40  	file2 := filepath.Join(tempDir, "file2.txt")
    41  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
    42  
    43  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
    44  	assert.NoError(t, err)
    45  
    46  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
    47  	assert.NoError(t, err)
    48  
    49  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
    50  	assert.NoError(t, err)
    51  
    52  	extractor := scraper.NewArchiveFilesystemExtractor(filesystem.NewOSFileSystem())
    53  
    54  	loader, err := scraper.NewMinIOUploader(
    55  		"localhost:9000",
    56  		"minio99",
    57  		"minio123",
    58  		"us-east-1",
    59  		"",
    60  		"test-bucket-asdf",
    61  		false,
    62  		false,
    63  		"",
    64  		"",
    65  		"",
    66  	)
    67  	if err != nil {
    68  		t.Fatalf("error creating minio loader: %v", err)
    69  	}
    70  
    71  	execution := testkube.Execution{Id: "minio-test"}
    72  
    73  	// given
    74  	testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    75  		_, err := cloudevents.NewEventFromHTTPRequest(r)
    76  		// then
    77  		assert.NoError(t, err)
    78  	})
    79  
    80  	svr := httptest.NewServer(testHandler)
    81  	defer svr.Close()
    82  
    83  	client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
    84  	assert.NoError(t, err)
    85  
    86  	s := scraper.NewExtractLoadScraper(extractor, loader, client, "", "")
    87  	err = s.Scrape(context.Background(), []string{tempDir}, []string{".*"}, execution)
    88  	if err != nil {
    89  		t.Fatalf("error scraping: %v", err)
    90  	}
    91  
    92  	c := minio.NewClient("localhost:9000", "minio99", "minio123", "us-east-1", "", "test-bucket-asdf")
    93  	assert.NoError(t, c.Connect())
    94  	artifacts, err := c.ListFiles(context.Background(), "test-bucket-asdf")
    95  	if err != nil {
    96  		t.Fatalf("error listing files from bucket: %v", err)
    97  	}
    98  	assert.True(t, containsArtifact(t, artifacts, "minio-test/file1.txt"))
    99  	assert.True(t, containsArtifact(t, artifacts, "minio-test/file2.txt"))
   100  	assert.True(t, containsArtifact(t, artifacts, "minio-test/subdir/file3.txt"))
   101  }
   102  
   103  func TestMinIOScraper_Recursive_Integration(t *testing.T) {
   104  	test.IntegrationTest(t)
   105  	t.Parallel()
   106  
   107  	mockCtrl := gomock.NewController(t)
   108  	defer mockCtrl.Finish()
   109  
   110  	tempDir, err := os.MkdirTemp("", "test")
   111  	require.NoError(t, err)
   112  
   113  	defer os.RemoveAll(tempDir)
   114  
   115  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
   116  	require.NoError(t, err)
   117  
   118  	file1 := filepath.Join(tempDir, "file1.txt")
   119  	file2 := filepath.Join(tempDir, "file2.txt")
   120  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
   121  
   122  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
   123  	assert.NoError(t, err)
   124  
   125  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
   126  	assert.NoError(t, err)
   127  
   128  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
   129  	assert.NoError(t, err)
   130  
   131  	extractor := scraper.NewRecursiveFilesystemExtractor(filesystem.NewOSFileSystem())
   132  
   133  	bucketName := "test-bucket-asdf1"
   134  	loader, err := scraper.NewMinIOUploader(
   135  		"localhost:9000",
   136  		"minio99",
   137  		"minio123",
   138  		"us-east-1",
   139  		"",
   140  		bucketName,
   141  		false,
   142  		false,
   143  		"",
   144  		"",
   145  		"",
   146  	)
   147  	if err != nil {
   148  		t.Fatalf("error creating minio loader: %v", err)
   149  	}
   150  
   151  	execution := testkube.Execution{Id: "minio-test"}
   152  
   153  	// given
   154  	testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   155  		_, err := cloudevents.NewEventFromHTTPRequest(r)
   156  		// then
   157  		assert.NoError(t, err)
   158  	})
   159  
   160  	svr := httptest.NewServer(testHandler)
   161  	defer svr.Close()
   162  
   163  	client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
   164  	assert.NoError(t, err)
   165  
   166  	s := scraper.NewExtractLoadScraper(extractor, loader, client, "", "")
   167  	err = s.Scrape(context.Background(), []string{tempDir}, []string{".*"}, execution)
   168  	if err != nil {
   169  		t.Fatalf("error scraping: %v", err)
   170  	}
   171  
   172  	c := minio.NewClient("localhost:9000", "minio99", "minio123", "us-east-1", "", bucketName)
   173  	assert.NoError(t, c.Connect())
   174  	artifacts, err := c.ListFiles(context.Background(), bucketName)
   175  	if err != nil {
   176  		t.Fatalf("error listing files from bucket: %v", err)
   177  	}
   178  	assert.True(t, containsArtifact(t, artifacts, "minio-test/file1.txt"))
   179  	assert.True(t, containsArtifact(t, artifacts, "minio-test/file2.txt"))
   180  	assert.True(t, containsArtifact(t, artifacts, "minio-test/subdir/file3.txt"))
   181  }
   182  
   183  func containsArtifact(t *testing.T, artifacts []testkube.Artifact, name string) bool {
   184  	t.Helper()
   185  
   186  	for _, a := range artifacts {
   187  		if a.Name == name {
   188  			return true
   189  		}
   190  	}
   191  	return false
   192  }