github.com/kubeshop/testkube@v1.17.23/pkg/cloud/data/artifact/scraper_integration_test.go (about)

     1  package artifact_test
     2  
     3  import (
     4  	"context"
     5  	"net/http"
     6  	"net/http/httptest"
     7  	"os"
     8  	"path/filepath"
     9  	"testing"
    10  
    11  	"github.com/kubeshop/testkube/pkg/utils/test"
    12  
    13  	cloudevents "github.com/cloudevents/sdk-go/v2"
    14  	"github.com/golang/mock/gomock"
    15  	"github.com/stretchr/testify/assert"
    16  
    17  	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
    18  	cloudscraper "github.com/kubeshop/testkube/pkg/cloud/data/artifact"
    19  	"github.com/kubeshop/testkube/pkg/cloud/data/executor"
    20  	"github.com/kubeshop/testkube/pkg/executor/scraper"
    21  	"github.com/kubeshop/testkube/pkg/filesystem"
    22  )
    23  
    24  func TestCloudScraper_ArchiveFilesystemExtractor_Integration(t *testing.T) {
    25  	test.IntegrationTest(t)
    26  	t.Parallel()
    27  
    28  	mockCtrl := gomock.NewController(t)
    29  	defer mockCtrl.Finish()
    30  
    31  	tempDir, err := os.MkdirTemp("", "test")
    32  	assert.NoError(t, err)
    33  
    34  	defer os.RemoveAll(tempDir)
    35  
    36  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
    37  	assert.NoError(t, err)
    38  
    39  	file1 := filepath.Join(tempDir, "file1.txt")
    40  	file2 := filepath.Join(tempDir, "file2.txt")
    41  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
    42  
    43  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
    44  	assert.NoError(t, err)
    45  
    46  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
    47  	assert.NoError(t, err)
    48  
    49  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
    50  	assert.NoError(t, err)
    51  
    52  	extractor := scraper.NewArchiveFilesystemExtractor(filesystem.NewOSFileSystem(), scraper.GenerateTarballMetaFile())
    53  
    54  	testServerRequests := 0
    55  	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    56  		assert.Equal(t, http.MethodPut, r.Method)
    57  		assert.Equal(t, "/dummy", r.URL.Path)
    58  		testServerRequests++
    59  		w.WriteHeader(http.StatusOK)
    60  	}))
    61  	defer testServer.Close()
    62  
    63  	mockExecutor := executor.NewMockExecutor(mockCtrl)
    64  	cloudLoader := cloudscraper.NewCloudUploader(mockExecutor, false)
    65  	req := &cloudscraper.PutObjectSignedURLRequest{
    66  		Object:        "artifacts.tar.gz",
    67  		ExecutionID:   "my-execution-id",
    68  		TestName:      "my-test",
    69  		TestSuiteName: "my-test-suite",
    70  	}
    71  	mockExecutor.
    72  		EXPECT().
    73  		Execute(gomock.Any(), cloudscraper.CmdScraperPutObjectSignedURL, gomock.Eq(req)).
    74  		Return([]byte(`{"URL":"`+testServer.URL+`/dummy"}`), nil)
    75  	req2 := &cloudscraper.PutObjectSignedURLRequest{
    76  		Object:        ".testkube-meta-files.json",
    77  		ExecutionID:   "my-execution-id",
    78  		TestName:      "my-test",
    79  		TestSuiteName: "my-test-suite",
    80  	}
    81  	mockExecutor.
    82  		EXPECT().
    83  		Execute(gomock.Any(), cloudscraper.CmdScraperPutObjectSignedURL, gomock.Eq(req2)).
    84  		Return([]byte(`{"URL":"`+testServer.URL+`/dummy"}`), nil)
    85  
    86  	// given
    87  	testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    88  		_, err := cloudevents.NewEventFromHTTPRequest(r)
    89  		// then
    90  		assert.NoError(t, err)
    91  	})
    92  
    93  	svr := httptest.NewServer(testHandler)
    94  	defer svr.Close()
    95  
    96  	client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
    97  	assert.NoError(t, err)
    98  
    99  	s := scraper.NewExtractLoadScraper(extractor, cloudLoader, client, "", "")
   100  	execution := testkube.Execution{
   101  		Id:            "my-execution-id",
   102  		TestName:      "my-test",
   103  		TestSuiteName: "my-test-suite",
   104  	}
   105  	err = s.Scrape(context.Background(), []string{tempDir}, []string{".*"}, execution)
   106  
   107  	assert.NoError(t, err)
   108  	assert.Equal(t, 2, testServerRequests)
   109  }
   110  
   111  func TestCloudScraper_RecursiveFilesystemExtractor_Integration(t *testing.T) {
   112  	test.IntegrationTest(t)
   113  	t.Parallel()
   114  
   115  	mockCtrl := gomock.NewController(t)
   116  	defer mockCtrl.Finish()
   117  
   118  	tempDir, err := os.MkdirTemp("", "test")
   119  	assert.NoError(t, err)
   120  
   121  	defer os.RemoveAll(tempDir)
   122  
   123  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
   124  	assert.NoError(t, err)
   125  
   126  	file1 := filepath.Join(tempDir, "file1.txt")
   127  	file2 := filepath.Join(tempDir, "file2.txt")
   128  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
   129  
   130  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
   131  	assert.NoError(t, err)
   132  
   133  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
   134  	assert.NoError(t, err)
   135  
   136  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
   137  	assert.NoError(t, err)
   138  
   139  	extractor := scraper.NewRecursiveFilesystemExtractor(filesystem.NewOSFileSystem())
   140  
   141  	testServerRequests := 0
   142  	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   143  		assert.Equal(t, http.MethodPut, r.Method)
   144  		assert.Equal(t, "/dummy", r.URL.Path)
   145  		testServerRequests++
   146  		w.WriteHeader(http.StatusOK)
   147  	}))
   148  	defer testServer.Close()
   149  
   150  	mockExecutor := executor.NewMockExecutor(mockCtrl)
   151  	cloudLoader := cloudscraper.NewCloudUploader(mockExecutor, false)
   152  	req1 := &cloudscraper.PutObjectSignedURLRequest{
   153  		Object:        "file1.txt",
   154  		ExecutionID:   "my-execution-id",
   155  		TestName:      "my-test",
   156  		TestSuiteName: "my-test-suite",
   157  	}
   158  	mockExecutor.
   159  		EXPECT().
   160  		Execute(gomock.Any(), cloudscraper.CmdScraperPutObjectSignedURL, gomock.Eq(req1)).
   161  		Return([]byte(`{"URL":"`+testServer.URL+`/dummy"}`), nil)
   162  
   163  	req2 := &cloudscraper.PutObjectSignedURLRequest{
   164  		Object:        "file2.txt",
   165  		ExecutionID:   "my-execution-id",
   166  		TestName:      "my-test",
   167  		TestSuiteName: "my-test-suite",
   168  	}
   169  	mockExecutor.
   170  		EXPECT().
   171  		Execute(gomock.Any(), cloudscraper.CmdScraperPutObjectSignedURL, gomock.Eq(req2)).
   172  		Return([]byte(`{"URL":"`+testServer.URL+`/dummy"}`), nil)
   173  
   174  	req3 := &cloudscraper.PutObjectSignedURLRequest{
   175  		Object:        "subdir/file3.txt",
   176  		ExecutionID:   "my-execution-id",
   177  		TestName:      "my-test",
   178  		TestSuiteName: "my-test-suite",
   179  	}
   180  	mockExecutor.
   181  		EXPECT().
   182  		Execute(gomock.Any(), cloudscraper.CmdScraperPutObjectSignedURL, gomock.Eq(req3)).
   183  		Return([]byte(`{"URL":"`+testServer.URL+`/dummy"}`), nil)
   184  
   185  	// given
   186  	testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   187  		_, err := cloudevents.NewEventFromHTTPRequest(r)
   188  		// then
   189  		assert.NoError(t, err)
   190  	})
   191  
   192  	svr := httptest.NewServer(testHandler)
   193  	defer svr.Close()
   194  
   195  	client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
   196  	assert.NoError(t, err)
   197  
   198  	s := scraper.NewExtractLoadScraper(extractor, cloudLoader, client, "", "")
   199  	execution := testkube.Execution{
   200  		Id:            "my-execution-id",
   201  		TestName:      "my-test",
   202  		TestSuiteName: "my-test-suite",
   203  	}
   204  	err = s.Scrape(context.Background(), []string{tempDir}, []string{".*"}, execution)
   205  	assert.NoError(t, err)
   206  	assert.Equal(t, 3, testServerRequests)
   207  }