github.com/kubeshop/testkube@v1.17.23/pkg/executor/scraper/filesystem_extractor_integration_test.go (about)

     1  package scraper_test
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"io"
     7  	"os"
     8  	"path/filepath"
     9  	"strings"
    10  	"testing"
    11  
    12  	"github.com/kubeshop/testkube/pkg/utils/test"
    13  
    14  	"github.com/stretchr/testify/assert"
    15  	"github.com/stretchr/testify/require"
    16  
    17  	"github.com/kubeshop/testkube/pkg/executor/scraper"
    18  	"github.com/kubeshop/testkube/pkg/filesystem"
    19  )
    20  
    21  func TestArchiveFilesystemExtractor_Extract_NoMeta_Integration(t *testing.T) {
    22  	test.IntegrationTest(t)
    23  	t.Parallel()
    24  
    25  	tempDir, err := os.MkdirTemp("", "test")
    26  	require.NoError(t, err)
    27  
    28  	defer os.RemoveAll(tempDir)
    29  
    30  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
    31  	require.NoError(t, err)
    32  
    33  	file1 := filepath.Join(tempDir, "file1.txt")
    34  	file2 := filepath.Join(tempDir, "file2.txt")
    35  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
    36  
    37  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
    38  	assert.NoError(t, err)
    39  
    40  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
    41  	assert.NoError(t, err)
    42  
    43  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
    44  	assert.NoError(t, err)
    45  
    46  	processCallCount := 0
    47  	processFn := func(ctx context.Context, object *scraper.Object) error {
    48  		processCallCount++
    49  		assert.Equal(t, "artifacts.tar.gz", object.Name)
    50  		assert.Equal(t, scraper.DataTypeTarball, object.DataType)
    51  
    52  		return nil
    53  	}
    54  
    55  	notifyFn := func(ctx context.Context, path string) error {
    56  		if !strings.Contains(path, "file1.txt") && !strings.Contains(path, "file2.txt") && !strings.Contains(path, "subdir/file3.txt") {
    57  			t.Fatalf("Unexpected path: %s", path)
    58  		}
    59  		return nil
    60  	}
    61  
    62  	extractor := scraper.NewArchiveFilesystemExtractor(filesystem.NewOSFileSystem())
    63  	scrapeDirs := []string{tempDir}
    64  	masks := []string{".*"}
    65  	err = extractor.Extract(context.Background(), scrapeDirs, masks, processFn, notifyFn)
    66  	require.NoError(t, err)
    67  	assert.Equal(t, 1, processCallCount)
    68  }
    69  
    70  func TestArchiveFilesystemExtractor_Extract_Meta_Integration(t *testing.T) {
    71  	test.IntegrationTest(t)
    72  	t.Parallel()
    73  
    74  	tempDir, err := os.MkdirTemp("", "test")
    75  	require.NoError(t, err)
    76  
    77  	defer os.RemoveAll(tempDir)
    78  
    79  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
    80  	require.NoError(t, err)
    81  
    82  	file1 := filepath.Join(tempDir, "file1.txt")
    83  	file2 := filepath.Join(tempDir, "file2.txt")
    84  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
    85  
    86  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
    87  	assert.NoError(t, err)
    88  
    89  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
    90  	assert.NoError(t, err)
    91  
    92  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
    93  	assert.NoError(t, err)
    94  
    95  	processCallCount := 0
    96  	processFn := func(ctx context.Context, object *scraper.Object) error {
    97  		switch object.Name {
    98  		case "artifacts.tar.gz":
    99  			processCallCount++
   100  			assert.Equal(t, scraper.DataTypeTarball, object.DataType)
   101  		case ".testkube-meta-files.json":
   102  			processCallCount++
   103  			var meta scraper.FilesMeta
   104  			jsonData, err := io.ReadAll(object.Data)
   105  			if err != nil {
   106  				t.Fatalf("Failed to read meta files: %v", err)
   107  			}
   108  			if err := json.Unmarshal(jsonData, &meta); err != nil {
   109  				t.Fatalf("Failed to unmarshal meta files: %v", err)
   110  			}
   111  			assert.Len(t, meta.Files, 3)
   112  			assert.Equal(t, "artifacts.tar.gz", meta.Archive)
   113  			assert.Equal(t, scraper.DataTypeTarball, meta.DataType)
   114  			assert.Equal(t, "file1.txt", meta.Files[0].Name)
   115  			assert.Equal(t, int64(5), meta.Files[0].Size)
   116  			assert.Equal(t, "file2.txt", meta.Files[1].Name)
   117  			assert.Equal(t, int64(5), meta.Files[1].Size)
   118  			assert.Equal(t, "subdir/file3.txt", meta.Files[2].Name)
   119  			assert.Equal(t, int64(5), meta.Files[2].Size)
   120  			assert.Equal(t, scraper.DataTypeRaw, object.DataType)
   121  		default:
   122  			t.Fatalf("Unexpected object name: %s", object.Name)
   123  		}
   124  
   125  		return nil
   126  	}
   127  
   128  	notifyFn := func(ctx context.Context, path string) error {
   129  		if !strings.Contains(path, "file1.txt") && !strings.Contains(path, "file2.txt") && !strings.Contains(path, "subdir/file3.txt") {
   130  			t.Fatalf("Unexpected path: %s", path)
   131  		}
   132  		return nil
   133  	}
   134  
   135  	extractor := scraper.NewArchiveFilesystemExtractor(filesystem.NewOSFileSystem(), scraper.GenerateTarballMetaFile())
   136  	scrapeDirs := []string{tempDir}
   137  	masks := []string{".*"}
   138  	err = extractor.Extract(context.Background(), scrapeDirs, masks, processFn, notifyFn)
   139  	require.NoError(t, err)
   140  	assert.Equal(t, 2, processCallCount)
   141  }
   142  
   143  func TestRecursiveFilesystemExtractor_Extract_Integration(t *testing.T) {
   144  	test.IntegrationTest(t)
   145  	t.Parallel()
   146  
   147  	tempDir, err := os.MkdirTemp("", "test")
   148  	require.NoError(t, err)
   149  
   150  	defer os.RemoveAll(tempDir)
   151  
   152  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
   153  	require.NoError(t, err)
   154  
   155  	file1 := filepath.Join(tempDir, "file1.txt")
   156  	file2 := filepath.Join(tempDir, "file2.txt")
   157  	file3 := filepath.Join(tempDir, "subdir", "file3.txt")
   158  
   159  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
   160  	assert.NoError(t, err)
   161  
   162  	err = os.WriteFile(file2, []byte("test2"), os.ModePerm)
   163  	assert.NoError(t, err)
   164  
   165  	err = os.WriteFile(file3, []byte("test3"), os.ModePerm)
   166  	assert.NoError(t, err)
   167  
   168  	processCallCount := 0
   169  	processFn := func(ctx context.Context, object *scraper.Object) error {
   170  		processCallCount++
   171  		b, err := io.ReadAll(object.Data)
   172  		if err != nil {
   173  			t.Fatalf("error reading %s: %v", object.Name, err)
   174  		}
   175  		assert.Equal(t, scraper.DataTypeRaw, object.DataType)
   176  		switch object.Name {
   177  		case "file1.txt":
   178  			assert.Equal(t, b, []byte("test1"))
   179  		case "file2.txt":
   180  			assert.Equal(t, b, []byte("test2"))
   181  		case "subdir/file3.txt":
   182  			assert.Equal(t, b, []byte("test3"))
   183  		default:
   184  			t.Fatalf("unexpected file: %s", object.Name)
   185  		}
   186  
   187  		return nil
   188  	}
   189  
   190  	notifyFn := func(ctx context.Context, path string) error {
   191  		if !strings.Contains(path, "file1.txt") && !strings.Contains(path, "file2.txt") && !strings.Contains(path, "subdir/file3.txt") {
   192  			t.Fatalf("unexpected file: %s", path)
   193  		}
   194  
   195  		return nil
   196  	}
   197  
   198  	extractor := scraper.NewRecursiveFilesystemExtractor(filesystem.NewOSFileSystem())
   199  	scrapeDirs := []string{tempDir, "/nonexistent"}
   200  	masks := []string{".*"}
   201  	err = extractor.Extract(context.Background(), scrapeDirs, masks, processFn, notifyFn)
   202  	require.NoError(t, err)
   203  	assert.Equal(t, processCallCount, 3)
   204  }
   205  
   206  func TestRecursiveFilesystemExtractor_Extract_RelPath_Integration(t *testing.T) {
   207  	test.IntegrationTest(t)
   208  	t.Parallel()
   209  
   210  	tempDir, err := os.MkdirTemp("", "test")
   211  	require.NoError(t, err)
   212  
   213  	defer os.RemoveAll(tempDir)
   214  
   215  	err = os.Mkdir(filepath.Join(tempDir, "subdir"), os.ModePerm)
   216  	require.NoError(t, err)
   217  
   218  	file1 := filepath.Join(tempDir, "file1.txt")
   219  
   220  	err = os.WriteFile(file1, []byte("test1"), os.ModePerm)
   221  	assert.NoError(t, err)
   222  
   223  	processCallCount := 0
   224  	processFn := func(ctx context.Context, object *scraper.Object) error {
   225  		processCallCount++
   226  		b, err := io.ReadAll(object.Data)
   227  		if err != nil {
   228  			t.Fatalf("error reading %s: %v", object.Name, err)
   229  		}
   230  		assert.Equal(t, b, []byte("test1"))
   231  		assert.Equal(t, scraper.DataTypeRaw, object.DataType)
   232  		return nil
   233  	}
   234  
   235  	notifyFn := func(ctx context.Context, path string) error {
   236  		if !strings.Contains(path, "file1.txt") {
   237  			t.Fatalf("unexpected path: %s", path)
   238  		}
   239  		return nil
   240  	}
   241  
   242  	extractor := scraper.NewRecursiveFilesystemExtractor(filesystem.NewOSFileSystem())
   243  	scrapeDirs := []string{filepath.Join(tempDir, "file1.txt"), "/nonexistent"}
   244  	masks := []string{".*"}
   245  	err = extractor.Extract(context.Background(), scrapeDirs, masks, processFn, notifyFn)
   246  	require.NoError(t, err)
   247  	assert.Equal(t, processCallCount, 1)
   248  }