github.com/jfrog/jfrog-cli-core/v2@v2.51.0/artifactory/commands/transferfiles/errorshandler_test.go (about)

     1  package transferfiles
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
     7  	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
     8  	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
     9  	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
    10  	"github.com/stretchr/testify/assert"
    11  	"math"
    12  	"os"
    13  	"path/filepath"
    14  	"sync"
    15  	"testing"
    16  	"time"
    17  )
    18  
    19  var (
    20  	testRetryableStatus = api.Fail
    21  	testSkippedStatus   = api.SkippedLargeProps
    22  	testRepoKey         = "repo"
    23  )
    24  
    25  func TestTransferErrorsMng(t *testing.T) {
    26  	cleanUpJfrogHome, err := tests.SetJfrogHome()
    27  	assert.NoError(t, err)
    28  	defer cleanUpJfrogHome()
    29  
    30  	errorsNumber := 50
    31  	// We reduce the maximum number of entities per file to test the creation of multiple errors files.
    32  	originalMaxErrorsInFile := maxErrorsInFile
    33  	maxErrorsInFile = 20
    34  	defer func() { maxErrorsInFile = originalMaxErrorsInFile }()
    35  	errorsChannelMng := createErrorsChannelMng()
    36  	transferErrorsMng, err := newTransferErrorsToFile(testRepoKey, 0, state.ConvertTimeToEpochMilliseconds(time.Now()), &errorsChannelMng, nil, nil)
    37  	assert.NoError(t, err)
    38  
    39  	var writeWaitGroup sync.WaitGroup
    40  	var readWaitGroup sync.WaitGroup
    41  
    42  	// "Writing transfer's errors to files" mechanism returned error
    43  	var writingErrorsErr error
    44  	// Start reading from the errors channel, and write errors into the relevant files.
    45  	readWaitGroup.Add(1)
    46  	go func() {
    47  		defer readWaitGroup.Done()
    48  		writingErrorsErr = transferErrorsMng.start()
    49  	}()
    50  
    51  	// Add 'retryable errors' to the common errors channel.
    52  	// These errors will be written into files located in the "retryable" directory under the Jfrog CLI home directory.
    53  	addErrorsToChannel(&writeWaitGroup, errorsNumber, errorsChannelMng, api.Fail)
    54  	// Add 'skipped errors' to the common errors channel.
    55  	// These errors will be written into files located in the "skipped" directory under the Jfrog CLI home directory.
    56  	addErrorsToChannel(&writeWaitGroup, errorsNumber, errorsChannelMng, api.SkippedLargeProps)
    57  
    58  	writeWaitGroup.Wait()
    59  	errorsChannelMng.close()
    60  	readWaitGroup.Wait()
    61  	assert.NoError(t, writingErrorsErr)
    62  	expectedNumberOfFiles := int(math.Ceil(float64(errorsNumber) / float64(maxErrorsInFile)))
    63  	validateErrorsFiles(t, expectedNumberOfFiles, errorsNumber, true)
    64  	validateErrorsFiles(t, expectedNumberOfFiles, errorsNumber, false)
    65  
    66  	retryEntityCount, err := getRetryErrorCount([]string{testRepoKey})
    67  	assert.NoError(t, err)
    68  	assert.Equal(t, errorsNumber, retryEntityCount)
    69  }
    70  
    71  func addErrorsToChannel(writeWaitGroup *sync.WaitGroup, errorsNumber int, errorsChannelMng ErrorsChannelMng, status api.ChunkFileStatusType) {
    72  	writeWaitGroup.Add(1)
    73  	go func() {
    74  		defer writeWaitGroup.Done()
    75  		for i := 0; i < errorsNumber; i++ {
    76  			errorsChannelMng.add(api.FileUploadStatusResponse{FileRepresentation: api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("name%d", i)}, Status: status, StatusCode: 404, Reason: "reason"})
    77  		}
    78  	}()
    79  }
    80  
    81  // Ensure that all retryable/skipped errors files have been created and that they contain the expected content
    82  func validateErrorsFiles(t *testing.T, filesNum, errorsNum int, isRetryable bool) {
    83  	errorsFiles, err := getErrorsFiles([]string{testRepoKey}, isRetryable)
    84  	status := getStatusType(isRetryable)
    85  	assert.NoError(t, err)
    86  	assert.Lenf(t, errorsFiles, filesNum, "unexpected number of error files.")
    87  	var entitiesNum int
    88  	for i := 0; i < filesNum; i++ {
    89  		entitiesNum += validateErrorsFileContent(t, errorsFiles[i], status)
    90  	}
    91  	assert.Equal(t, errorsNum, entitiesNum)
    92  }
    93  
    94  func getStatusType(isRetryable bool) api.ChunkFileStatusType {
    95  	if isRetryable {
    96  		return testRetryableStatus
    97  	}
    98  	return testSkippedStatus
    99  }
   100  
   101  // Check the number of errors, their status and their uniqueness by reading the file's content.
   102  func validateErrorsFileContent(t *testing.T, path string, status api.ChunkFileStatusType) (entitiesNum int) {
   103  	exists, err := fileutils.IsFileExists(path, false)
   104  	assert.NoError(t, err)
   105  	assert.True(t, exists, fmt.Sprintf("file: %s does not exist", path))
   106  
   107  	var content []byte
   108  	content, err = fileutils.ReadFile(path)
   109  	assert.NoError(t, err)
   110  
   111  	filesErrors := new(FilesErrors)
   112  	assert.NoError(t, json.Unmarshal(content, &filesErrors))
   113  
   114  	// Verify all unique errors were written with the correct status
   115  	errorsNamesMap := make(map[string]bool)
   116  	for _, entity := range filesErrors.Errors {
   117  		// Verify error's status
   118  		assert.Equal(t, status, entity.Status, fmt.Sprintf("expecting error status to be: %s but got %s", status, entity.Status))
   119  		// Verify error's unique name
   120  		assert.False(t, errorsNamesMap[entity.Name], fmt.Sprintf("an error with the uniqe name \"%s\" was written more than once", entity.Name))
   121  		errorsNamesMap[entity.Name] = true
   122  		// Verify time
   123  		assert.NotEmptyf(t, entity.Time, "expecting error's time stamp, but got empty string")
   124  	}
   125  	return len(filesErrors.Errors)
   126  }
   127  
   128  func TestGetErrorsFiles(t *testing.T) {
   129  	cleanUpJfrogHome, err := tests.SetJfrogHome()
   130  	assert.NoError(t, err)
   131  	defer cleanUpJfrogHome()
   132  
   133  	// Create 3 retryable and 1 skipped errors files that belong to repo1.
   134  	writeEmptyErrorsFile(t, repo1Key, true, 0, 0)
   135  	writeEmptyErrorsFile(t, repo1Key, true, 0, 1)
   136  	writeEmptyErrorsFile(t, repo1Key, true, 1, 123)
   137  	writeEmptyErrorsFile(t, repo1Key, false, 0, 1)
   138  
   139  	// Create 2 retryable and 2 skipped errors files that belong to repo2.
   140  	writeEmptyErrorsFile(t, repo2Key, true, 0, 0)
   141  	writeEmptyErrorsFile(t, repo2Key, true, 2, 1)
   142  	writeEmptyErrorsFile(t, repo2Key, false, 1, 0)
   143  	writeEmptyErrorsFile(t, repo2Key, false, 0, 1)
   144  
   145  	paths, err := getErrorsFiles([]string{repo1Key}, true)
   146  	assert.NoError(t, err)
   147  	assert.Len(t, paths, 3)
   148  	paths, err = getErrorsFiles([]string{repo1Key}, false)
   149  	assert.NoError(t, err)
   150  	assert.Len(t, paths, 1)
   151  	paths, err = getErrorsFiles([]string{repo1Key, repo2Key}, true)
   152  	assert.NoError(t, err)
   153  	assert.Len(t, paths, 5)
   154  }
   155  
   156  func writeEmptyErrorsFile(t *testing.T, repoKey string, retryable bool, phase, counter int) {
   157  	var errorsDirPath string
   158  	var err error
   159  	if retryable {
   160  		errorsDirPath, err = getJfrogTransferRepoRetryableDir(repoKey)
   161  	} else {
   162  		errorsDirPath, err = getJfrogTransferRepoSkippedDir(repoKey)
   163  	}
   164  	assert.NoError(t, err)
   165  	assert.NoError(t, fileutils.CreateDirIfNotExist(errorsDirPath))
   166  
   167  	fileName := fmt.Sprintf("%s-%d.json", getErrorsFileNamePrefix(repoKey, phase, state.ConvertTimeToEpochMilliseconds(time.Now())), counter)
   168  	assert.NoError(t, os.WriteFile(filepath.Join(errorsDirPath, fileName), nil, 0644))
   169  }