github.com/jfrog/jfrog-cli-core/v2@v2.51.0/artifactory/commands/transferfiles/delayedartifactshandler_test.go (about)

     1  package transferfiles
     2  
     3  import (
     4  	"fmt"
     5  	"math"
     6  	"os"
     7  	"path/filepath"
     8  	"sync"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
    13  	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
    14  	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
    15  	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
    16  	"github.com/stretchr/testify/assert"
    17  )
    18  
    19  var delayTestRepoKey = "delay-local-repo"
    20  
    21  func TestDelayedArtifactsMng(t *testing.T) {
    22  	// Set testing environment
    23  	cleanUpJfrogHome, err := tests.SetJfrogHome()
    24  	assert.NoError(t, err)
    25  	defer cleanUpJfrogHome()
    26  
    27  	delaysDirPath, err := getJfrogTransferRepoDelaysDir(testRepoKey)
    28  	assert.NoError(t, err)
    29  	assert.NoError(t, fileutils.CreateDirIfNotExist(delaysDirPath))
    30  
    31  	artifactsNumber := 50
    32  	// We reduce the maximum number of entities per file to test the creation of multiple delayed artifacts files.
    33  	originalMaxArtifactsInFile := maxDelayedArtifactsInFile
    34  	maxDelayedArtifactsInFile = 20
    35  	defer func() { maxDelayedArtifactsInFile = originalMaxArtifactsInFile }()
    36  	artifactsChannelMng := createdDelayedArtifactsChannelMng()
    37  	transferDelayedArtifactsToFile, err := newTransferDelayedArtifactsManager(&artifactsChannelMng, testRepoKey, state.ConvertTimeToEpochMilliseconds(time.Now()))
    38  	assert.NoError(t, err)
    39  	var writeWaitGroup sync.WaitGroup
    40  	var readWaitGroup sync.WaitGroup
    41  
    42  	// "Writing delayed artifacts to files" mechanism returned error
    43  	var delayedArtifactsErr error
    44  	// Start reading from the delayed artifacts channel, and write artifacts into files.
    45  	readWaitGroup.Add(1)
    46  	go func() {
    47  		defer readWaitGroup.Done()
    48  		delayedArtifactsErr = transferDelayedArtifactsToFile.start()
    49  	}()
    50  
    51  	// Add artifacts to the common channel.
    52  	writeWaitGroup.Add(1)
    53  	go func() {
    54  		defer writeWaitGroup.Done()
    55  		for i := 0; i < artifactsNumber; i++ {
    56  			artifactsChannelMng.channel <- api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("name%d", i), Size: int64(i)}
    57  		}
    58  	}()
    59  
    60  	writeWaitGroup.Wait()
    61  	artifactsChannelMng.close()
    62  	readWaitGroup.Wait()
    63  	assert.NoError(t, delayedArtifactsErr)
    64  
    65  	// add not relevant files to confuse
    66  	writeEmptyConfuseFiles(t, repo1Key)
    67  	writeEmptyConfuseFiles(t, delayTestRepoKey+"-0")
    68  
    69  	delayFiles, err := getDelayFiles([]string{testRepoKey})
    70  	assert.NoError(t, err)
    71  
    72  	expectedNumberOfFiles := int(math.Ceil(float64(artifactsNumber) / float64(maxDelayedArtifactsInFile)))
    73  	validateDelayedArtifactsFiles(t, delayFiles, expectedNumberOfFiles, artifactsNumber)
    74  
    75  	delayCount, storage, err := countDelayFilesContent(delayFiles)
    76  	assert.NoError(t, err)
    77  	assert.Equal(t, int64(1225), storage)
    78  	assert.Equal(t, delayCount, artifactsNumber)
    79  }
    80  
    81  func writeEmptyConfuseFiles(t *testing.T, repoKey string) {
    82  	delaysDirPath, err := getJfrogTransferRepoDelaysDir(repoKey)
    83  	assert.NoError(t, err)
    84  	assert.NoError(t, fileutils.CreateDirIfNotExist(delaysDirPath))
    85  	for i := 0; i < 4; i++ {
    86  		writeEmptyFile(t, delaysDirPath, delayTestRepoKey, i)
    87  	}
    88  	writeEmptyFile(t, delaysDirPath, "wrong-"+testRepoKey, 1)
    89  	writeEmptyFile(t, delaysDirPath, testRepoKey+"-wrong", 1)
    90  	writeEmptyFile(t, delaysDirPath, testRepoKey+"-0-0", 0)
    91  	writeEmptyFile(t, delaysDirPath, testRepoKey+"-1", 22)
    92  	writeEmptyFile(t, delaysDirPath, "wrong-"+testRepoKey+"-wrong", 0)
    93  
    94  	delayFiles, err := getDelayFiles([]string{repoKey})
    95  	assert.NoError(t, err)
    96  	assert.Len(t, delayFiles, 9)
    97  }
    98  
    99  func writeEmptyFile(t *testing.T, delaysDirPath string, repoName string, index int) {
   100  	fullName := fmt.Sprintf("%s-%d.json", getDelaysFilePrefix(repoName, state.ConvertTimeToEpochMilliseconds(time.Now())), index)
   101  	assert.NoError(t, os.WriteFile(filepath.Join(delaysDirPath, fullName), nil, 0644))
   102  }
   103  
   104  // Ensure that all 'delayed artifacts files' have been created and that they contain the expected content
   105  func validateDelayedArtifactsFiles(t *testing.T, delayedArtifactsFile []string, filesNum, artifactsNum int) {
   106  	assert.Lenf(t, delayedArtifactsFile, filesNum, "unexpected number of delayed artifacts files.")
   107  	var entitiesNum int
   108  	for i := 0; i < filesNum; i++ {
   109  		entitiesNum += validateDelayedArtifactsFilesContent(t, delayedArtifactsFile[i])
   110  	}
   111  	assert.Equal(t, artifactsNum, entitiesNum)
   112  }
   113  
   114  // Check the number of artifacts and their uniqueness by reading the file's content.
   115  func validateDelayedArtifactsFilesContent(t *testing.T, path string) (entitiesNum int) {
   116  	exists, err := fileutils.IsFileExists(path, false)
   117  	assert.NoError(t, err)
   118  	assert.True(t, exists, fmt.Sprintf("file: %s does not exist", path))
   119  
   120  	delayedArtifacts, err := readDelayFile(path)
   121  	assert.NoError(t, err)
   122  
   123  	// Verify all artifacts were written with their unique name
   124  	artifactsNamesMap := make(map[string]bool)
   125  	for _, entity := range delayedArtifacts.DelayedArtifacts {
   126  		if artifactsNamesMap[entity.Name] == true {
   127  			assert.Fail(t, fmt.Sprintf("an artifacts with the uniqe name \"%s\" was written more than once", entity.Name))
   128  			return
   129  		}
   130  		artifactsNamesMap[entity.Name] = true
   131  	}
   132  	return len(delayedArtifacts.DelayedArtifacts)
   133  }