github.com/jfrog/jfrog-cli-core/v2@v2.51.0/artifactory/commands/transferfiles/state/timeestimation_test.go (about)

     1  package state
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	"github.com/jfrog/build-info-go/utils"
     8  	rtServicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
     9  
    10  	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
    11  
    12  	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
    13  	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
    14  	"github.com/stretchr/testify/assert"
    15  )
    16  
    17  func initTimeEstimationTestSuite(t *testing.T) func() {
    18  	cleanUpJfrogHome, err := tests.SetJfrogHome()
    19  	assert.NoError(t, err)
    20  
    21  	// Create transfer directory
    22  	transferDir, err := coreutils.GetJfrogTransferDir()
    23  	assert.NoError(t, err)
    24  	err = utils.CreateDirIfNotExist(transferDir)
    25  	assert.NoError(t, err)
    26  
    27  	undoSaveInterval := SetAutoSaveState()
    28  	return func() {
    29  		undoSaveInterval()
    30  		cleanUpJfrogHome()
    31  	}
    32  }
    33  
    34  func initTimeEstimationDataTest(t *testing.T) (*TimeEstimationManager, func()) {
    35  	cleanUpJfrogHome := initTimeEstimationTestSuite(t)
    36  	return newDefaultTimeEstimationManager(t, false), cleanUpJfrogHome
    37  }
    38  
    39  func TestGetSpeed(t *testing.T) {
    40  	timeEstMng, cleanUp := initTimeEstimationDataTest(t)
    41  	defer cleanUp()
    42  
    43  	// Chunk 1: one of the files is checksum-deployed
    44  	chunkStatus1 := api.ChunkStatus{
    45  		Files: []api.FileUploadStatusResponse{
    46  			createFileUploadStatusResponse(repo1Key, 10*rtServicesUtils.SizeMiB, false, api.Success),
    47  			createFileUploadStatusResponse(repo1Key, 15*rtServicesUtils.SizeMiB, false, api.Success),
    48  			createFileUploadStatusResponse(repo1Key, 8*rtServicesUtils.SizeMiB, true, api.Success),
    49  		},
    50  	}
    51  	addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
    52  	assert.Equal(t, 7.5, timeEstMng.getSpeed())
    53  	assert.Equal(t, "7.500 MB/s", timeEstMng.GetSpeedString())
    54  	assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds())
    55  
    56  	// Chunk 2: the upload of one of the files failed and the files are not included in the repository's total size (includedInTotalSize == false)
    57  	chunkStatus2 := api.ChunkStatus{
    58  		Files: []api.FileUploadStatusResponse{
    59  			createFileUploadStatusResponse(repo1Key, int64(21.25*float64(rtServicesUtils.SizeMiB)), false, api.Success),
    60  			createFileUploadStatusResponse(repo1Key, 6*rtServicesUtils.SizeMiB, false, api.Fail),
    61  		},
    62  	}
    63  	addChunkStatus(t, timeEstMng, chunkStatus2, 2, false, 5*milliSecsInSecond)
    64  	assert.Equal(t, float64(8), timeEstMng.getSpeed())
    65  	assert.Equal(t, "8.000 MB/s", timeEstMng.GetSpeedString())
    66  	assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds())
    67  }
    68  
    69  func TestGetEstimatedRemainingSeconds(t *testing.T) {
    70  	timeEstMng, cleanUp := initTimeEstimationDataTest(t)
    71  	defer cleanUp()
    72  
    73  	timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes)
    74  	timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes
    75  	assert.Zero(t, timeEstMng.getEstimatedRemainingSeconds())
    76  
    77  	timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) / 2
    78  	timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes / 2
    79  	calculatedEstimatedSeconds := timeEstMng.getEstimatedRemainingSeconds()
    80  	assert.NotZero(t, calculatedEstimatedSeconds)
    81  }
    82  
    83  func TestGetEstimatedRemainingTimeStringNotAvailableYet(t *testing.T) {
    84  	timeEstMng, cleanUp := initTimeEstimationDataTest(t)
    85  	defer cleanUp()
    86  
    87  	// Chunk 1: one of the files is checksum-deployed
    88  	chunkStatus1 := api.ChunkStatus{
    89  		Files: []api.FileUploadStatusResponse{
    90  			createFileUploadStatusResponse(repo1Key, 8*rtServicesUtils.SizeMiB, true, api.Success),
    91  		},
    92  	}
    93  	assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
    94  	addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
    95  	assert.Equal(t, "Not available yet", timeEstMng.GetSpeedString())
    96  }
    97  
    98  func TestGetEstimatedRemainingTimeString(t *testing.T) {
    99  	timeEstMng, cleanUp := initTimeEstimationDataTest(t)
   100  	defer cleanUp()
   101  
   102  	// Test "Not available yet" by setting the TotalTransferredBytes to 0
   103  	timeEstMng.CurrentTotalTransferredBytes = 0
   104  	assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
   105  
   106  	// Test "About 1 minute" by setting the transferred bytes to 80%
   107  	timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8)
   108  	timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8)
   109  	assert.Equal(t, "About 1 minute", timeEstMng.GetEstimatedRemainingTimeString())
   110  
   111  	// Test "Less than a minute" by setting the transferred bytes to 90%
   112  	timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9)
   113  	timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9)
   114  	assert.Equal(t, "Less than a minute", timeEstMng.GetEstimatedRemainingTimeString())
   115  }
   116  
   117  func newDefaultTimeEstimationManager(t *testing.T, buildInfoRepos bool) *TimeEstimationManager {
   118  	stateManager, err := NewTransferStateManager(true)
   119  	stateManager.startTimestamp = time.Now().Add(-minTransferTimeToShowEstimation)
   120  	assert.NoError(t, err)
   121  	assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, buildInfoRepos, true))
   122  	assert.NoError(t, stateManager.SetRepoState(repo2Key, 0, 0, buildInfoRepos, true))
   123  
   124  	assert.NoError(t, stateManager.IncTransferredSizeAndFilesPhase1(0, 100*rtServicesUtils.SizeMiB))
   125  	stateManager.OverallTransfer.TotalSizeBytes = 600 * rtServicesUtils.SizeMiB
   126  	return &TimeEstimationManager{stateManager: stateManager}
   127  }
   128  
   129  func TestAddingToFullLastSpeedsSlice(t *testing.T) {
   130  	timeEstMng, cleanUp := initTimeEstimationDataTest(t)
   131  	defer cleanUp()
   132  
   133  	numOfSpeedsToKeepPerWorkingThread = 3
   134  
   135  	// Fill the last speeds slice (up to threads * numOfSpeedsToKeepPerWorkingThread).
   136  	timeEstMng.LastSpeeds = []float64{1.1, 2.2, 3.3, 4.4, 5.5, 6.6}
   137  
   138  	// Add a chunk and assert the oldest speed is removed, new is added, and the chunk len remains the same.
   139  	firstChunkSpeed := addOneFileChunk(t, timeEstMng, 2, 20, 1)
   140  	assert.Equal(t, []float64{2.2, 3.3, 4.4, 5.5, 6.6, firstChunkSpeed}, timeEstMng.LastSpeeds)
   141  
   142  	// Lower threads and add a chunk. Expecting the slice to shrink and the new speed to be added in the end.
   143  	secondChunkSpeed := addOneFileChunk(t, timeEstMng, 1, 30, 2)
   144  	assert.Equal(t, []float64{6.6, firstChunkSpeed, secondChunkSpeed}, timeEstMng.LastSpeeds)
   145  
   146  	// Increase threads and add a chunk. Expecting the slice len to increase with the new speed.
   147  	thirdChunkSpeed := addOneFileChunk(t, timeEstMng, 3, 40, 3)
   148  	assert.Equal(t, []float64{6.6, firstChunkSpeed, secondChunkSpeed, thirdChunkSpeed}, timeEstMng.LastSpeeds)
   149  }
   150  
   151  // Adds a chunk with one non checksum-deployed file and calculates and returns the chunk speed.
   152  func addOneFileChunk(t *testing.T, timeEstMng *TimeEstimationManager, workingThreads, chunkDurationMilli, chunkSizeMb int) float64 {
   153  	chunkDuration := int64(chunkDurationMilli * milliSecsInSecond)
   154  	chunkSize := int64(chunkSizeMb) * rtServicesUtils.SizeMiB
   155  	chunkStatus := api.ChunkStatus{
   156  		Files: []api.FileUploadStatusResponse{
   157  			createFileUploadStatusResponse(repo1Key, chunkSize, false, api.Success),
   158  		},
   159  	}
   160  	addChunkStatus(t, timeEstMng, chunkStatus, workingThreads, true, chunkDuration)
   161  	return calculateChunkSpeed(workingThreads, chunkSize, chunkDuration)
   162  }
   163  
   164  func TestTransferredSizeInState(t *testing.T) {
   165  	cleanUp := initTimeEstimationTestSuite(t)
   166  	defer cleanUp()
   167  
   168  	stateManager, err := NewTransferStateManager(true)
   169  	assert.NoError(t, err)
   170  	timeEstMng := &TimeEstimationManager{stateManager: stateManager}
   171  
   172  	// Create repo1 in state.
   173  	assert.NoError(t, timeEstMng.stateManager.SetRepoState(repo1Key, 0, 0, false, true))
   174  	assertTransferredSizes(t, timeEstMng.stateManager, 0, 0)
   175  
   176  	// Add a chunk of repo1 with multiple successful files, which are included in total.
   177  	chunkStatus1 := api.ChunkStatus{
   178  		Files: []api.FileUploadStatusResponse{
   179  			createFileUploadStatusResponse(repo1Key, 10*rtServicesUtils.SizeMiB, false, api.Success),
   180  			// Checksum-deploy should not affect the update size.
   181  			createFileUploadStatusResponse(repo1Key, 15*rtServicesUtils.SizeMiB, true, api.Success),
   182  		},
   183  	}
   184  	addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
   185  
   186  	// Add another chunk of repo1 which is not included in total. Expected not to be included in update.
   187  	chunkStatus2 := api.ChunkStatus{
   188  		Files: []api.FileUploadStatusResponse{
   189  			createFileUploadStatusResponse(repo1Key, 21*rtServicesUtils.SizeMiB, false, api.Success),
   190  		},
   191  	}
   192  	addChunkStatus(t, timeEstMng, chunkStatus2, 3, false, 10*milliSecsInSecond)
   193  
   194  	// Create repo2 in state.
   195  	assert.NoError(t, timeEstMng.stateManager.SetRepoState(repo2Key, 0, 0, false, true))
   196  
   197  	// Add a chunk of repo2 which is included in total. The failed file should be ignored.
   198  	chunkStatus3 := api.ChunkStatus{
   199  		Files: []api.FileUploadStatusResponse{
   200  			createFileUploadStatusResponse(repo2Key, 13*rtServicesUtils.SizeMiB, false, api.Success),
   201  			createFileUploadStatusResponse(repo2Key, 133*rtServicesUtils.SizeMiB, false, api.Fail),
   202  		},
   203  	}
   204  	addChunkStatus(t, timeEstMng, chunkStatus3, 3, true, 10*milliSecsInSecond)
   205  	assertTransferredSizes(t, timeEstMng.stateManager, chunkStatus1.Files[0].SizeBytes+chunkStatus1.Files[1].SizeBytes, chunkStatus3.Files[0].SizeBytes)
   206  
   207  	// Add one more chunk of repo2.
   208  	chunkStatus4 := api.ChunkStatus{
   209  		Files: []api.FileUploadStatusResponse{
   210  			createFileUploadStatusResponse(repo2Key, 9*rtServicesUtils.SizeMiB, false, api.Success),
   211  		},
   212  	}
   213  	addChunkStatus(t, timeEstMng, chunkStatus4, 3, true, 10*milliSecsInSecond)
   214  	assertTransferredSizes(t, timeEstMng.stateManager, chunkStatus1.Files[0].SizeBytes+chunkStatus1.Files[1].SizeBytes, chunkStatus3.Files[0].SizeBytes+chunkStatus4.Files[0].SizeBytes)
   215  }
   216  
   217  func addChunkStatus(t *testing.T, timeEstMng *TimeEstimationManager, chunkStatus api.ChunkStatus, workingThreads int, includedInTotalSize bool, durationMillis int64) {
   218  	if includedInTotalSize {
   219  		err := UpdateChunkInState(timeEstMng.stateManager, &chunkStatus)
   220  		assert.NoError(t, err)
   221  	}
   222  	assert.NoError(t, timeEstMng.stateManager.SetWorkingThreads(workingThreads))
   223  	timeEstMng.AddChunkStatus(chunkStatus, durationMillis)
   224  }
   225  
   226  func assertTransferredSizes(t *testing.T, stateManager *TransferStateManager, repo1expected, repo2expected int64) {
   227  	assertReposTransferredSize(t, stateManager, repo1expected, repo1Key)
   228  	assertReposTransferredSize(t, stateManager, repo2expected, repo2Key)
   229  }
   230  
   231  func createFileUploadStatusResponse(repoKey string, sizeBytes int64, checksumDeployed bool, status api.ChunkFileStatusType) api.FileUploadStatusResponse {
   232  	return api.FileUploadStatusResponse{
   233  		FileRepresentation: api.FileRepresentation{
   234  			Repo: repoKey,
   235  			Path: "path",
   236  			Name: "name",
   237  		},
   238  		SizeBytes:        sizeBytes,
   239  		ChecksumDeployed: checksumDeployed,
   240  		Status:           status,
   241  	}
   242  }