github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ext/dload/utils_test.go (about)

     1  // Package dloader_test is a unit test
     2  /*
     3   * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package dload_test
     6  
     7  import (
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/NVIDIA/aistore/api/apc"
    12  	"github.com/NVIDIA/aistore/cmn"
    13  	"github.com/NVIDIA/aistore/cmn/cos"
    14  	"github.com/NVIDIA/aistore/core"
    15  	"github.com/NVIDIA/aistore/ext/dload"
    16  	"github.com/NVIDIA/aistore/fs"
    17  	"github.com/NVIDIA/aistore/tools"
    18  	"github.com/NVIDIA/aistore/tools/tassert"
    19  )
    20  
    21  func TestNormalizeObjName(t *testing.T) {
    22  	normalizeObjTests := []struct {
    23  		objName  string
    24  		expected string
    25  	}{
    26  		{"?objname", "?objname"},
    27  		{"filename", "filename"},
    28  		{"dir/file", "dir/file"},
    29  		{"dir%2Ffile", "dir/file"},
    30  		{"path1/path2/path3%2Fpath4%2Ffile", "path1/path2/path3/path4/file"},
    31  		{"file?arg1=a&arg2=b", "file"},
    32  		{"imagenet%2Fimagenet_train-000000.tgz?alt=media", "imagenet/imagenet_train-000000.tgz"},
    33  	}
    34  
    35  	for _, test := range normalizeObjTests {
    36  		actual, err := dload.NormalizeObjName(test.objName)
    37  		if err != nil {
    38  			t.Errorf("Unexpected error while normalizing %s: %v", test.objName, err)
    39  		}
    40  
    41  		if actual != test.expected {
    42  			t.Errorf("NormalizeObjName(%s) expected: %s, got: %s", test.objName, test.expected, actual)
    43  		}
    44  	}
    45  }
    46  
    47  func TestCompareObject(t *testing.T) {
    48  	tools.CheckSkip(t, &tools.SkipTestArgs{Long: true})
    49  	var (
    50  		src = prepareObject(t)
    51  		dst = &dload.DstElement{
    52  			Link: "https://storage.googleapis.com/minikube/iso/minikube-v0.23.2.iso.sha256",
    53  		}
    54  		clientConf cmn.ClientConf
    55  	)
    56  	// initialize http clients
    57  	clientConf.Timeout = 5 * cos.Duration(time.Second)
    58  	clientConf.TimeoutLong = 15 * cos.Duration(time.Second)
    59  	dload.Init(nil, nil, &clientConf)
    60  
    61  	// Modify local object to contain invalid (meta)data.
    62  	customMD := cos.StrKVs{
    63  		cmn.SourceObjMD:  apc.AWS,
    64  		cmn.VersionObjMD: "none",
    65  		cmn.CRC32CObjMD:  "bad",
    66  		cmn.MD5ObjMD:     "worse",
    67  	}
    68  	src.SetSize(10)
    69  	src.SetCustomMD(customMD)
    70  	equal, err := dload.CompareObjects(src, dst)
    71  	tassert.CheckFatal(t, err)
    72  	tassert.Errorf(t, !equal, "expected the objects not to be equal")
    73  
    74  	// Check that objects are still not equal after size update.
    75  	src.SetSize(65)
    76  	equal, err = dload.CompareObjects(src, dst)
    77  	tassert.CheckFatal(t, err)
    78  	tassert.Errorf(t, !equal, "expected the objects not to be equal")
    79  
    80  	// Check that correct CRC doesn't make them equal
    81  	customMD[cmn.CRC32CObjMD] = "30a991bd"
    82  	equal, err = dload.CompareObjects(src, dst)
    83  	tassert.CheckFatal(t, err)
    84  	tassert.Errorf(t, !equal, "expected the objects not to be equal")
    85  
    86  	// Check that the same provider still doesn't make them equal
    87  	customMD[cmn.SourceObjMD] = apc.GCP
    88  	equal, err = dload.CompareObjects(src, dst)
    89  	tassert.CheckFatal(t, err)
    90  	tassert.Errorf(t, !equal, "expected the objects not to be equal")
    91  
    92  	// Finally, check that objects are equal when they have the same (provider, version)
    93  	src.SetCustomKey(cmn.VersionObjMD, "1503349750687573")
    94  	src.SetCustomKey(cmn.MD5ObjMD, "7b01d3eacc5869db6eb9137f15335d27")
    95  	customMD[cmn.VersionObjMD] = "1503349750687573"
    96  	equal, err = dload.CompareObjects(src, dst)
    97  	tassert.CheckFatal(t, err)
    98  	tassert.Errorf(t, equal, "expected the objects to be equal")
    99  }
   100  
   101  func prepareObject(t *testing.T) *core.LOM {
   102  	out := tools.PrepareObjects(t, tools.ObjectsDesc{
   103  		CTs: []tools.ContentTypeDesc{{
   104  			Type:       fs.ObjectType,
   105  			ContentCnt: 1,
   106  		}},
   107  		MountpathsCnt: 1,
   108  		ObjectSize:    1024,
   109  	})
   110  	lom := &core.LOM{}
   111  	err := lom.InitFQN(out.FQNs[fs.ObjectType][0], &out.Bck)
   112  	tassert.CheckFatal(t, err)
   113  	err = lom.Load(false, false)
   114  	tassert.CheckFatal(t, err)
   115  	return lom
   116  }