github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/backend/chunker/chunker_internal_test.go (about) 1 package chunker 2 3 import ( 4 "bytes" 5 "context" 6 "flag" 7 "fmt" 8 "io/ioutil" 9 "path" 10 "regexp" 11 "strings" 12 "testing" 13 14 "github.com/rclone/rclone/fs" 15 "github.com/rclone/rclone/fs/hash" 16 "github.com/rclone/rclone/fs/operations" 17 "github.com/rclone/rclone/fstest" 18 "github.com/rclone/rclone/fstest/fstests" 19 "github.com/rclone/rclone/lib/random" 20 "github.com/stretchr/testify/assert" 21 "github.com/stretchr/testify/require" 22 ) 23 24 // Command line flags 25 var ( 26 UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads") 27 ) 28 29 // test that chunking does not break large uploads 30 func testPutLarge(t *testing.T, f *Fs, kilobytes int) { 31 t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) { 32 fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{ 33 ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), 34 Path: fmt.Sprintf("chunker-upload-%dk", kilobytes), 35 Size: int64(kilobytes) * int64(fs.KibiByte), 36 }) 37 }) 38 } 39 40 // test chunk name parser 41 func testChunkNameFormat(t *testing.T, f *Fs) { 42 saveOpt := f.opt 43 defer func() { 44 // restore original settings (f is pointer, f.opt is struct) 45 f.opt = saveOpt 46 _ = f.setChunkNameFormat(f.opt.NameFormat) 47 }() 48 49 assertFormat := func(pattern, wantDataFormat, wantCtrlFormat, wantNameRegexp string) { 50 err := f.setChunkNameFormat(pattern) 51 assert.NoError(t, err) 52 assert.Equal(t, wantDataFormat, f.dataNameFmt) 53 assert.Equal(t, wantCtrlFormat, f.ctrlNameFmt) 54 assert.Equal(t, wantNameRegexp, f.nameRegexp.String()) 55 } 56 57 assertFormatValid := func(pattern string) { 58 err := f.setChunkNameFormat(pattern) 59 assert.NoError(t, err) 60 } 61 62 assertFormatInvalid := func(pattern string) { 63 err := f.setChunkNameFormat(pattern) 64 assert.Error(t, err) 65 } 66 67 assertMakeName := func(wantChunkName, mainName string, chunkNo int, ctrlType, xactID string) { 68 gotChunkName := "" 69 assert.NotPanics(t, func() { 70 gotChunkName = f.makeChunkName(mainName, chunkNo, ctrlType, xactID) 71 }, "makeChunkName(%q,%d,%q,%q) must not panic", mainName, chunkNo, ctrlType, xactID) 72 if gotChunkName != "" { 73 assert.Equal(t, wantChunkName, gotChunkName) 74 } 75 } 76 77 assertMakeNamePanics := func(mainName string, chunkNo int, ctrlType, xactID string) { 78 assert.Panics(t, func() { 79 _ = f.makeChunkName(mainName, chunkNo, ctrlType, xactID) 80 }, "makeChunkName(%q,%d,%q,%q) should panic", mainName, chunkNo, ctrlType, xactID) 81 } 82 83 assertParseName := func(fileName, wantMainName string, wantChunkNo int, wantCtrlType, wantXactID string) { 84 gotMainName, gotChunkNo, gotCtrlType, gotXactID := f.parseChunkName(fileName) 85 assert.Equal(t, wantMainName, gotMainName) 86 assert.Equal(t, wantChunkNo, gotChunkNo) 87 assert.Equal(t, wantCtrlType, gotCtrlType) 88 assert.Equal(t, wantXactID, gotXactID) 89 } 90 91 const newFormatSupported = false // support for patterns not starting with base name (*) 92 93 // valid formats 94 assertFormat(`*.rclone_chunk.###`, `%s.rclone_chunk.%03d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 95 assertFormat(`*.rclone_chunk.#`, `%s.rclone_chunk.%d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 96 assertFormat(`*_chunk_#####`, `%s_chunk_%05d`, `%s_chunk__%s`, `^(.+?)_chunk_(?:([0-9]{5,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 97 assertFormat(`*-chunk-#`, `%s-chunk-%d`, `%s-chunk-_%s`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 98 assertFormat(`*-chunk-#-%^$()[]{}.+-!?:\`, `%s-chunk-%d-%%^$()[]{}.+-!?:\`, `%s-chunk-_%s-%%^$()[]{}.+-!?:\`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))-%\^\$\(\)\[\]\{\}\.\+-!\?:\\(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 99 if newFormatSupported { 100 assertFormat(`_*-chunk-##,`, `_%s-chunk-%02d,`, `_%s-chunk-_%s,`, `^_(.+?)-chunk-(?:([0-9]{2,})|_([a-z][a-z0-9]{2,6})),(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 101 } 102 103 // invalid formats 104 assertFormatInvalid(`chunk-#`) 105 assertFormatInvalid(`*-chunk`) 106 assertFormatInvalid(`*-*-chunk-#`) 107 assertFormatInvalid(`*-chunk-#-#`) 108 assertFormatInvalid(`#-chunk-*`) 109 assertFormatInvalid(`*/#`) 110 111 assertFormatValid(`*#`) 112 assertFormatInvalid(`**#`) 113 assertFormatInvalid(`#*`) 114 assertFormatInvalid(``) 115 assertFormatInvalid(`-`) 116 117 // quick tests 118 if newFormatSupported { 119 assertFormat(`part_*_#`, `part_%s_%d`, `part_%s__%s`, `^part_(.+?)_(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9][0-9a-z]{3,8})\.\.tmp_([0-9]{10,13}))?$`) 120 f.opt.StartFrom = 1 121 122 assertMakeName(`part_fish_1`, "fish", 0, "", "") 123 assertParseName(`part_fish_43`, "fish", 42, "", "") 124 assertMakeName(`part_fish__locks`, "fish", -2, "locks", "") 125 assertParseName(`part_fish__locks`, "fish", -1, "locks", "") 126 assertMakeName(`part_fish__x2y`, "fish", -2, "x2y", "") 127 assertParseName(`part_fish__x2y`, "fish", -1, "x2y", "") 128 assertMakeName(`part_fish_3_0004`, "fish", 2, "", "4") 129 assertParseName(`part_fish_4_0005`, "fish", 3, "", "0005") 130 assertMakeName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -3, "blkinfo", "jj5fvo3wr") 131 assertParseName(`part_fish__blkinfo_zz9fvo3wr`, "fish", -1, "blkinfo", "zz9fvo3wr") 132 133 // old-style temporary suffix (parse only) 134 assertParseName(`part_fish_4..tmp_0000000011`, "fish", 3, "", "000b") 135 assertParseName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -1, "blkinfo", "jj5fvo3wr") 136 } 137 138 // prepare format for long tests 139 assertFormat(`*.chunk.###`, `%s.chunk.%03d`, `%s.chunk._%s`, `^(.+?)\.chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`) 140 f.opt.StartFrom = 2 141 142 // valid data chunks 143 assertMakeName(`fish.chunk.003`, "fish", 1, "", "") 144 assertParseName(`fish.chunk.003`, "fish", 1, "", "") 145 assertMakeName(`fish.chunk.021`, "fish", 19, "", "") 146 assertParseName(`fish.chunk.021`, "fish", 19, "", "") 147 148 // valid temporary data chunks 149 assertMakeName(`fish.chunk.011_4321`, "fish", 9, "", "4321") 150 assertParseName(`fish.chunk.011_4321`, "fish", 9, "", "4321") 151 assertMakeName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc") 152 assertParseName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc") 153 assertMakeName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr") 154 assertParseName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr") 155 assertMakeName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr") 156 assertParseName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr") 157 158 // valid temporary data chunks (old temporary suffix, only parse) 159 assertParseName(`fish.chunk.004..tmp_0000000047`, "fish", 2, "", "001b") 160 assertParseName(`fish.chunk.323..tmp_9994567890123`, "fish", 321, "", "3jjfvo3wr") 161 162 // parsing invalid data chunk names 163 assertParseName(`fish.chunk.3`, "", -1, "", "") 164 assertParseName(`fish.chunk.001`, "", -1, "", "") 165 assertParseName(`fish.chunk.21`, "", -1, "", "") 166 assertParseName(`fish.chunk.-21`, "", -1, "", "") 167 168 assertParseName(`fish.chunk.004abcd`, "", -1, "", "") // missing underscore delimiter 169 assertParseName(`fish.chunk.004__1234`, "", -1, "", "") // extra underscore delimiter 170 assertParseName(`fish.chunk.004_123`, "", -1, "", "") // too short temporary suffix 171 assertParseName(`fish.chunk.004_1234567890`, "", -1, "", "") // too long temporary suffix 172 assertParseName(`fish.chunk.004_-1234`, "", -1, "", "") // temporary suffix must be positive 173 assertParseName(`fish.chunk.004_123E`, "", -1, "", "") // uppercase not allowed 174 assertParseName(`fish.chunk.004_12.3`, "", -1, "", "") // punctuation not allowed 175 176 // parsing invalid data chunk names (old temporary suffix) 177 assertParseName(`fish.chunk.004.tmp_0000000021`, "", -1, "", "") 178 assertParseName(`fish.chunk.003..tmp_123456789`, "", -1, "", "") 179 assertParseName(`fish.chunk.003..tmp_012345678901234567890123456789`, "", -1, "", "") 180 assertParseName(`fish.chunk.323..tmp_12345678901234`, "", -1, "", "") 181 assertParseName(`fish.chunk.003..tmp_-1`, "", -1, "", "") 182 183 // valid control chunks 184 assertMakeName(`fish.chunk._info`, "fish", -1, "info", "") 185 assertMakeName(`fish.chunk._locks`, "fish", -2, "locks", "") 186 assertMakeName(`fish.chunk._blkinfo`, "fish", -3, "blkinfo", "") 187 assertMakeName(`fish.chunk._x2y`, "fish", -4, "x2y", "") 188 189 assertParseName(`fish.chunk._info`, "fish", -1, "info", "") 190 assertParseName(`fish.chunk._locks`, "fish", -1, "locks", "") 191 assertParseName(`fish.chunk._blkinfo`, "fish", -1, "blkinfo", "") 192 assertParseName(`fish.chunk._x2y`, "fish", -1, "x2y", "") 193 194 // valid temporary control chunks 195 assertMakeName(`fish.chunk._info_0001`, "fish", -1, "info", "1") 196 assertMakeName(`fish.chunk._locks_4321`, "fish", -2, "locks", "4321") 197 assertMakeName(`fish.chunk._uploads_abcd`, "fish", -3, "uploads", "abcd") 198 assertMakeName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -4, "blkinfo", "xyzabcdef") 199 assertMakeName(`fish.chunk._x2y_1aaa`, "fish", -5, "x2y", "1aaa") 200 201 assertParseName(`fish.chunk._info_0001`, "fish", -1, "info", "0001") 202 assertParseName(`fish.chunk._locks_4321`, "fish", -1, "locks", "4321") 203 assertParseName(`fish.chunk._uploads_9abc`, "fish", -1, "uploads", "9abc") 204 assertParseName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -1, "blkinfo", "xyzabcdef") 205 assertParseName(`fish.chunk._x2y_1aaa`, "fish", -1, "x2y", "1aaa") 206 207 // valid temporary control chunks (old temporary suffix, parse only) 208 assertParseName(`fish.chunk._info..tmp_0000000047`, "fish", -1, "info", "001b") 209 assertParseName(`fish.chunk._locks..tmp_0000054321`, "fish", -1, "locks", "15wx") 210 assertParseName(`fish.chunk._uploads..tmp_0000000000`, "fish", -1, "uploads", "0000") 211 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123`, "fish", -1, "blkinfo", "3jjfvo3wr") 212 assertParseName(`fish.chunk._x2y..tmp_0000000000`, "fish", -1, "x2y", "0000") 213 214 // parsing invalid control chunk names 215 assertParseName(`fish.chunk.metadata`, "", -1, "", "") // must be prepended by underscore 216 assertParseName(`fish.chunk.info`, "", -1, "", "") 217 assertParseName(`fish.chunk.locks`, "", -1, "", "") 218 assertParseName(`fish.chunk.uploads`, "", -1, "", "") 219 220 assertParseName(`fish.chunk._os`, "", -1, "", "") // too short 221 assertParseName(`fish.chunk._metadata`, "", -1, "", "") // too long 222 assertParseName(`fish.chunk._blockinfo`, "", -1, "", "") // way too long 223 assertParseName(`fish.chunk._4me`, "", -1, "", "") // cannot start with digit 224 assertParseName(`fish.chunk._567`, "", -1, "", "") // cannot be all digits 225 assertParseName(`fish.chunk._me_ta`, "", -1, "", "") // punctuation not allowed 226 assertParseName(`fish.chunk._in-fo`, "", -1, "", "") 227 assertParseName(`fish.chunk._.bin`, "", -1, "", "") 228 assertParseName(`fish.chunk._.2xy`, "", -1, "", "") 229 230 // parsing invalid temporary control chunks 231 assertParseName(`fish.chunk._blkinfo1234`, "", -1, "", "") // missing underscore delimiter 232 assertParseName(`fish.chunk._info__1234`, "", -1, "", "") // extra underscore delimiter 233 assertParseName(`fish.chunk._info_123`, "", -1, "", "") // too short temporary suffix 234 assertParseName(`fish.chunk._info_1234567890`, "", -1, "", "") // too long temporary suffix 235 assertParseName(`fish.chunk._info_-1234`, "", -1, "", "") // temporary suffix must be positive 236 assertParseName(`fish.chunk._info_123E`, "", -1, "", "") // uppercase not allowed 237 assertParseName(`fish.chunk._info_12.3`, "", -1, "", "") // punctuation not allowed 238 239 assertParseName(`fish.chunk._locks..tmp_123456789`, "", -1, "", "") 240 assertParseName(`fish.chunk._meta..tmp_-1`, "", -1, "", "") 241 assertParseName(`fish.chunk._blockinfo..tmp_012345678901234567890123456789`, "", -1, "", "") 242 243 // short control chunk names: 3 letters ok, 1-2 letters not allowed 244 assertMakeName(`fish.chunk._ext`, "fish", -1, "ext", "") 245 assertParseName(`fish.chunk._int`, "fish", -1, "int", "") 246 247 assertMakeNamePanics("fish", -1, "in", "") 248 assertMakeNamePanics("fish", -1, "up", "4") 249 assertMakeNamePanics("fish", -1, "x", "") 250 assertMakeNamePanics("fish", -1, "c", "1z") 251 252 assertMakeName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0") 253 assertMakeName(`fish.chunk._ext_0026`, "fish", -1, "ext", "26") 254 assertMakeName(`fish.chunk._int_0abc`, "fish", -1, "int", "abc") 255 assertMakeName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz") 256 assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr") 257 assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr") 258 259 assertParseName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0000") 260 assertParseName(`fish.chunk._ext_0026`, "fish", -1, "ext", "0026") 261 assertParseName(`fish.chunk._int_0abc`, "fish", -1, "int", "0abc") 262 assertParseName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz") 263 assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr") 264 assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr") 265 266 // base file name can sometimes look like a valid chunk name 267 assertParseName(`fish.chunk.003.chunk.004`, "fish.chunk.003", 2, "", "") 268 assertParseName(`fish.chunk.003.chunk._info`, "fish.chunk.003", -1, "info", "") 269 assertParseName(`fish.chunk.003.chunk._Meta`, "", -1, "", "") 270 271 assertParseName(`fish.chunk._info.chunk.004`, "fish.chunk._info", 2, "", "") 272 assertParseName(`fish.chunk._info.chunk._info`, "fish.chunk._info", -1, "info", "") 273 assertParseName(`fish.chunk._info.chunk._info.chunk._Meta`, "", -1, "", "") 274 275 // base file name looking like a valid chunk name (old temporary suffix) 276 assertParseName(`fish.chunk.003.chunk.005..tmp_0000000022`, "fish.chunk.003", 3, "", "000m") 277 assertParseName(`fish.chunk.003.chunk._x..tmp_0000054321`, "", -1, "", "") 278 assertParseName(`fish.chunk._info.chunk.005..tmp_0000000023`, "fish.chunk._info", 3, "", "000n") 279 assertParseName(`fish.chunk._info.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "") 280 281 assertParseName(`fish.chunk.003.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.003", -1, "blkinfo", "3jjfvo3wr") 282 assertParseName(`fish.chunk._info.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._info", -1, "blkinfo", "3jjfvo3wr") 283 284 assertParseName(`fish.chunk.004..tmp_0000000021.chunk.004`, "fish.chunk.004..tmp_0000000021", 2, "", "") 285 assertParseName(`fish.chunk.004..tmp_0000000021.chunk.005..tmp_0000000025`, "fish.chunk.004..tmp_0000000021", 3, "", "000p") 286 assertParseName(`fish.chunk.004..tmp_0000000021.chunk._info`, "fish.chunk.004..tmp_0000000021", -1, "info", "") 287 assertParseName(`fish.chunk.004..tmp_0000000021.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.004..tmp_0000000021", -1, "blkinfo", "3jjfvo3wr") 288 assertParseName(`fish.chunk.004..tmp_0000000021.chunk._Meta`, "", -1, "", "") 289 assertParseName(`fish.chunk.004..tmp_0000000021.chunk._x..tmp_0000054321`, "", -1, "", "") 290 291 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.004`, "fish.chunk._blkinfo..tmp_9994567890123", 2, "", "") 292 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.005..tmp_0000000026`, "fish.chunk._blkinfo..tmp_9994567890123", 3, "", "000q") 293 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "info", "") 294 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "blkinfo", "3jjfvo3wr") 295 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._Meta`, "", -1, "", "") 296 assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "") 297 298 assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.004`, "fish.chunk._blkinfo..tmp_1234567890123456789", 2, "", "") 299 assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.005..tmp_0000000022`, "fish.chunk._blkinfo..tmp_1234567890123456789", 3, "", "000m") 300 assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "info", "") 301 assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "blkinfo", "3jjfvo3wr") 302 assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._Meta`, "", -1, "", "") 303 assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "") 304 305 // attempts to make invalid chunk names 306 assertMakeNamePanics("fish", -1, "", "") // neither data nor control 307 assertMakeNamePanics("fish", 0, "info", "") // both data and control 308 assertMakeNamePanics("fish", -1, "metadata", "") // control type too long 309 assertMakeNamePanics("fish", -1, "blockinfo", "") // control type way too long 310 assertMakeNamePanics("fish", -1, "2xy", "") // first digit not allowed 311 assertMakeNamePanics("fish", -1, "123", "") // all digits not allowed 312 assertMakeNamePanics("fish", -1, "Meta", "") // only lower case letters allowed 313 assertMakeNamePanics("fish", -1, "in-fo", "") // punctuation not allowed 314 assertMakeNamePanics("fish", -1, "_info", "") 315 assertMakeNamePanics("fish", -1, "info_", "") 316 assertMakeNamePanics("fish", -2, ".bind", "") 317 assertMakeNamePanics("fish", -2, "bind.", "") 318 319 assertMakeNamePanics("fish", -1, "", "1") // neither data nor control 320 assertMakeNamePanics("fish", 0, "info", "23") // both data and control 321 assertMakeNamePanics("fish", -1, "metadata", "45") // control type too long 322 assertMakeNamePanics("fish", -1, "blockinfo", "7") // control type way too long 323 assertMakeNamePanics("fish", -1, "2xy", "abc") // first digit not allowed 324 assertMakeNamePanics("fish", -1, "123", "def") // all digits not allowed 325 assertMakeNamePanics("fish", -1, "Meta", "mnk") // only lower case letters allowed 326 assertMakeNamePanics("fish", -1, "in-fo", "xyz") // punctuation not allowed 327 assertMakeNamePanics("fish", -1, "_info", "5678") 328 assertMakeNamePanics("fish", -1, "info_", "999") 329 assertMakeNamePanics("fish", -2, ".bind", "0") 330 assertMakeNamePanics("fish", -2, "bind.", "0") 331 332 assertMakeNamePanics("fish", 0, "", "1234567890") // temporary suffix too long 333 assertMakeNamePanics("fish", 0, "", "123F4") // uppercase not allowed 334 assertMakeNamePanics("fish", 0, "", "123.") // punctuation not allowed 335 assertMakeNamePanics("fish", 0, "", "_123") 336 } 337 338 func testSmallFileInternals(t *testing.T, f *Fs) { 339 const dir = "small" 340 ctx := context.Background() 341 saveOpt := f.opt 342 defer func() { 343 f.opt.FailHard = false 344 _ = operations.Purge(ctx, f.base, dir) 345 f.opt = saveOpt 346 }() 347 f.opt.FailHard = false 348 349 modTime := fstest.Time("2001-02-03T04:05:06.499999999Z") 350 351 checkSmallFileInternals := func(obj fs.Object) { 352 assert.NotNil(t, obj) 353 o, ok := obj.(*Object) 354 assert.True(t, ok) 355 assert.NotNil(t, o) 356 if o == nil { 357 return 358 } 359 switch { 360 case !f.useMeta: 361 // If meta format is "none", non-chunked file (even empty) 362 // internally is a single chunk without meta object. 363 assert.Nil(t, o.main) 364 assert.True(t, o.isComposite()) // sorry, sometimes a name is misleading 365 assert.Equal(t, 1, len(o.chunks)) 366 case f.hashAll: 367 // Consistent hashing forces meta object on small files too 368 assert.NotNil(t, o.main) 369 assert.True(t, o.isComposite()) 370 assert.Equal(t, 1, len(o.chunks)) 371 default: 372 // normally non-chunked file is kept in the Object's main field 373 assert.NotNil(t, o.main) 374 assert.False(t, o.isComposite()) 375 assert.Equal(t, 0, len(o.chunks)) 376 } 377 } 378 379 checkContents := func(obj fs.Object, contents string) { 380 assert.NotNil(t, obj) 381 assert.Equal(t, int64(len(contents)), obj.Size()) 382 383 r, err := obj.Open(ctx) 384 assert.NoError(t, err) 385 assert.NotNil(t, r) 386 if r == nil { 387 return 388 } 389 data, err := ioutil.ReadAll(r) 390 assert.NoError(t, err) 391 assert.Equal(t, contents, string(data)) 392 _ = r.Close() 393 } 394 395 checkHashsum := func(obj fs.Object) { 396 var ht hash.Type 397 switch { 398 case !f.hashAll: 399 return 400 case f.useMD5: 401 ht = hash.MD5 402 case f.useSHA1: 403 ht = hash.SHA1 404 default: 405 return 406 } 407 // even empty files must have hashsum in consistent mode 408 sum, err := obj.Hash(ctx, ht) 409 assert.NoError(t, err) 410 assert.NotEqual(t, sum, "") 411 } 412 413 checkSmallFile := func(name, contents string) { 414 filename := path.Join(dir, name) 415 item := fstest.Item{Path: filename, ModTime: modTime} 416 _, put := fstests.PutTestContents(ctx, t, f, &item, contents, false) 417 assert.NotNil(t, put) 418 checkSmallFileInternals(put) 419 checkContents(put, contents) 420 checkHashsum(put) 421 422 // objects returned by Put and NewObject must have similar structure 423 obj, err := f.NewObject(ctx, filename) 424 assert.NoError(t, err) 425 assert.NotNil(t, obj) 426 checkSmallFileInternals(obj) 427 checkContents(obj, contents) 428 checkHashsum(obj) 429 430 _ = obj.Remove(ctx) 431 _ = put.Remove(ctx) // for good 432 } 433 434 checkSmallFile("emptyfile", "") 435 checkSmallFile("smallfile", "Ok") 436 } 437 438 func testPreventCorruption(t *testing.T, f *Fs) { 439 if f.opt.ChunkSize > 50 { 440 t.Skip("this test requires small chunks") 441 } 442 const dir = "corrupted" 443 ctx := context.Background() 444 saveOpt := f.opt 445 defer func() { 446 f.opt.FailHard = false 447 _ = operations.Purge(ctx, f.base, dir) 448 f.opt = saveOpt 449 }() 450 f.opt.FailHard = true 451 452 contents := random.String(250) 453 modTime := fstest.Time("2001-02-03T04:05:06.499999999Z") 454 const overlapMessage = "chunk overlap" 455 456 assertOverlapError := func(err error) { 457 assert.Error(t, err) 458 if err != nil { 459 assert.Contains(t, err.Error(), overlapMessage) 460 } 461 } 462 463 newFile := func(name string) fs.Object { 464 item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime} 465 _, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) 466 require.NotNil(t, obj) 467 return obj 468 } 469 billyObj := newFile("billy") 470 471 billyChunkName := func(chunkNo int) string { 472 return f.makeChunkName(billyObj.Remote(), chunkNo, "", "") 473 } 474 475 err := f.Mkdir(ctx, billyChunkName(1)) 476 assertOverlapError(err) 477 478 _, err = f.Move(ctx, newFile("silly1"), billyChunkName(2)) 479 assert.Error(t, err) 480 assert.True(t, err == fs.ErrorCantMove || (err != nil && strings.Contains(err.Error(), overlapMessage))) 481 482 _, err = f.Copy(ctx, newFile("silly2"), billyChunkName(3)) 483 assert.Error(t, err) 484 assert.True(t, err == fs.ErrorCantCopy || (err != nil && strings.Contains(err.Error(), overlapMessage))) 485 486 // accessing chunks in strict mode is prohibited 487 f.opt.FailHard = true 488 billyChunk4Name := billyChunkName(4) 489 billyChunk4, err := f.NewObject(ctx, billyChunk4Name) 490 assertOverlapError(err) 491 492 f.opt.FailHard = false 493 billyChunk4, err = f.NewObject(ctx, billyChunk4Name) 494 assert.NoError(t, err) 495 require.NotNil(t, billyChunk4) 496 497 f.opt.FailHard = true 498 _, err = f.Put(ctx, bytes.NewBufferString(contents), billyChunk4) 499 assertOverlapError(err) 500 501 // you can freely read chunks (if you have an object) 502 r, err := billyChunk4.Open(ctx) 503 assert.NoError(t, err) 504 var chunkContents []byte 505 assert.NotPanics(t, func() { 506 chunkContents, err = ioutil.ReadAll(r) 507 _ = r.Close() 508 }) 509 assert.NoError(t, err) 510 assert.NotEqual(t, contents, string(chunkContents)) 511 512 // but you can't change them 513 err = billyChunk4.Update(ctx, bytes.NewBufferString(contents), newFile("silly3")) 514 assertOverlapError(err) 515 516 // Remove isn't special, you can't corrupt files even if you have an object 517 err = billyChunk4.Remove(ctx) 518 assertOverlapError(err) 519 520 // recreate billy in case it was anyhow corrupted 521 willyObj := newFile("willy") 522 willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", "") 523 f.opt.FailHard = false 524 willyChunk, err := f.NewObject(ctx, willyChunkName) 525 f.opt.FailHard = true 526 assert.NoError(t, err) 527 require.NotNil(t, willyChunk) 528 529 _, err = operations.Copy(ctx, f, willyChunk, willyChunkName, newFile("silly4")) 530 assertOverlapError(err) 531 532 // operations.Move will return error when chunker's Move refused 533 // to corrupt target file, but reverts to copy/delete method 534 // still trying to delete target chunk. Chunker must come to rescue. 535 _, err = operations.Move(ctx, f, willyChunk, willyChunkName, newFile("silly5")) 536 assertOverlapError(err) 537 r, err = willyChunk.Open(ctx) 538 assert.NoError(t, err) 539 assert.NotPanics(t, func() { 540 _, err = ioutil.ReadAll(r) 541 _ = r.Close() 542 }) 543 assert.NoError(t, err) 544 } 545 546 func testChunkNumberOverflow(t *testing.T, f *Fs) { 547 if f.opt.ChunkSize > 50 { 548 t.Skip("this test requires small chunks") 549 } 550 const dir = "wreaked" 551 const wreakNumber = 10200300 552 ctx := context.Background() 553 saveOpt := f.opt 554 defer func() { 555 f.opt.FailHard = false 556 _ = operations.Purge(ctx, f.base, dir) 557 f.opt = saveOpt 558 }() 559 560 modTime := fstest.Time("2001-02-03T04:05:06.499999999Z") 561 contents := random.String(100) 562 563 newFile := func(f fs.Fs, name string) (fs.Object, string) { 564 filename := path.Join(dir, name) 565 item := fstest.Item{Path: filename, ModTime: modTime} 566 _, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) 567 require.NotNil(t, obj) 568 return obj, filename 569 } 570 571 f.opt.FailHard = false 572 file, fileName := newFile(f, "wreaker") 573 wreak, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", "")) 574 575 f.opt.FailHard = false 576 fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision()) 577 _, err := f.NewObject(ctx, fileName) 578 assert.Error(t, err) 579 580 f.opt.FailHard = true 581 _, err = f.List(ctx, dir) 582 assert.Error(t, err) 583 _, err = f.NewObject(ctx, fileName) 584 assert.Error(t, err) 585 586 f.opt.FailHard = false 587 _ = wreak.Remove(ctx) 588 _ = file.Remove(ctx) 589 } 590 591 func testMetadataInput(t *testing.T, f *Fs) { 592 const minChunkForTest = 50 593 if f.opt.ChunkSize < minChunkForTest { 594 t.Skip("this test requires chunks that fit metadata") 595 } 596 597 const dir = "usermeta" 598 ctx := context.Background() 599 saveOpt := f.opt 600 defer func() { 601 f.opt.FailHard = false 602 _ = operations.Purge(ctx, f.base, dir) 603 f.opt = saveOpt 604 }() 605 f.opt.FailHard = false 606 607 modTime := fstest.Time("2001-02-03T04:05:06.499999999Z") 608 609 putFile := func(f fs.Fs, name, contents, message string, check bool) fs.Object { 610 item := fstest.Item{Path: name, ModTime: modTime} 611 _, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check) 612 assert.NotNil(t, obj, message) 613 return obj 614 } 615 616 runSubtest := func(contents, name string) { 617 description := fmt.Sprintf("file with %s metadata", name) 618 filename := path.Join(dir, name) 619 require.True(t, len(contents) > 2 && len(contents) < minChunkForTest, description+" test data is correct") 620 621 part := putFile(f.base, f.makeChunkName(filename, 0, "", ""), "oops", "", true) 622 _ = putFile(f, filename, contents, "upload "+description, false) 623 624 obj, err := f.NewObject(ctx, filename) 625 assert.NoError(t, err, "access "+description) 626 assert.NotNil(t, obj) 627 assert.Equal(t, int64(len(contents)), obj.Size(), "size "+description) 628 629 o, ok := obj.(*Object) 630 assert.NotNil(t, ok) 631 if o != nil { 632 assert.True(t, o.isComposite() && len(o.chunks) == 1, description+" is forced composite") 633 o = nil 634 } 635 636 defer func() { 637 _ = obj.Remove(ctx) 638 _ = part.Remove(ctx) 639 }() 640 641 r, err := obj.Open(ctx) 642 assert.NoError(t, err, "open "+description) 643 assert.NotNil(t, r, "open stream of "+description) 644 if err == nil && r != nil { 645 data, err := ioutil.ReadAll(r) 646 assert.NoError(t, err, "read all of "+description) 647 assert.Equal(t, contents, string(data), description+" contents is ok") 648 _ = r.Close() 649 } 650 } 651 652 metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "") 653 require.NoError(t, err) 654 todaysMeta := string(metaData) 655 runSubtest(todaysMeta, "today") 656 657 pastMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":1`) 658 pastMeta = regexp.MustCompile(`"size":[0-9]+`).ReplaceAllLiteralString(pastMeta, `"size":0`) 659 runSubtest(pastMeta, "past") 660 661 futureMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":999`) 662 futureMeta = regexp.MustCompile(`"nchunks":[0-9]+`).ReplaceAllLiteralString(futureMeta, `"nchunks":0,"x":"y"`) 663 runSubtest(futureMeta, "future") 664 } 665 666 // InternalTest dispatches all internal tests 667 func (f *Fs) InternalTest(t *testing.T) { 668 t.Run("PutLarge", func(t *testing.T) { 669 if *UploadKilobytes <= 0 { 670 t.Skip("-upload-kilobytes is not set") 671 } 672 testPutLarge(t, f, *UploadKilobytes) 673 }) 674 t.Run("ChunkNameFormat", func(t *testing.T) { 675 testChunkNameFormat(t, f) 676 }) 677 t.Run("SmallFileInternals", func(t *testing.T) { 678 testSmallFileInternals(t, f) 679 }) 680 t.Run("PreventCorruption", func(t *testing.T) { 681 testPreventCorruption(t, f) 682 }) 683 t.Run("ChunkNumberOverflow", func(t *testing.T) { 684 testChunkNumberOverflow(t, f) 685 }) 686 t.Run("MetadataInput", func(t *testing.T) { 687 testMetadataInput(t, f) 688 }) 689 } 690 691 var _ fstests.InternalTester = (*Fs)(nil)