github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/tar/reader_test.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package tar 6 7 import ( 8 "bytes" 9 "crypto/md5" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "math" 14 "os" 15 "path" 16 "reflect" 17 "strconv" 18 "strings" 19 "testing" 20 "time" 21 22 "github.com/pachyderm/pachyderm/src/client/pkg/errors" 23 ) 24 25 func TestReader(t *testing.T) { 26 vectors := []struct { 27 file string // Test input file 28 headers []*Header // Expected output headers 29 chksums []string // MD5 checksum of files, leave as nil if not checked 30 err error // Expected error to occur 31 }{{ 32 file: "testdata/gnu.tar", 33 headers: []*Header{{ 34 Name: "small.txt", 35 Mode: 0640, 36 Uid: 73025, 37 Gid: 5000, 38 Size: 5, 39 ModTime: time.Unix(1244428340, 0), 40 Typeflag: '0', 41 Uname: "dsymonds", 42 Gname: "eng", 43 Format: FormatGNU, 44 }, { 45 Name: "small2.txt", 46 Mode: 0640, 47 Uid: 73025, 48 Gid: 5000, 49 Size: 11, 50 ModTime: time.Unix(1244436044, 0), 51 Typeflag: '0', 52 Uname: "dsymonds", 53 Gname: "eng", 54 Format: FormatGNU, 55 }}, 56 chksums: []string{ 57 "e38b27eaccb4391bdec553a7f3ae6b2f", 58 "c65bd2e50a56a2138bf1716f2fd56fe9", 59 }, 60 }, { 61 file: "testdata/sparse-formats.tar", 62 headers: []*Header{{ 63 Name: "sparse-gnu", 64 Mode: 420, 65 Uid: 1000, 66 Gid: 1000, 67 Size: 200, 68 ModTime: time.Unix(1392395740, 0), 69 Typeflag: 0x53, 70 Linkname: "", 71 Uname: "david", 72 Gname: "david", 73 Devmajor: 0, 74 Devminor: 0, 75 Format: FormatGNU, 76 }, { 77 Name: "sparse-posix-0.0", 78 Mode: 420, 79 Uid: 1000, 80 Gid: 1000, 81 Size: 200, 82 ModTime: time.Unix(1392342187, 0), 83 Typeflag: 0x30, 84 Linkname: "", 85 Uname: "david", 86 Gname: "david", 87 Devmajor: 0, 88 Devminor: 0, 89 PAXRecords: map[string]string{ 90 "GNU.sparse.size": "200", 91 "GNU.sparse.numblocks": "95", 92 "GNU.sparse.map": "1,1,3,1,5,1,7,1,9,1,11,1,13,1,15,1,17,1,19,1,21,1,23,1,25,1,27,1,29,1,31,1,33,1,35,1,37,1,39,1,41,1,43,1,45,1,47,1,49,1,51,1,53,1,55,1,57,1,59,1,61,1,63,1,65,1,67,1,69,1,71,1,73,1,75,1,77,1,79,1,81,1,83,1,85,1,87,1,89,1,91,1,93,1,95,1,97,1,99,1,101,1,103,1,105,1,107,1,109,1,111,1,113,1,115,1,117,1,119,1,121,1,123,1,125,1,127,1,129,1,131,1,133,1,135,1,137,1,139,1,141,1,143,1,145,1,147,1,149,1,151,1,153,1,155,1,157,1,159,1,161,1,163,1,165,1,167,1,169,1,171,1,173,1,175,1,177,1,179,1,181,1,183,1,185,1,187,1,189,1", 93 }, 94 Format: FormatPAX, 95 }, { 96 Name: "sparse-posix-0.1", 97 Mode: 420, 98 Uid: 1000, 99 Gid: 1000, 100 Size: 200, 101 ModTime: time.Unix(1392340456, 0), 102 Typeflag: 0x30, 103 Linkname: "", 104 Uname: "david", 105 Gname: "david", 106 Devmajor: 0, 107 Devminor: 0, 108 PAXRecords: map[string]string{ 109 "GNU.sparse.size": "200", 110 "GNU.sparse.numblocks": "95", 111 "GNU.sparse.map": "1,1,3,1,5,1,7,1,9,1,11,1,13,1,15,1,17,1,19,1,21,1,23,1,25,1,27,1,29,1,31,1,33,1,35,1,37,1,39,1,41,1,43,1,45,1,47,1,49,1,51,1,53,1,55,1,57,1,59,1,61,1,63,1,65,1,67,1,69,1,71,1,73,1,75,1,77,1,79,1,81,1,83,1,85,1,87,1,89,1,91,1,93,1,95,1,97,1,99,1,101,1,103,1,105,1,107,1,109,1,111,1,113,1,115,1,117,1,119,1,121,1,123,1,125,1,127,1,129,1,131,1,133,1,135,1,137,1,139,1,141,1,143,1,145,1,147,1,149,1,151,1,153,1,155,1,157,1,159,1,161,1,163,1,165,1,167,1,169,1,171,1,173,1,175,1,177,1,179,1,181,1,183,1,185,1,187,1,189,1", 112 "GNU.sparse.name": "sparse-posix-0.1", 113 }, 114 Format: FormatPAX, 115 }, { 116 Name: "sparse-posix-1.0", 117 Mode: 420, 118 Uid: 1000, 119 Gid: 1000, 120 Size: 200, 121 ModTime: time.Unix(1392337404, 0), 122 Typeflag: 0x30, 123 Linkname: "", 124 Uname: "david", 125 Gname: "david", 126 Devmajor: 0, 127 Devminor: 0, 128 PAXRecords: map[string]string{ 129 "GNU.sparse.major": "1", 130 "GNU.sparse.minor": "0", 131 "GNU.sparse.realsize": "200", 132 "GNU.sparse.name": "sparse-posix-1.0", 133 }, 134 Format: FormatPAX, 135 }, { 136 Name: "end", 137 Mode: 420, 138 Uid: 1000, 139 Gid: 1000, 140 Size: 4, 141 ModTime: time.Unix(1392398319, 0), 142 Typeflag: 0x30, 143 Linkname: "", 144 Uname: "david", 145 Gname: "david", 146 Devmajor: 0, 147 Devminor: 0, 148 Format: FormatGNU, 149 }}, 150 chksums: []string{ 151 "6f53234398c2449fe67c1812d993012f", 152 "6f53234398c2449fe67c1812d993012f", 153 "6f53234398c2449fe67c1812d993012f", 154 "6f53234398c2449fe67c1812d993012f", 155 "b0061974914468de549a2af8ced10316", 156 }, 157 }, { 158 file: "testdata/star.tar", 159 headers: []*Header{{ 160 Name: "small.txt", 161 Mode: 0640, 162 Uid: 73025, 163 Gid: 5000, 164 Size: 5, 165 ModTime: time.Unix(1244592783, 0), 166 Typeflag: '0', 167 Uname: "dsymonds", 168 Gname: "eng", 169 AccessTime: time.Unix(1244592783, 0), 170 ChangeTime: time.Unix(1244592783, 0), 171 }, { 172 Name: "small2.txt", 173 Mode: 0640, 174 Uid: 73025, 175 Gid: 5000, 176 Size: 11, 177 ModTime: time.Unix(1244592783, 0), 178 Typeflag: '0', 179 Uname: "dsymonds", 180 Gname: "eng", 181 AccessTime: time.Unix(1244592783, 0), 182 ChangeTime: time.Unix(1244592783, 0), 183 }}, 184 }, { 185 file: "testdata/v7.tar", 186 headers: []*Header{{ 187 Name: "small.txt", 188 Mode: 0444, 189 Uid: 73025, 190 Gid: 5000, 191 Size: 5, 192 ModTime: time.Unix(1244593104, 0), 193 Typeflag: '0', 194 }, { 195 Name: "small2.txt", 196 Mode: 0444, 197 Uid: 73025, 198 Gid: 5000, 199 Size: 11, 200 ModTime: time.Unix(1244593104, 0), 201 Typeflag: '0', 202 }}, 203 }, { 204 file: "testdata/pax.tar", 205 headers: []*Header{{ 206 Name: "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", 207 Mode: 0664, 208 Uid: 1000, 209 Gid: 1000, 210 Uname: "shane", 211 Gname: "shane", 212 Size: 7, 213 ModTime: time.Unix(1350244992, 23960108), 214 ChangeTime: time.Unix(1350244992, 23960108), 215 AccessTime: time.Unix(1350244992, 23960108), 216 Typeflag: TypeReg, 217 PAXRecords: map[string]string{ 218 "path": "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", 219 "mtime": "1350244992.023960108", 220 "atime": "1350244992.023960108", 221 "ctime": "1350244992.023960108", 222 }, 223 Format: FormatPAX, 224 }, { 225 Name: "a/b", 226 Mode: 0777, 227 Uid: 1000, 228 Gid: 1000, 229 Uname: "shane", 230 Gname: "shane", 231 Size: 0, 232 ModTime: time.Unix(1350266320, 910238425), 233 ChangeTime: time.Unix(1350266320, 910238425), 234 AccessTime: time.Unix(1350266320, 910238425), 235 Typeflag: TypeSymlink, 236 Linkname: "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", 237 PAXRecords: map[string]string{ 238 "linkpath": "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", 239 "mtime": "1350266320.910238425", 240 "atime": "1350266320.910238425", 241 "ctime": "1350266320.910238425", 242 }, 243 Format: FormatPAX, 244 }}, 245 }, { 246 file: "testdata/pax-bad-hdr-file.tar", 247 err: ErrHeader, 248 }, { 249 file: "testdata/pax-bad-mtime-file.tar", 250 err: ErrHeader, 251 }, { 252 file: "testdata/pax-pos-size-file.tar", 253 headers: []*Header{{ 254 Name: "foo", 255 Mode: 0640, 256 Uid: 319973, 257 Gid: 5000, 258 Size: 999, 259 ModTime: time.Unix(1442282516, 0), 260 Typeflag: '0', 261 Uname: "joetsai", 262 Gname: "eng", 263 PAXRecords: map[string]string{ 264 "size": "000000000000000000000999", 265 }, 266 Format: FormatPAX, 267 }}, 268 chksums: []string{ 269 "0afb597b283fe61b5d4879669a350556", 270 }, 271 }, { 272 file: "testdata/pax-records.tar", 273 headers: []*Header{{ 274 Typeflag: TypeReg, 275 Name: "file", 276 Uname: strings.Repeat("long", 10), 277 ModTime: time.Unix(0, 0), 278 PAXRecords: map[string]string{ 279 "GOLANG.pkg": "tar", 280 "comment": "Hello, 世界", 281 "uname": strings.Repeat("long", 10), 282 }, 283 Format: FormatPAX, 284 }}, 285 }, { 286 file: "testdata/pax-global-records.tar", 287 headers: []*Header{{ 288 Typeflag: TypeXGlobalHeader, 289 Name: "global1", 290 PAXRecords: map[string]string{"path": "global1", "mtime": "1500000000.0"}, 291 Format: FormatPAX, 292 }, { 293 Typeflag: TypeReg, 294 Name: "file1", 295 ModTime: time.Unix(0, 0), 296 Format: FormatUSTAR, 297 }, { 298 Typeflag: TypeReg, 299 Name: "file2", 300 PAXRecords: map[string]string{"path": "file2"}, 301 ModTime: time.Unix(0, 0), 302 Format: FormatPAX, 303 }, { 304 Typeflag: TypeXGlobalHeader, 305 Name: "GlobalHead.0.0", 306 PAXRecords: map[string]string{"path": ""}, 307 Format: FormatPAX, 308 }, { 309 Typeflag: TypeReg, 310 Name: "file3", 311 ModTime: time.Unix(0, 0), 312 Format: FormatUSTAR, 313 }, { 314 Typeflag: TypeReg, 315 Name: "file4", 316 ModTime: time.Unix(1400000000, 0), 317 PAXRecords: map[string]string{"mtime": "1400000000"}, 318 Format: FormatPAX, 319 }}, 320 }, { 321 file: "testdata/nil-uid.tar", // golang.org/issue/5290 322 headers: []*Header{{ 323 Name: "P1050238.JPG.log", 324 Mode: 0664, 325 Uid: 0, 326 Gid: 0, 327 Size: 14, 328 ModTime: time.Unix(1365454838, 0), 329 Typeflag: TypeReg, 330 Linkname: "", 331 Uname: "eyefi", 332 Gname: "eyefi", 333 Devmajor: 0, 334 Devminor: 0, 335 Format: FormatGNU, 336 }}, 337 }, { 338 file: "testdata/xattrs.tar", 339 headers: []*Header{{ 340 Name: "small.txt", 341 Mode: 0644, 342 Uid: 1000, 343 Gid: 10, 344 Size: 5, 345 ModTime: time.Unix(1386065770, 448252320), 346 Typeflag: '0', 347 Uname: "alex", 348 Gname: "wheel", 349 AccessTime: time.Unix(1389782991, 419875220), 350 ChangeTime: time.Unix(1389782956, 794414986), 351 Xattrs: map[string]string{ 352 "user.key": "value", 353 "user.key2": "value2", 354 // Interestingly, selinux encodes the terminating null inside the xattr 355 "security.selinux": "unconfined_u:object_r:default_t:s0\x00", 356 }, 357 PAXRecords: map[string]string{ 358 "mtime": "1386065770.44825232", 359 "atime": "1389782991.41987522", 360 "ctime": "1389782956.794414986", 361 "SCHILY.xattr.user.key": "value", 362 "SCHILY.xattr.user.key2": "value2", 363 "SCHILY.xattr.security.selinux": "unconfined_u:object_r:default_t:s0\x00", 364 }, 365 Format: FormatPAX, 366 }, { 367 Name: "small2.txt", 368 Mode: 0644, 369 Uid: 1000, 370 Gid: 10, 371 Size: 11, 372 ModTime: time.Unix(1386065770, 449252304), 373 Typeflag: '0', 374 Uname: "alex", 375 Gname: "wheel", 376 AccessTime: time.Unix(1389782991, 419875220), 377 ChangeTime: time.Unix(1386065770, 449252304), 378 Xattrs: map[string]string{ 379 "security.selinux": "unconfined_u:object_r:default_t:s0\x00", 380 }, 381 PAXRecords: map[string]string{ 382 "mtime": "1386065770.449252304", 383 "atime": "1389782991.41987522", 384 "ctime": "1386065770.449252304", 385 "SCHILY.xattr.security.selinux": "unconfined_u:object_r:default_t:s0\x00", 386 }, 387 Format: FormatPAX, 388 }}, 389 }, { 390 // Matches the behavior of GNU, BSD, and STAR tar utilities. 391 file: "testdata/gnu-multi-hdrs.tar", 392 headers: []*Header{{ 393 Name: "GNU2/GNU2/long-path-name", 394 Linkname: "GNU4/GNU4/long-linkpath-name", 395 ModTime: time.Unix(0, 0), 396 Typeflag: '2', 397 Format: FormatGNU, 398 }}, 399 }, { 400 // GNU tar file with atime and ctime fields set. 401 // Created with the GNU tar v1.27.1. 402 // tar --incremental -S -cvf gnu-incremental.tar test2 403 file: "testdata/gnu-incremental.tar", 404 headers: []*Header{{ 405 Name: "test2/", 406 Mode: 16877, 407 Uid: 1000, 408 Gid: 1000, 409 Size: 14, 410 ModTime: time.Unix(1441973427, 0), 411 Typeflag: 'D', 412 Uname: "rawr", 413 Gname: "dsnet", 414 AccessTime: time.Unix(1441974501, 0), 415 ChangeTime: time.Unix(1441973436, 0), 416 Format: FormatGNU, 417 }, { 418 Name: "test2/foo", 419 Mode: 33188, 420 Uid: 1000, 421 Gid: 1000, 422 Size: 64, 423 ModTime: time.Unix(1441973363, 0), 424 Typeflag: '0', 425 Uname: "rawr", 426 Gname: "dsnet", 427 AccessTime: time.Unix(1441974501, 0), 428 ChangeTime: time.Unix(1441973436, 0), 429 Format: FormatGNU, 430 }, { 431 Name: "test2/sparse", 432 Mode: 33188, 433 Uid: 1000, 434 Gid: 1000, 435 Size: 536870912, 436 ModTime: time.Unix(1441973427, 0), 437 Typeflag: 'S', 438 Uname: "rawr", 439 Gname: "dsnet", 440 AccessTime: time.Unix(1441991948, 0), 441 ChangeTime: time.Unix(1441973436, 0), 442 Format: FormatGNU, 443 }}, 444 }, { 445 // Matches the behavior of GNU and BSD tar utilities. 446 file: "testdata/pax-multi-hdrs.tar", 447 headers: []*Header{{ 448 Name: "bar", 449 Linkname: "PAX4/PAX4/long-linkpath-name", 450 ModTime: time.Unix(0, 0), 451 Typeflag: '2', 452 PAXRecords: map[string]string{ 453 "linkpath": "PAX4/PAX4/long-linkpath-name", 454 }, 455 Format: FormatPAX, 456 }}, 457 }, { 458 // Both BSD and GNU tar truncate long names at first NUL even 459 // if there is data following that NUL character. 460 // This is reasonable as GNU long names are C-strings. 461 file: "testdata/gnu-long-nul.tar", 462 headers: []*Header{{ 463 Name: "0123456789", 464 Mode: 0644, 465 Uid: 1000, 466 Gid: 1000, 467 ModTime: time.Unix(1486082191, 0), 468 Typeflag: '0', 469 Uname: "rawr", 470 Gname: "dsnet", 471 Format: FormatGNU, 472 }}, 473 }, { 474 // This archive was generated by Writer but is readable by both 475 // GNU and BSD tar utilities. 476 // The archive generated by GNU is nearly byte-for-byte identical 477 // to the Go version except the Go version sets a negative Devminor 478 // just to force the GNU format. 479 file: "testdata/gnu-utf8.tar", 480 headers: []*Header{{ 481 Name: "☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹☺☻☹", 482 Mode: 0644, 483 Uid: 1000, Gid: 1000, 484 ModTime: time.Unix(0, 0), 485 Typeflag: '0', 486 Uname: "☺", 487 Gname: "⚹", 488 Format: FormatGNU, 489 }}, 490 }, { 491 // This archive was generated by Writer but is readable by both 492 // GNU and BSD tar utilities. 493 // The archive generated by GNU is nearly byte-for-byte identical 494 // to the Go version except the Go version sets a negative Devminor 495 // just to force the GNU format. 496 file: "testdata/gnu-not-utf8.tar", 497 headers: []*Header{{ 498 Name: "hi\x80\x81\x82\x83bye", 499 Mode: 0644, 500 Uid: 1000, 501 Gid: 1000, 502 ModTime: time.Unix(0, 0), 503 Typeflag: '0', 504 Uname: "rawr", 505 Gname: "dsnet", 506 Format: FormatGNU, 507 }}, 508 }, { 509 // BSD tar v3.1.2 and GNU tar v1.27.1 both rejects PAX records 510 // with NULs in the key. 511 file: "testdata/pax-nul-xattrs.tar", 512 err: ErrHeader, 513 }, { 514 // BSD tar v3.1.2 rejects a PAX path with NUL in the value, while 515 // GNU tar v1.27.1 simply truncates at first NUL. 516 // We emulate the behavior of BSD since it is strange doing NUL 517 // truncations since PAX records are length-prefix strings instead 518 // of NUL-terminated C-strings. 519 file: "testdata/pax-nul-path.tar", 520 err: ErrHeader, 521 }, { 522 file: "testdata/neg-size.tar", 523 err: ErrHeader, 524 }, { 525 file: "testdata/issue10968.tar", 526 err: ErrHeader, 527 }, { 528 file: "testdata/issue11169.tar", 529 err: ErrHeader, 530 }, { 531 file: "testdata/issue12435.tar", 532 err: ErrHeader, 533 }, { 534 // Ensure that we can read back the original Header as written with 535 // a buggy pre-Go1.8 tar.Writer. 536 file: "testdata/invalid-go17.tar", 537 headers: []*Header{{ 538 Name: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/foo", 539 Uid: 010000000, 540 ModTime: time.Unix(0, 0), 541 Typeflag: '0', 542 }}, 543 }, { 544 // USTAR archive with a regular entry with non-zero device numbers. 545 file: "testdata/ustar-file-devs.tar", 546 headers: []*Header{{ 547 Name: "file", 548 Mode: 0644, 549 Typeflag: '0', 550 ModTime: time.Unix(0, 0), 551 Devmajor: 1, 552 Devminor: 1, 553 Format: FormatUSTAR, 554 }}, 555 }, { 556 // Generated by Go, works on BSD tar v3.1.2 and GNU tar v.1.27.1. 557 file: "testdata/gnu-nil-sparse-data.tar", 558 headers: []*Header{{ 559 Name: "sparse.db", 560 Typeflag: TypeGNUSparse, 561 Size: 1000, 562 ModTime: time.Unix(0, 0), 563 Format: FormatGNU, 564 }}, 565 }, { 566 // Generated by Go, works on BSD tar v3.1.2 and GNU tar v.1.27.1. 567 file: "testdata/gnu-nil-sparse-hole.tar", 568 headers: []*Header{{ 569 Name: "sparse.db", 570 Typeflag: TypeGNUSparse, 571 Size: 1000, 572 ModTime: time.Unix(0, 0), 573 Format: FormatGNU, 574 }}, 575 }, { 576 // Generated by Go, works on BSD tar v3.1.2 and GNU tar v.1.27.1. 577 file: "testdata/pax-nil-sparse-data.tar", 578 headers: []*Header{{ 579 Name: "sparse.db", 580 Typeflag: TypeReg, 581 Size: 1000, 582 ModTime: time.Unix(0, 0), 583 PAXRecords: map[string]string{ 584 "size": "1512", 585 "GNU.sparse.major": "1", 586 "GNU.sparse.minor": "0", 587 "GNU.sparse.realsize": "1000", 588 "GNU.sparse.name": "sparse.db", 589 }, 590 Format: FormatPAX, 591 }}, 592 }, { 593 // Generated by Go, works on BSD tar v3.1.2 and GNU tar v.1.27.1. 594 file: "testdata/pax-nil-sparse-hole.tar", 595 headers: []*Header{{ 596 Name: "sparse.db", 597 Typeflag: TypeReg, 598 Size: 1000, 599 ModTime: time.Unix(0, 0), 600 PAXRecords: map[string]string{ 601 "size": "512", 602 "GNU.sparse.major": "1", 603 "GNU.sparse.minor": "0", 604 "GNU.sparse.realsize": "1000", 605 "GNU.sparse.name": "sparse.db", 606 }, 607 Format: FormatPAX, 608 }}, 609 }, { 610 file: "testdata/trailing-slash.tar", 611 headers: []*Header{{ 612 Typeflag: TypeDir, 613 Name: strings.Repeat("123456789/", 30), 614 ModTime: time.Unix(0, 0), 615 PAXRecords: map[string]string{ 616 "path": strings.Repeat("123456789/", 30), 617 }, 618 Format: FormatPAX, 619 }}, 620 }} 621 622 for _, v := range vectors { 623 t.Run(path.Base(v.file), func(t *testing.T) { 624 f, err := os.Open(v.file) 625 if err != nil { 626 t.Fatalf("unexpected error: %v", err) 627 } 628 defer f.Close() 629 630 // Capture all headers and checksums. 631 var ( 632 tr = NewReader(f) 633 hdrs []*Header 634 chksums []string 635 rdbuf = make([]byte, 8) 636 ) 637 for { 638 var hdr *Header 639 hdr, err = tr.Next() 640 if err != nil { 641 if errors.Is(err, io.EOF) { 642 err = nil // Expected error 643 } 644 break 645 } 646 hdrs = append(hdrs, hdr) 647 648 if v.chksums == nil { 649 continue 650 } 651 h := md5.New() 652 _, err = io.CopyBuffer(h, tr, rdbuf) // Effectively an incremental read 653 if err != nil { 654 break 655 } 656 chksums = append(chksums, fmt.Sprintf("%x", h.Sum(nil))) 657 } 658 659 for i, hdr := range hdrs { 660 if i >= len(v.headers) { 661 t.Fatalf("entry %d: unexpected header:\ngot %+v", i, *hdr) 662 continue 663 } 664 if !reflect.DeepEqual(*hdr, *v.headers[i]) { 665 t.Fatalf("entry %d: incorrect header:\ngot %+v\nwant %+v", i, *hdr, *v.headers[i]) 666 } 667 } 668 if len(hdrs) != len(v.headers) { 669 t.Fatalf("got %d headers, want %d headers", len(hdrs), len(v.headers)) 670 } 671 672 for i, sum := range chksums { 673 if i >= len(v.chksums) { 674 t.Fatalf("entry %d: unexpected sum: got %s", i, sum) 675 continue 676 } 677 if sum != v.chksums[i] { 678 t.Fatalf("entry %d: incorrect checksum: got %s, want %s", i, sum, v.chksums[i]) 679 } 680 } 681 682 if !errors.Is(err, v.err) { 683 t.Fatalf("unexpected error: got %v, want %v", err, v.err) 684 } 685 f.Close() 686 }) 687 } 688 } 689 690 func TestPartialRead(t *testing.T) { 691 type testCase struct { 692 cnt int // Number of bytes to read 693 output string // Expected value of string read 694 } 695 vectors := []struct { 696 file string 697 cases []testCase 698 }{{ 699 file: "testdata/gnu.tar", 700 cases: []testCase{ 701 {4, "Kilt"}, 702 {6, "Google"}, 703 }, 704 }, { 705 file: "testdata/sparse-formats.tar", 706 cases: []testCase{ 707 {2, "\x00G"}, 708 {4, "\x00G\x00o"}, 709 {6, "\x00G\x00o\x00G"}, 710 {8, "\x00G\x00o\x00G\x00o"}, 711 {4, "end\n"}, 712 }, 713 }} 714 715 for _, v := range vectors { 716 t.Run(path.Base(v.file), func(t *testing.T) { 717 f, err := os.Open(v.file) 718 if err != nil { 719 t.Fatalf("Open() error: %v", err) 720 } 721 defer f.Close() 722 723 tr := NewReader(f) 724 for i, tc := range v.cases { 725 hdr, err := tr.Next() 726 if err != nil || hdr == nil { 727 t.Fatalf("entry %d, Next(): got %v, want %v", i, err, nil) 728 } 729 buf := make([]byte, tc.cnt) 730 if _, err := io.ReadFull(tr, buf); err != nil { 731 t.Fatalf("entry %d, ReadFull(): got %v, want %v", i, err, nil) 732 } 733 if string(buf) != tc.output { 734 t.Fatalf("entry %d, ReadFull(): got %q, want %q", i, string(buf), tc.output) 735 } 736 } 737 738 if _, err := tr.Next(); !errors.Is(err, io.EOF) { 739 t.Fatalf("Next(): got %v, want EOF", err) 740 } 741 }) 742 } 743 } 744 745 func TestUninitializedRead(t *testing.T) { 746 f, err := os.Open("testdata/gnu.tar") 747 if err != nil { 748 t.Fatalf("Unexpected error: %v", err) 749 } 750 defer f.Close() 751 752 tr := NewReader(f) 753 _, err = tr.Read([]byte{}) 754 if err == nil || !errors.Is(err, io.EOF) { 755 t.Errorf("Unexpected error: %v, wanted %v", err, io.EOF) 756 } 757 758 } 759 760 type reader struct{ io.Reader } 761 type readSeeker struct{ io.ReadSeeker } 762 type readBadSeeker struct{ io.ReadSeeker } 763 764 func (rbs *readBadSeeker) Seek(int64, int) (int64, error) { return 0, errors.Errorf("illegal seek") } 765 766 // TestReadTruncation test the ending condition on various truncated files and 767 // that truncated files are still detected even if the underlying io.Reader 768 // satisfies io.Seeker. 769 func TestReadTruncation(t *testing.T) { 770 var ss []string 771 for _, p := range []string{ 772 "testdata/gnu.tar", 773 "testdata/ustar-file-reg.tar", 774 "testdata/pax-path-hdr.tar", 775 "testdata/sparse-formats.tar", 776 } { 777 buf, err := ioutil.ReadFile(p) 778 if err != nil { 779 t.Fatalf("unexpected error: %v", err) 780 } 781 ss = append(ss, string(buf)) 782 } 783 784 data1, data2, pax, sparse := ss[0], ss[1], ss[2], ss[3] 785 data2 += strings.Repeat("\x00", 10*512) 786 trash := strings.Repeat("garbage ", 64) // Exactly 512 bytes 787 788 vectors := []struct { 789 input string // Input stream 790 cnt int // Expected number of headers read 791 err error // Expected error outcome 792 }{ 793 {"", 0, io.EOF}, // Empty file is a "valid" tar file 794 {data1[:511], 0, io.ErrUnexpectedEOF}, 795 {data1[:512], 1, io.ErrUnexpectedEOF}, 796 {data1[:1024], 1, io.EOF}, 797 {data1[:1536], 2, io.ErrUnexpectedEOF}, 798 {data1[:2048], 2, io.EOF}, 799 {data1, 2, io.EOF}, 800 {data1[:2048] + data2[:1536], 3, io.EOF}, 801 {data2[:511], 0, io.ErrUnexpectedEOF}, 802 {data2[:512], 1, io.ErrUnexpectedEOF}, 803 {data2[:1195], 1, io.ErrUnexpectedEOF}, 804 {data2[:1196], 1, io.EOF}, // Exact end of data and start of padding 805 {data2[:1200], 1, io.EOF}, 806 {data2[:1535], 1, io.EOF}, 807 {data2[:1536], 1, io.EOF}, // Exact end of padding 808 {data2[:1536] + trash[:1], 1, io.ErrUnexpectedEOF}, 809 {data2[:1536] + trash[:511], 1, io.ErrUnexpectedEOF}, 810 {data2[:1536] + trash, 1, ErrHeader}, 811 {data2[:2048], 1, io.EOF}, // Exactly 1 empty block 812 {data2[:2048] + trash[:1], 1, io.ErrUnexpectedEOF}, 813 {data2[:2048] + trash[:511], 1, io.ErrUnexpectedEOF}, 814 {data2[:2048] + trash, 1, ErrHeader}, 815 {data2[:2560], 1, io.EOF}, // Exactly 2 empty blocks (normal end-of-stream) 816 {data2[:2560] + trash[:1], 1, io.EOF}, 817 {data2[:2560] + trash[:511], 1, io.EOF}, 818 {data2[:2560] + trash, 1, io.EOF}, 819 {data2[:3072], 1, io.EOF}, 820 {pax, 0, io.EOF}, // PAX header without data is a "valid" tar file 821 {pax + trash[:1], 0, io.ErrUnexpectedEOF}, 822 {pax + trash[:511], 0, io.ErrUnexpectedEOF}, 823 {sparse[:511], 0, io.ErrUnexpectedEOF}, 824 {sparse[:512], 0, io.ErrUnexpectedEOF}, 825 {sparse[:3584], 1, io.EOF}, 826 {sparse[:9200], 1, io.EOF}, // Terminate in padding of sparse header 827 {sparse[:9216], 1, io.EOF}, 828 {sparse[:9728], 2, io.ErrUnexpectedEOF}, 829 {sparse[:10240], 2, io.EOF}, 830 {sparse[:11264], 2, io.ErrUnexpectedEOF}, 831 {sparse, 5, io.EOF}, 832 {sparse + trash, 5, io.EOF}, 833 } 834 835 for i, v := range vectors { 836 for j := 0; j < 6; j++ { 837 var tr *Reader 838 var s1, s2 string 839 840 switch j { 841 case 0: 842 tr = NewReader(&reader{strings.NewReader(v.input)}) 843 s1, s2 = "io.Reader", "auto" 844 case 1: 845 tr = NewReader(&reader{strings.NewReader(v.input)}) 846 s1, s2 = "io.Reader", "manual" 847 case 2: 848 tr = NewReader(&readSeeker{strings.NewReader(v.input)}) 849 s1, s2 = "io.ReadSeeker", "auto" 850 case 3: 851 tr = NewReader(&readSeeker{strings.NewReader(v.input)}) 852 s1, s2 = "io.ReadSeeker", "manual" 853 case 4: 854 tr = NewReader(&readBadSeeker{strings.NewReader(v.input)}) 855 s1, s2 = "ReadBadSeeker", "auto" 856 case 5: 857 tr = NewReader(&readBadSeeker{strings.NewReader(v.input)}) 858 s1, s2 = "ReadBadSeeker", "manual" 859 } 860 861 var cnt int 862 var err error 863 for { 864 if _, err = tr.Next(); err != nil { 865 break 866 } 867 cnt++ 868 if s2 == "manual" { 869 if _, err = tr.writeTo(ioutil.Discard); err != nil { 870 break 871 } 872 } 873 } 874 if !errors.Is(err, v.err) { 875 t.Errorf("test %d, NewReader(%s) with %s discard: got %v, want %v", 876 i, s1, s2, err, v.err) 877 } 878 if cnt != v.cnt { 879 t.Errorf("test %d, NewReader(%s) with %s discard: got %d headers, want %d headers", 880 i, s1, s2, cnt, v.cnt) 881 } 882 } 883 } 884 } 885 886 // TestReadHeaderOnly tests that Reader does not attempt to read special 887 // header-only files. 888 func TestReadHeaderOnly(t *testing.T) { 889 f, err := os.Open("testdata/hdr-only.tar") 890 if err != nil { 891 t.Fatalf("unexpected error: %v", err) 892 } 893 defer f.Close() 894 895 var hdrs []*Header 896 tr := NewReader(f) 897 for { 898 hdr, err := tr.Next() 899 if errors.Is(err, io.EOF) { 900 break 901 } 902 if err != nil { 903 t.Errorf("Next(): got %v, want %v", err, nil) 904 continue 905 } 906 hdrs = append(hdrs, hdr) 907 908 // If a special flag, we should read nothing. 909 cnt, _ := io.ReadFull(tr, []byte{0}) 910 if cnt > 0 && hdr.Typeflag != TypeReg { 911 t.Errorf("ReadFull(...): got %d bytes, want 0 bytes", cnt) 912 } 913 } 914 915 // File is crafted with 16 entries. The later 8 are identical to the first 916 // 8 except that the size is set. 917 if len(hdrs) != 16 { 918 t.Fatalf("len(hdrs): got %d, want %d", len(hdrs), 16) 919 } 920 for i := 0; i < 8; i++ { 921 hdr1, hdr2 := hdrs[i+0], hdrs[i+8] 922 hdr1.Size, hdr2.Size = 0, 0 923 if !reflect.DeepEqual(*hdr1, *hdr2) { 924 t.Errorf("incorrect header:\ngot %+v\nwant %+v", *hdr1, *hdr2) 925 } 926 } 927 } 928 929 func TestMergePAX(t *testing.T) { 930 vectors := []struct { 931 in map[string]string 932 want *Header 933 ok bool 934 }{{ 935 in: map[string]string{ 936 "path": "a/b/c", 937 "uid": "1000", 938 "mtime": "1350244992.023960108", 939 }, 940 want: &Header{ 941 Name: "a/b/c", 942 Uid: 1000, 943 ModTime: time.Unix(1350244992, 23960108), 944 PAXRecords: map[string]string{ 945 "path": "a/b/c", 946 "uid": "1000", 947 "mtime": "1350244992.023960108", 948 }, 949 }, 950 ok: true, 951 }, { 952 in: map[string]string{ 953 "gid": "gtgergergersagersgers", 954 }, 955 ok: false, 956 }, { 957 in: map[string]string{ 958 "missing": "missing", 959 "SCHILY.xattr.key": "value", 960 }, 961 want: &Header{ 962 Xattrs: map[string]string{"key": "value"}, 963 PAXRecords: map[string]string{ 964 "missing": "missing", 965 "SCHILY.xattr.key": "value", 966 }, 967 }, 968 ok: true, 969 }} 970 971 for i, v := range vectors { 972 got := new(Header) 973 err := mergePAX(got, v.in) 974 if v.ok && !reflect.DeepEqual(*got, *v.want) { 975 t.Errorf("test %d, mergePAX(...):\ngot %+v\nwant %+v", i, *got, *v.want) 976 } 977 if ok := err == nil; ok != v.ok { 978 t.Errorf("test %d, mergePAX(...): got %v, want %v", i, ok, v.ok) 979 } 980 } 981 } 982 983 func TestParsePAX(t *testing.T) { 984 vectors := []struct { 985 in string 986 want map[string]string 987 ok bool 988 }{ 989 {"", nil, true}, 990 {"6 k=1\n", map[string]string{"k": "1"}, true}, 991 {"10 a=name\n", map[string]string{"a": "name"}, true}, 992 {"9 a=name\n", map[string]string{"a": "name"}, true}, 993 {"30 mtime=1350244992.023960108\n", map[string]string{"mtime": "1350244992.023960108"}, true}, 994 {"3 somelongkey=\n", nil, false}, 995 {"50 tooshort=\n", nil, false}, 996 {"13 key1=haha\n13 key2=nana\n13 key3=kaka\n", 997 map[string]string{"key1": "haha", "key2": "nana", "key3": "kaka"}, true}, 998 {"13 key1=val1\n13 key2=val2\n8 key1=\n", 999 map[string]string{"key1": "", "key2": "val2"}, true}, 1000 {"22 GNU.sparse.size=10\n26 GNU.sparse.numblocks=2\n" + 1001 "23 GNU.sparse.offset=1\n25 GNU.sparse.numbytes=2\n" + 1002 "23 GNU.sparse.offset=3\n25 GNU.sparse.numbytes=4\n", 1003 map[string]string{paxGNUSparseSize: "10", paxGNUSparseNumBlocks: "2", paxGNUSparseMap: "1,2,3,4"}, true}, 1004 {"22 GNU.sparse.size=10\n26 GNU.sparse.numblocks=1\n" + 1005 "25 GNU.sparse.numbytes=2\n23 GNU.sparse.offset=1\n", 1006 nil, false}, 1007 {"22 GNU.sparse.size=10\n26 GNU.sparse.numblocks=1\n" + 1008 "25 GNU.sparse.offset=1,2\n25 GNU.sparse.numbytes=2\n", 1009 nil, false}, 1010 } 1011 1012 for i, v := range vectors { 1013 r := strings.NewReader(v.in) 1014 got, err := parsePAX(r) 1015 if !reflect.DeepEqual(got, v.want) && !(len(got) == 0 && len(v.want) == 0) { 1016 t.Errorf("test %d, parsePAX():\ngot %v\nwant %v", i, got, v.want) 1017 } 1018 if ok := err == nil; ok != v.ok { 1019 t.Errorf("test %d, parsePAX(): got %v, want %v", i, ok, v.ok) 1020 } 1021 } 1022 } 1023 1024 func TestReadOldGNUSparseMap(t *testing.T) { 1025 populateSparseMap := func(sa sparseArray, sps []string) []string { 1026 for i := 0; len(sps) > 0 && i < sa.MaxEntries(); i++ { 1027 copy(sa.Entry(i), sps[0]) 1028 sps = sps[1:] 1029 } 1030 if len(sps) > 0 { 1031 copy(sa.IsExtended(), "\x80") 1032 } 1033 return sps 1034 } 1035 1036 makeInput := func(format Format, size string, sps ...string) (out []byte) { 1037 // Write the initial GNU header. 1038 var blk block 1039 gnu := blk.GNU() 1040 sparse := gnu.Sparse() 1041 copy(gnu.RealSize(), size) 1042 sps = populateSparseMap(sparse, sps) 1043 if format != FormatUnknown { 1044 blk.SetFormat(format) 1045 } 1046 out = append(out, blk[:]...) 1047 1048 // Write extended sparse blocks. 1049 for len(sps) > 0 { 1050 var blk block 1051 sps = populateSparseMap(blk.Sparse(), sps) 1052 out = append(out, blk[:]...) 1053 } 1054 return out 1055 } 1056 1057 makeSparseStrings := func(sp []sparseEntry) (out []string) { 1058 var f formatter 1059 for _, s := range sp { 1060 var b [24]byte 1061 f.formatNumeric(b[:12], s.Offset) 1062 f.formatNumeric(b[12:], s.Length) 1063 out = append(out, string(b[:])) 1064 } 1065 return out 1066 } 1067 1068 vectors := []struct { 1069 input []byte 1070 wantMap sparseDatas 1071 wantSize int64 1072 wantErr error 1073 }{{ 1074 input: makeInput(FormatUnknown, ""), 1075 wantErr: ErrHeader, 1076 }, { 1077 input: makeInput(FormatGNU, "1234", "fewa"), 1078 wantSize: 01234, 1079 wantErr: ErrHeader, 1080 }, { 1081 input: makeInput(FormatGNU, "0031"), 1082 wantSize: 031, 1083 }, { 1084 input: makeInput(FormatGNU, "80"), 1085 wantErr: ErrHeader, 1086 }, { 1087 input: makeInput(FormatGNU, "1234", 1088 makeSparseStrings(sparseDatas{{0, 0}, {1, 1}})...), 1089 wantMap: sparseDatas{{0, 0}, {1, 1}}, 1090 wantSize: 01234, 1091 }, { 1092 input: makeInput(FormatGNU, "1234", 1093 append(makeSparseStrings(sparseDatas{{0, 0}, {1, 1}}), []string{"", "blah"}...)...), 1094 wantMap: sparseDatas{{0, 0}, {1, 1}}, 1095 wantSize: 01234, 1096 }, { 1097 input: makeInput(FormatGNU, "3333", 1098 makeSparseStrings(sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}})...), 1099 wantMap: sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}}, 1100 wantSize: 03333, 1101 }, { 1102 input: makeInput(FormatGNU, "", 1103 append(append( 1104 makeSparseStrings(sparseDatas{{0, 1}, {2, 1}}), 1105 []string{"", ""}...), 1106 makeSparseStrings(sparseDatas{{4, 1}, {6, 1}})...)...), 1107 wantMap: sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}}, 1108 }, { 1109 input: makeInput(FormatGNU, "", 1110 makeSparseStrings(sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}, {8, 1}, {10, 1}})...)[:blockSize], 1111 wantErr: io.ErrUnexpectedEOF, 1112 }, { 1113 input: makeInput(FormatGNU, "", 1114 makeSparseStrings(sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}, {8, 1}, {10, 1}})...)[:3*blockSize/2], 1115 wantErr: io.ErrUnexpectedEOF, 1116 }, { 1117 input: makeInput(FormatGNU, "", 1118 makeSparseStrings(sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}, {8, 1}, {10, 1}})...), 1119 wantMap: sparseDatas{{0, 1}, {2, 1}, {4, 1}, {6, 1}, {8, 1}, {10, 1}}, 1120 }, { 1121 input: makeInput(FormatGNU, "", 1122 makeSparseStrings(sparseDatas{{10 << 30, 512}, {20 << 30, 512}})...), 1123 wantMap: sparseDatas{{10 << 30, 512}, {20 << 30, 512}}, 1124 }} 1125 1126 for i, v := range vectors { 1127 var blk block 1128 var hdr Header 1129 v.input = v.input[copy(blk[:], v.input):] 1130 tr := Reader{r: bytes.NewReader(v.input)} 1131 got, err := tr.readOldGNUSparseMap(&hdr, &blk) 1132 if !equalSparseEntries(got, v.wantMap) { 1133 t.Errorf("test %d, readOldGNUSparseMap(): got %v, want %v", i, got, v.wantMap) 1134 } 1135 if !errors.Is(err, v.wantErr) { 1136 t.Errorf("test %d, readOldGNUSparseMap() = %v, want %v", i, err, v.wantErr) 1137 } 1138 if hdr.Size != v.wantSize { 1139 t.Errorf("test %d, Header.Size = %d, want %d", i, hdr.Size, v.wantSize) 1140 } 1141 } 1142 } 1143 1144 func TestReadGNUSparsePAXHeaders(t *testing.T) { 1145 padInput := func(s string) string { 1146 return s + string(zeroBlock[:blockPadding(int64(len(s)))]) 1147 } 1148 1149 vectors := []struct { 1150 inputData string 1151 inputHdrs map[string]string 1152 wantMap sparseDatas 1153 wantSize int64 1154 wantName string 1155 wantErr error 1156 }{{ 1157 inputHdrs: nil, 1158 wantErr: nil, 1159 }, { 1160 inputHdrs: map[string]string{ 1161 paxGNUSparseNumBlocks: strconv.FormatInt(math.MaxInt64, 10), 1162 paxGNUSparseMap: "0,1,2,3", 1163 }, 1164 wantErr: ErrHeader, 1165 }, { 1166 inputHdrs: map[string]string{ 1167 paxGNUSparseNumBlocks: "4\x00", 1168 paxGNUSparseMap: "0,1,2,3", 1169 }, 1170 wantErr: ErrHeader, 1171 }, { 1172 inputHdrs: map[string]string{ 1173 paxGNUSparseNumBlocks: "4", 1174 paxGNUSparseMap: "0,1,2,3", 1175 }, 1176 wantErr: ErrHeader, 1177 }, { 1178 inputHdrs: map[string]string{ 1179 paxGNUSparseNumBlocks: "2", 1180 paxGNUSparseMap: "0,1,2,3", 1181 }, 1182 wantMap: sparseDatas{{0, 1}, {2, 3}}, 1183 }, { 1184 inputHdrs: map[string]string{ 1185 paxGNUSparseNumBlocks: "2", 1186 paxGNUSparseMap: "0, 1,2,3", 1187 }, 1188 wantErr: ErrHeader, 1189 }, { 1190 inputHdrs: map[string]string{ 1191 paxGNUSparseNumBlocks: "2", 1192 paxGNUSparseMap: "0,1,02,3", 1193 paxGNUSparseRealSize: "4321", 1194 }, 1195 wantMap: sparseDatas{{0, 1}, {2, 3}}, 1196 wantSize: 4321, 1197 }, { 1198 inputHdrs: map[string]string{ 1199 paxGNUSparseNumBlocks: "2", 1200 paxGNUSparseMap: "0,one1,2,3", 1201 }, 1202 wantErr: ErrHeader, 1203 }, { 1204 inputHdrs: map[string]string{ 1205 paxGNUSparseMajor: "0", 1206 paxGNUSparseMinor: "0", 1207 paxGNUSparseNumBlocks: "2", 1208 paxGNUSparseMap: "0,1,2,3", 1209 paxGNUSparseSize: "1234", 1210 paxGNUSparseRealSize: "4321", 1211 paxGNUSparseName: "realname", 1212 }, 1213 wantMap: sparseDatas{{0, 1}, {2, 3}}, 1214 wantSize: 1234, 1215 wantName: "realname", 1216 }, { 1217 inputHdrs: map[string]string{ 1218 paxGNUSparseMajor: "0", 1219 paxGNUSparseMinor: "0", 1220 paxGNUSparseNumBlocks: "1", 1221 paxGNUSparseMap: "10737418240,512", 1222 paxGNUSparseSize: "10737418240", 1223 paxGNUSparseName: "realname", 1224 }, 1225 wantMap: sparseDatas{{10737418240, 512}}, 1226 wantSize: 10737418240, 1227 wantName: "realname", 1228 }, { 1229 inputHdrs: map[string]string{ 1230 paxGNUSparseMajor: "0", 1231 paxGNUSparseMinor: "0", 1232 paxGNUSparseNumBlocks: "0", 1233 paxGNUSparseMap: "", 1234 }, 1235 wantMap: sparseDatas{}, 1236 }, { 1237 inputHdrs: map[string]string{ 1238 paxGNUSparseMajor: "0", 1239 paxGNUSparseMinor: "1", 1240 paxGNUSparseNumBlocks: "4", 1241 paxGNUSparseMap: "0,5,10,5,20,5,30,5", 1242 }, 1243 wantMap: sparseDatas{{0, 5}, {10, 5}, {20, 5}, {30, 5}}, 1244 }, { 1245 inputHdrs: map[string]string{ 1246 paxGNUSparseMajor: "1", 1247 paxGNUSparseMinor: "0", 1248 paxGNUSparseNumBlocks: "4", 1249 paxGNUSparseMap: "0,5,10,5,20,5,30,5", 1250 }, 1251 wantErr: io.ErrUnexpectedEOF, 1252 }, { 1253 inputData: padInput("0\n"), 1254 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1255 wantMap: sparseDatas{}, 1256 }, { 1257 inputData: padInput("0\n")[:blockSize-1] + "#", 1258 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1259 wantMap: sparseDatas{}, 1260 }, { 1261 inputData: padInput("0"), 1262 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1263 wantErr: io.ErrUnexpectedEOF, 1264 }, { 1265 inputData: padInput("ab\n"), 1266 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1267 wantErr: ErrHeader, 1268 }, { 1269 inputData: padInput("1\n2\n3\n"), 1270 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1271 wantMap: sparseDatas{{2, 3}}, 1272 }, { 1273 inputData: padInput("1\n2\n"), 1274 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1275 wantErr: io.ErrUnexpectedEOF, 1276 }, { 1277 inputData: padInput("1\n2\n\n"), 1278 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1279 wantErr: ErrHeader, 1280 }, { 1281 inputData: string(zeroBlock[:]) + padInput("0\n"), 1282 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1283 wantErr: ErrHeader, 1284 }, { 1285 inputData: strings.Repeat("0", blockSize) + padInput("1\n5\n1\n"), 1286 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1287 wantMap: sparseDatas{{5, 1}}, 1288 }, { 1289 inputData: padInput(fmt.Sprintf("%d\n", int64(math.MaxInt64))), 1290 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1291 wantErr: ErrHeader, 1292 }, { 1293 inputData: padInput(strings.Repeat("0", 300) + "1\n" + strings.Repeat("0", 1000) + "5\n" + strings.Repeat("0", 800) + "2\n"), 1294 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1295 wantMap: sparseDatas{{5, 2}}, 1296 }, { 1297 inputData: padInput("2\n10737418240\n512\n21474836480\n512\n"), 1298 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1299 wantMap: sparseDatas{{10737418240, 512}, {21474836480, 512}}, 1300 }, { 1301 inputData: padInput("100\n" + func() string { 1302 var ss []string 1303 for i := 0; i < 100; i++ { 1304 ss = append(ss, fmt.Sprintf("%d\n%d\n", int64(i)<<30, 512)) 1305 } 1306 return strings.Join(ss, "") 1307 }()), 1308 inputHdrs: map[string]string{paxGNUSparseMajor: "1", paxGNUSparseMinor: "0"}, 1309 wantMap: func() (spd sparseDatas) { 1310 for i := 0; i < 100; i++ { 1311 spd = append(spd, sparseEntry{int64(i) << 30, 512}) 1312 } 1313 return spd 1314 }(), 1315 }} 1316 1317 for i, v := range vectors { 1318 var hdr Header 1319 hdr.PAXRecords = v.inputHdrs 1320 r := strings.NewReader(v.inputData + "#") // Add canary byte 1321 tr := Reader{curr: ®FileReader{r, int64(r.Len())}} 1322 got, err := tr.readGNUSparsePAXHeaders(&hdr) 1323 if !equalSparseEntries(got, v.wantMap) { 1324 t.Errorf("test %d, readGNUSparsePAXHeaders(): got %v, want %v", i, got, v.wantMap) 1325 } 1326 if !errors.Is(err, v.wantErr) { 1327 t.Errorf("test %d, readGNUSparsePAXHeaders() = %v, want %v", i, err, v.wantErr) 1328 } 1329 if hdr.Size != v.wantSize { 1330 t.Errorf("test %d, Header.Size = %d, want %d", i, hdr.Size, v.wantSize) 1331 } 1332 if hdr.Name != v.wantName { 1333 t.Errorf("test %d, Header.Name = %s, want %s", i, hdr.Name, v.wantName) 1334 } 1335 if v.wantErr == nil && r.Len() == 0 { 1336 t.Errorf("test %d, canary byte unexpectedly consumed", i) 1337 } 1338 } 1339 } 1340 1341 // testNonEmptyReader wraps an io.Reader and ensures that 1342 // Read is never called with an empty buffer. 1343 type testNonEmptyReader struct{ io.Reader } 1344 1345 func (r testNonEmptyReader) Read(b []byte) (int, error) { 1346 if len(b) == 0 { 1347 return 0, errors.New("unexpected empty Read call") 1348 } 1349 return r.Reader.Read(b) 1350 } 1351 1352 func TestFileReader(t *testing.T) { 1353 type ( 1354 testRead struct { // Read(cnt) == (wantStr, wantErr) 1355 cnt int 1356 wantStr string 1357 wantErr error 1358 } 1359 testWriteTo struct { // WriteTo(testFile{ops}) == (wantCnt, wantErr) 1360 ops fileOps 1361 wantCnt int64 1362 wantErr error 1363 } 1364 testRemaining struct { // LogicalRemaining() == wantLCnt, PhysicalRemaining() == wantPCnt 1365 wantLCnt int64 1366 wantPCnt int64 1367 } 1368 testFnc interface{} // testRead | testWriteTo | testRemaining 1369 ) 1370 1371 type ( 1372 makeReg struct { 1373 str string 1374 size int64 1375 } 1376 makeSparse struct { 1377 makeReg makeReg 1378 spd sparseDatas 1379 size int64 1380 } 1381 fileMaker interface{} // makeReg | makeSparse 1382 ) 1383 1384 vectors := []struct { 1385 maker fileMaker 1386 tests []testFnc 1387 }{{ 1388 maker: makeReg{"", 0}, 1389 tests: []testFnc{ 1390 testRemaining{0, 0}, 1391 testRead{0, "", io.EOF}, 1392 testRead{1, "", io.EOF}, 1393 testWriteTo{nil, 0, nil}, 1394 testRemaining{0, 0}, 1395 }, 1396 }, { 1397 maker: makeReg{"", 1}, 1398 tests: []testFnc{ 1399 testRemaining{1, 1}, 1400 testRead{5, "", io.ErrUnexpectedEOF}, 1401 testWriteTo{nil, 0, io.ErrUnexpectedEOF}, 1402 testRemaining{1, 1}, 1403 }, 1404 }, { 1405 maker: makeReg{"hello", 5}, 1406 tests: []testFnc{ 1407 testRemaining{5, 5}, 1408 testRead{5, "hello", io.EOF}, 1409 testRemaining{0, 0}, 1410 }, 1411 }, { 1412 maker: makeReg{"hello, world", 50}, 1413 tests: []testFnc{ 1414 testRemaining{50, 50}, 1415 testRead{7, "hello, ", nil}, 1416 testRemaining{43, 43}, 1417 testRead{5, "world", nil}, 1418 testRemaining{38, 38}, 1419 testWriteTo{nil, 0, io.ErrUnexpectedEOF}, 1420 testRead{1, "", io.ErrUnexpectedEOF}, 1421 testRemaining{38, 38}, 1422 }, 1423 }, { 1424 maker: makeReg{"hello, world", 5}, 1425 tests: []testFnc{ 1426 testRemaining{5, 5}, 1427 testRead{0, "", nil}, 1428 testRead{4, "hell", nil}, 1429 testRemaining{1, 1}, 1430 testWriteTo{fileOps{"o"}, 1, nil}, 1431 testRemaining{0, 0}, 1432 testWriteTo{nil, 0, nil}, 1433 testRead{0, "", io.EOF}, 1434 }, 1435 }, { 1436 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{0, 2}, {5, 3}}, 8}, 1437 tests: []testFnc{ 1438 testRemaining{8, 5}, 1439 testRead{3, "ab\x00", nil}, 1440 testRead{10, "\x00\x00cde", io.EOF}, 1441 testRemaining{0, 0}, 1442 }, 1443 }, { 1444 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{0, 2}, {5, 3}}, 8}, 1445 tests: []testFnc{ 1446 testRemaining{8, 5}, 1447 testWriteTo{fileOps{"ab", int64(3), "cde"}, 8, nil}, 1448 testRemaining{0, 0}, 1449 }, 1450 }, { 1451 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{0, 2}, {5, 3}}, 10}, 1452 tests: []testFnc{ 1453 testRemaining{10, 5}, 1454 testRead{100, "ab\x00\x00\x00cde\x00\x00", io.EOF}, 1455 testRemaining{0, 0}, 1456 }, 1457 }, { 1458 maker: makeSparse{makeReg{"abc", 5}, sparseDatas{{0, 2}, {5, 3}}, 10}, 1459 tests: []testFnc{ 1460 testRemaining{10, 5}, 1461 testRead{100, "ab\x00\x00\x00c", io.ErrUnexpectedEOF}, 1462 testRemaining{4, 2}, 1463 }, 1464 }, { 1465 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}}, 8}, 1466 tests: []testFnc{ 1467 testRemaining{8, 5}, 1468 testRead{8, "\x00abc\x00\x00de", io.EOF}, 1469 testRemaining{0, 0}, 1470 }, 1471 }, { 1472 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 0}, {6, 0}, {6, 2}}, 8}, 1473 tests: []testFnc{ 1474 testRemaining{8, 5}, 1475 testRead{8, "\x00abc\x00\x00de", io.EOF}, 1476 testRemaining{0, 0}, 1477 }, 1478 }, { 1479 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 0}, {6, 0}, {6, 2}}, 8}, 1480 tests: []testFnc{ 1481 testRemaining{8, 5}, 1482 testWriteTo{fileOps{int64(1), "abc", int64(2), "de"}, 8, nil}, 1483 testRemaining{0, 0}, 1484 }, 1485 }, { 1486 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}}, 10}, 1487 tests: []testFnc{ 1488 testRead{100, "\x00abc\x00\x00de\x00\x00", io.EOF}, 1489 }, 1490 }, { 1491 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}}, 10}, 1492 tests: []testFnc{ 1493 testWriteTo{fileOps{int64(1), "abc", int64(2), "de", int64(1), "\x00"}, 10, nil}, 1494 }, 1495 }, { 1496 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}, {8, 0}, {8, 0}, {8, 0}, {8, 0}}, 10}, 1497 tests: []testFnc{ 1498 testRead{100, "\x00abc\x00\x00de\x00\x00", io.EOF}, 1499 }, 1500 }, { 1501 maker: makeSparse{makeReg{"", 0}, sparseDatas{}, 2}, 1502 tests: []testFnc{ 1503 testRead{100, "\x00\x00", io.EOF}, 1504 }, 1505 }, { 1506 maker: makeSparse{makeReg{"", 8}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1507 tests: []testFnc{ 1508 testRead{100, "\x00", io.ErrUnexpectedEOF}, 1509 }, 1510 }, { 1511 maker: makeSparse{makeReg{"ab", 2}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1512 tests: []testFnc{ 1513 testRead{100, "\x00ab", errMissData}, 1514 }, 1515 }, { 1516 maker: makeSparse{makeReg{"ab", 8}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1517 tests: []testFnc{ 1518 testRead{100, "\x00ab", io.ErrUnexpectedEOF}, 1519 }, 1520 }, { 1521 maker: makeSparse{makeReg{"abc", 3}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1522 tests: []testFnc{ 1523 testRead{100, "\x00abc\x00\x00", errMissData}, 1524 }, 1525 }, { 1526 maker: makeSparse{makeReg{"abc", 8}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1527 tests: []testFnc{ 1528 testRead{100, "\x00abc\x00\x00", io.ErrUnexpectedEOF}, 1529 }, 1530 }, { 1531 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1532 tests: []testFnc{ 1533 testRead{100, "\x00abc\x00\x00de", errMissData}, 1534 }, 1535 }, { 1536 maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1537 tests: []testFnc{ 1538 testWriteTo{fileOps{int64(1), "abc", int64(2), "de"}, 8, errMissData}, 1539 }, 1540 }, { 1541 maker: makeSparse{makeReg{"abcde", 8}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1542 tests: []testFnc{ 1543 testRead{100, "\x00abc\x00\x00de", io.ErrUnexpectedEOF}, 1544 }, 1545 }, { 1546 maker: makeSparse{makeReg{"abcdefghEXTRA", 13}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1547 tests: []testFnc{ 1548 testRemaining{15, 13}, 1549 testRead{100, "\x00abc\x00\x00defgh\x00\x00\x00\x00", errUnrefData}, 1550 testWriteTo{nil, 0, errUnrefData}, 1551 testRemaining{0, 5}, 1552 }, 1553 }, { 1554 maker: makeSparse{makeReg{"abcdefghEXTRA", 13}, sparseDatas{{1, 3}, {6, 5}}, 15}, 1555 tests: []testFnc{ 1556 testRemaining{15, 13}, 1557 testWriteTo{fileOps{int64(1), "abc", int64(2), "defgh", int64(4)}, 15, errUnrefData}, 1558 testRead{100, "", errUnrefData}, 1559 testRemaining{0, 5}, 1560 }, 1561 }} 1562 1563 for i, v := range vectors { 1564 var fr fileReader 1565 switch maker := v.maker.(type) { 1566 case makeReg: 1567 r := testNonEmptyReader{strings.NewReader(maker.str)} 1568 fr = ®FileReader{r, maker.size} 1569 case makeSparse: 1570 if !validateSparseEntries(maker.spd, maker.size) { 1571 t.Fatalf("invalid sparse map: %v", maker.spd) 1572 } 1573 sph := invertSparseEntries(maker.spd, maker.size) 1574 r := testNonEmptyReader{strings.NewReader(maker.makeReg.str)} 1575 fr = ®FileReader{r, maker.makeReg.size} 1576 fr = &sparseFileReader{fr, sph, 0} 1577 default: 1578 t.Fatalf("test %d, unknown make operation: %T", i, maker) 1579 } 1580 1581 for j, tf := range v.tests { 1582 switch tf := tf.(type) { 1583 case testRead: 1584 b := make([]byte, tf.cnt) 1585 n, err := fr.Read(b) 1586 if got := string(b[:n]); got != tf.wantStr || !errors.Is(err, tf.wantErr) { 1587 t.Errorf("test %d.%d, Read(%d):\ngot (%q, %v)\nwant (%q, %v)", i, j, tf.cnt, got, err, tf.wantStr, tf.wantErr) 1588 } 1589 case testWriteTo: 1590 f := &testFile{ops: tf.ops} 1591 got, err := fr.WriteTo(f) 1592 if errors.As(err, &testError{}) { 1593 t.Errorf("test %d.%d, WriteTo(): %v", i, j, err) 1594 } else if got != tf.wantCnt || !errors.Is(err, tf.wantErr) { 1595 t.Errorf("test %d.%d, WriteTo() = (%d, %v), want (%d, %v)", i, j, got, err, tf.wantCnt, tf.wantErr) 1596 } 1597 if len(f.ops) > 0 { 1598 t.Errorf("test %d.%d, expected %d more operations", i, j, len(f.ops)) 1599 } 1600 case testRemaining: 1601 if got := fr.LogicalRemaining(); got != tf.wantLCnt { 1602 t.Errorf("test %d.%d, LogicalRemaining() = %d, want %d", i, j, got, tf.wantLCnt) 1603 } 1604 if got := fr.PhysicalRemaining(); got != tf.wantPCnt { 1605 t.Errorf("test %d.%d, PhysicalRemaining() = %d, want %d", i, j, got, tf.wantPCnt) 1606 } 1607 default: 1608 t.Fatalf("test %d.%d, unknown test operation: %T", i, j, tf) 1609 } 1610 } 1611 } 1612 }