github.com/minio/simdjson-go@v0.4.6-0.20231116094823-04d21cddf993/find_subroutines_amd64_test.go (about) 1 //go:build !noasm && !appengine && gc 2 // +build !noasm,!appengine,gc 3 4 /* 5 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 package simdjson 21 22 import ( 23 "reflect" 24 "runtime" 25 "strings" 26 "sync" 27 "testing" 28 29 "github.com/klauspost/cpuid/v2" 30 ) 31 32 func TestFinalizeStructurals(t *testing.T) { 33 if !SupportedCPU() { 34 t.SkipNow() 35 } 36 testCases := []struct { 37 structurals uint64 38 whitespace uint64 39 quote_mask uint64 40 quote_bits uint64 41 expected_strls uint64 42 expected_pseudo uint64 43 }{ 44 {0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 45 {0x1, 0x0, 0x0, 0x0, 0x3, 0x0}, 46 {0x2, 0x0, 0x0, 0x0, 0x6, 0x0}, 47 // test to mask off anything inside quotes 48 {0x2, 0x0, 0xf, 0x0, 0x0, 0x0}, 49 // test to add the real quote bits 50 {0x8, 0x0, 0x0, 0x10, 0x28, 0x0}, 51 // whether the previous iteration ended on a whitespace 52 {0x0, 0x8000000000000000, 0x0, 0x0, 0x0, 0x1}, 53 // whether the previous iteration ended on a structural character 54 {0x8000000000000000, 0x0, 0x0, 0x0, 0x8000000000000000, 0x1}, 55 {0xf, 0xf0, 0xf00, 0xf000, 0x1000f, 0x0}, 56 } 57 58 for i, tc := range testCases { 59 prev_iter_ends_pseudo_pred := uint64(0) 60 61 structurals := finalize_structurals(tc.structurals, tc.whitespace, tc.quote_mask, tc.quote_bits, &prev_iter_ends_pseudo_pred) 62 63 if structurals != tc.expected_strls { 64 t.Errorf("TestFinalizeStructurals(%d): got: 0x%x want: 0x%x", i, structurals, tc.expected_strls) 65 } 66 67 if prev_iter_ends_pseudo_pred != tc.expected_pseudo { 68 t.Errorf("TestFinalizeStructurals(%d): got: 0x%x want: 0x%x", i, prev_iter_ends_pseudo_pred, tc.expected_pseudo) 69 } 70 } 71 } 72 73 func testFindNewlineDelimiters(t *testing.T, f func([]byte, uint64) uint64) { 74 75 want := []uint64{ 76 0b0000000000000000000000000000000000000000000000000000000000000000, 77 0b0000000000000000000000000000000000000000000000000000000000000000, 78 0b0000000000000000000000000000000000000000000000000000000000000000, 79 0b0000000000000000000000000000000000000000000000000000000000010000, 80 0b0000000000000000000000000000000000000000000000000000000000000000, 81 0b0000000000000000000000000000000000000000000000000000000000000000, 82 0b0000000000000000000000000000000000000000000000000000001000000000, 83 0b0000000000000000000000000000000000000000000000000000000000000000, 84 0b0000000000000000000000000000000000000000000000000000000000000000, 85 } 86 87 for offset := 0; offset < len(demo_ndjson)-64; offset += 64 { 88 mask := f([]byte(demo_ndjson)[offset:], 0) 89 if mask != want[offset>>6] { 90 t.Errorf("testFindNewlineDelimiters: got: %064b want: %064b", mask, want[offset>>6]) 91 } 92 } 93 } 94 95 func TestFindNewlineDelimiters(t *testing.T) { 96 if !SupportedCPU() { 97 t.SkipNow() 98 } 99 t.Run("avx2", func(t *testing.T) { 100 testFindNewlineDelimiters(t, _find_newline_delimiters) 101 }) 102 if cpuid.CPU.Has(cpuid.AVX512F) { 103 t.Run("avx512", func(t *testing.T) { 104 testFindNewlineDelimiters(t, _find_newline_delimiters_avx512) 105 }) 106 } 107 } 108 109 func testExcludeNewlineDelimitersWithinQuotes(t *testing.T, f func([]byte, uint64) uint64) { 110 if !SupportedCPU() { 111 t.SkipNow() 112 } 113 input := []byte(` "-------------------------------------" `) 114 input[10] = 0x0a // within quoted string, so should be ignored 115 input[50] = 0x0a // outside quoted string, so should be found 116 117 prev_iter_inside_quote, quote_bits, error_mask := uint64(0), uint64(0), uint64(0) 118 119 odd_ends := uint64(0) 120 quotemask := find_quote_mask_and_bits(input, odd_ends, &prev_iter_inside_quote, "e_bits, &error_mask) 121 122 mask := f(input, quotemask) 123 want := uint64(1 << 50) 124 125 if mask != want { 126 t.Errorf("testExcludeNewlineDelimitersWithinQuotes: got: %064b want: %064b", mask, want) 127 } 128 } 129 130 func TestExcludeNewlineDelimitersWithinQuotes(t *testing.T) { 131 if !SupportedCPU() { 132 t.SkipNow() 133 } 134 t.Run("avx2", func(t *testing.T) { 135 testExcludeNewlineDelimitersWithinQuotes(t, _find_newline_delimiters) 136 }) 137 if cpuid.CPU.Has(cpuid.AVX512F) { 138 t.Run("avx512", func(t *testing.T) { 139 testExcludeNewlineDelimitersWithinQuotes(t, _find_newline_delimiters_avx512) 140 }) 141 } 142 143 } 144 145 func testFindOddBackslashSequences(t *testing.T, f func([]byte, *uint64) uint64) { 146 147 testCases := []struct { 148 prev_ends_odd uint64 149 input string 150 expected uint64 151 ends_odd_backslash uint64 152 }{ 153 {0, ` `, 0x0, 0}, 154 {0, `\" `, 0x2, 0}, 155 {0, ` \" `, 0x8, 0}, 156 {0, ` \" `, 0x200, 0}, 157 {0, ` \" `, 0x10000000, 0}, 158 {0, ` \" `, 0x100000000, 0}, 159 {0, ` \"`, 0x8000000000000000, 0}, 160 {0, ` \`, 0x0, 1}, 161 {0, `\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"`, 0xaaaaaaaaaaaaaaaa, 0}, 162 {0, `"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\`, 0x5555555555555554, 1}, 163 {1, ` `, 0x1, 0}, 164 {1, `\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"`, 0xaaaaaaaaaaaaaaa8, 0}, 165 {1, `"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\`, 0x5555555555555555, 1}, 166 } 167 168 for i, tc := range testCases { 169 prev_iter_ends_odd_backslash := tc.prev_ends_odd 170 mask := f([]byte(tc.input), &prev_iter_ends_odd_backslash) 171 172 if mask != tc.expected { 173 t.Errorf("testFindOddBackslashSequences(%d): got: 0x%x want: 0x%x", i, mask, tc.expected) 174 } 175 176 if prev_iter_ends_odd_backslash != tc.ends_odd_backslash { 177 t.Errorf("testFindOddBackslashSequences(%d): got: %v want: %v", i, prev_iter_ends_odd_backslash, tc.ends_odd_backslash) 178 } 179 } 180 181 // prepend test string with longer space, making sure shift to next 256-bit word is fine 182 for i := uint(1); i <= 128; i++ { 183 test := strings.Repeat(" ", int(i-1)) + `\"` + strings.Repeat(" ", 62+64) 184 185 prev_iter_ends_odd_backslash := uint64(0) 186 mask_lo := f([]byte(test), &prev_iter_ends_odd_backslash) 187 mask_hi := f([]byte(test[64:]), &prev_iter_ends_odd_backslash) 188 189 if i < 64 { 190 if mask_lo != 1<<i || mask_hi != 0 { 191 t.Errorf("testFindOddBackslashSequences(%d): got: lo = 0x%x; hi = 0x%x want: 0x%x 0x0", i, mask_lo, mask_hi, 1<<i) 192 } 193 } else { 194 if mask_lo != 0 || mask_hi != 1<<(i-64) { 195 t.Errorf("testFindOddBackslashSequences(%d): got: lo = 0x%x; hi = 0x%x want: 0x0 0x%x", i, mask_lo, mask_hi, 1<<(i-64)) 196 } 197 } 198 } 199 } 200 201 func TestFindOddBackslashSequences(t *testing.T) { 202 if !SupportedCPU() { 203 t.SkipNow() 204 } 205 t.Run("avx2", func(t *testing.T) { 206 testFindOddBackslashSequences(t, find_odd_backslash_sequences) 207 }) 208 if cpuid.CPU.Has(cpuid.AVX512F) { 209 t.Run("avx512", func(t *testing.T) { 210 testFindOddBackslashSequences(t, find_odd_backslash_sequences_avx512) 211 }) 212 } 213 } 214 215 func testFindQuoteMaskAndBits(t *testing.T, f func([]byte, uint64, *uint64, *uint64, *uint64) uint64) { 216 217 testCases := []struct { 218 inputOE uint64 // odd_ends 219 input string 220 expected uint64 221 expectedQB uint64 // quote_bits 222 expectedPIIQ uint64 // prev_iter_inside_quote 223 expectedEM uint64 // error_mask 224 }{ 225 {0x0, ` "" `, 0x4, 0xc, 0, 0}, 226 {0x0, ` "-" `, 0xc, 0x14, 0, 0}, 227 {0x0, ` "--" `, 0x1c, 0x24, 0, 0}, 228 {0x0, ` "---" `, 0x3c, 0x44, 0, 0}, 229 {0x0, ` "-------------" `, 0xfffc, 0x10004, 0, 0}, 230 {0x0, ` "---------------------------------------" `, 0x3fffffffffc, 0x40000000004, 0, 0}, 231 {0x0, `"--------------------------------------------------------------"`, 0x7fffffffffffffff, 0x8000000000000001, 0, 0}, 232 233 // quote is not closed --> prev_iter_inside_quote should be set 234 {0x0, ` "---`, 0xf000000000000000, 0x1000000000000000, ^uint64(0), 0}, 235 {0x0, ` "", `, 0x1000000000000000, 0x3000000000000000, 0, 0}, 236 {0x0, ` "-",`, 0x3000000000000000, 0x5000000000000000, 0, 0}, 237 {0x0, ` "--"`, 0x7000000000000000, 0x9000000000000000, 0, 0}, 238 {0x0, ` "---`, 0xf000000000000000, 0x1000000000000000, ^uint64(0), 0}, 239 240 // test previous mask ending in backslash 241 {0x1, `" `, 0x0, 0x0, 0x0, 0x0}, 242 {0x1, `""" `, 0x2, 0x6, 0x0, 0x0}, 243 {0x0, `" `, 0xffffffffffffffff, 0x1, ^uint64(0), 0x0}, 244 {0x0, `""" `, 0xfffffffffffffffd, 0x7, ^uint64(0), 0x0}, 245 246 // test invalid chars (< 0x20) that are enclosed in quotes 247 {0x0, `"` + string([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}) + ` " `, 0x3ffffffff, 0x400000001, 0, 0x1fffffffe}, 248 {0x0, `"` + string([]byte{0, 32, 1, 32, 2, 32, 3, 32, 4, 32, 5, 32, 6, 32, 7, 32, 8, 32, 9, 32, 10, 32, 11, 32, 12, 32, 13, 32, 14, 32, 15, 32, 16, 32, 17, 32, 18, 32, 19, 32, 20, 32, 21, 32, 22, 32, 23, 32, 24, 32, 25, 32, 26, 32, 27, 32, 28, 32, 29, 32, 31}) + ` "`, 0x7fffffffffffffff, 0x8000000000000001, 0, 0x2aaaaaaaaaaaaaaa}, 249 {0x0, `" ` + string([]byte{0, 32, 1, 32, 2, 32, 3, 32, 4, 32, 5, 32, 6, 32, 7, 32, 8, 32, 9, 32, 10, 32, 11, 32, 12, 32, 13, 32, 14, 32, 15, 32, 16, 32, 17, 32, 18, 32, 19, 32, 20, 32, 21, 32, 22, 32, 23, 32, 24, 32, 25, 32, 26, 32, 27, 32, 28, 32, 29, 32, 31}) + `"`, 0x7fffffffffffffff, 0x8000000000000001, 0, 0x5555555555555554}, 250 } 251 252 for i, tc := range testCases { 253 254 prev_iter_inside_quote, quote_bits, error_mask := uint64(0), uint64(0), uint64(0) 255 256 mask := f([]byte(tc.input), tc.inputOE, &prev_iter_inside_quote, "e_bits, &error_mask) 257 258 if mask != tc.expected { 259 t.Errorf("testFindQuoteMaskAndBits(%d): got: 0x%x want: 0x%x", i, mask, tc.expected) 260 } 261 262 if quote_bits != tc.expectedQB { 263 t.Errorf("testFindQuoteMaskAndBits(%d): got quote_bits: 0x%x want: 0x%x", i, quote_bits, tc.expectedQB) 264 } 265 266 if prev_iter_inside_quote != tc.expectedPIIQ { 267 t.Errorf("testFindQuoteMaskAndBits(%d): got prev_iter_inside_quote: 0x%x want: 0x%x", i, prev_iter_inside_quote, tc.expectedPIIQ) 268 } 269 270 if error_mask != tc.expectedEM { 271 t.Errorf("testFindQuoteMaskAndBits(%d): got error_mask: 0x%x want: 0x%x", i, error_mask, tc.expectedEM) 272 } 273 } 274 275 testCasesPIIQ := []struct { 276 inputPIIQ uint64 277 input string 278 expectedPIIQ uint64 279 }{ 280 // prev_iter_inside_quote state remains unchanged 281 {uint64(0), `----------------------------------------------------------------`, uint64(0)}, 282 {^uint64(0), `----------------------------------------------------------------`, ^uint64(0)}, 283 284 // prev_iter_inside_quote state remains flips 285 {uint64(0), `---------------------------"------------------------------------`, ^uint64(0)}, 286 {^uint64(0), `---------------------------"------------------------------------`, uint64(0)}, 287 288 // prev_iter_inside_quote state remains flips twice (thus unchanged) 289 {uint64(0), `----------------"------------------------"----------------------`, uint64(0)}, 290 {^uint64(0), `----------------"------------------------"----------------------`, ^uint64(0)}, 291 } 292 293 for i, tc := range testCasesPIIQ { 294 295 prev_iter_inside_quote, quote_bits, error_mask := tc.inputPIIQ, uint64(0), uint64(0) 296 297 f([]byte(tc.input), 0, &prev_iter_inside_quote, "e_bits, &error_mask) 298 299 if prev_iter_inside_quote != tc.expectedPIIQ { 300 t.Errorf("testFindQuoteMaskAndBits(%d): got prev_iter_inside_quote: 0x%x want: 0x%x", i, prev_iter_inside_quote, tc.expectedPIIQ) 301 } 302 } 303 } 304 305 func TestFindQuoteMaskAndBits(t *testing.T) { 306 if !SupportedCPU() { 307 t.SkipNow() 308 } 309 t.Run("avx2", func(t *testing.T) { 310 testFindQuoteMaskAndBits(t, find_quote_mask_and_bits) 311 }) 312 if cpuid.CPU.Has(cpuid.AVX512F) { 313 t.Run("avx512", func(t *testing.T) { 314 testFindQuoteMaskAndBits(t, find_quote_mask_and_bits_avx512) 315 }) 316 } 317 } 318 319 func testFindStructuralBits(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, uint64, *uint64) uint64) { 320 321 testCases := []struct { 322 input string 323 }{ 324 {`{"Image":{"Width":800,"Height":600,"Title":"View from 15th Floor`}, 325 {`","Thumbnail":{"Url":"http://www.example.com/image/481989943","H`}, 326 {`eight":125,"Width":100},"Animated":false,"IDs":[116,943,234,3879`}, 327 } 328 329 prev_iter_ends_odd_backslash := uint64(0) 330 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 331 prev_iter_ends_pseudo_pred := uint64(1) 332 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 333 structurals := uint64(0) 334 335 // Declare same variables for 'multiple_calls' version 336 prev_iter_ends_odd_backslash_MC := uint64(0) 337 prev_iter_inside_quote_MC := uint64(0) // either all zeros or all ones 338 prev_iter_ends_pseudo_pred_MC := uint64(1) 339 error_mask_MC := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 340 structurals_MC := uint64(0) 341 342 for i, tc := range testCases { 343 344 // Call assembly routines as a single method 345 structurals := f([]byte(tc.input), &prev_iter_ends_odd_backslash, 346 &prev_iter_inside_quote, &error_mask, 347 structurals, 348 &prev_iter_ends_pseudo_pred) 349 350 // Call assembly routines individually 351 structurals_MC := find_structural_bits_multiple_calls([]byte(tc.input), &prev_iter_ends_odd_backslash_MC, 352 &prev_iter_inside_quote_MC, &error_mask_MC, 353 structurals_MC, 354 &prev_iter_ends_pseudo_pred_MC) 355 356 // And compare the results 357 if structurals != structurals_MC { 358 t.Errorf("TestFindStructuralBits(%d): got: 0x%x want: 0x%x", i, structurals, structurals_MC) 359 } 360 } 361 } 362 363 func TestFindStructuralBits(t *testing.T) { 364 if !SupportedCPU() { 365 t.SkipNow() 366 } 367 t.Run("avx2", func(t *testing.T) { 368 testFindStructuralBits(t, find_structural_bits) 369 }) 370 if cpuid.CPU.Has(cpuid.AVX512F) { 371 t.Run("avx512", func(t *testing.T) { 372 testFindStructuralBits(t, find_structural_bits_avx512) 373 }) 374 } 375 } 376 377 func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) { 378 379 // Test whitespace padding (for partial load of last 64 bytes) with 380 // string full of structural characters 381 msg := `::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::` 382 383 for l := len(msg); l >= 0; l-- { 384 385 prev_iter_ends_odd_backslash := uint64(0) 386 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 387 prev_iter_ends_pseudo_pred := uint64(1) 388 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 389 carried := ^uint64(0) 390 position := ^uint64(0) 391 392 index := indexChan{} 393 index.indexes = &[indexSize]uint32{} 394 395 processed := find_structural_bits_in_slice([]byte(msg[:l]), &prev_iter_ends_odd_backslash, 396 &prev_iter_inside_quote, &error_mask, 397 &prev_iter_ends_pseudo_pred, 398 index.indexes, &index.length, &carried, &position, 0) 399 400 if processed != uint64(l) { 401 t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, processed, l) 402 } 403 if index.length != l { 404 t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, index.length, l) 405 } 406 407 // Compute offset of last (structural) character and verify it points to the end of the message 408 lastChar := uint64(0) 409 for i := 0; i < index.length; i++ { 410 lastChar += uint64(index.indexes[i]) 411 } 412 if l > 0 { 413 if lastChar != uint64(l-1) { 414 t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, lastChar, uint64(l-1)) 415 } 416 } else { 417 if lastChar != uint64(l-1)-carried { 418 t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, lastChar, uint64(l-1)-carried) 419 } 420 } 421 } 422 } 423 424 func TestFindStructuralBitsWhitespacePadding(t *testing.T) { 425 if !SupportedCPU() { 426 t.SkipNow() 427 } 428 t.Run("avx2", func(t *testing.T) { 429 testFindStructuralBitsWhitespacePadding(t, find_structural_bits_in_slice) 430 }) 431 if cpuid.CPU.Has(cpuid.AVX512F) { 432 t.Run("avx512", func(t *testing.T) { 433 testFindStructuralBitsWhitespacePadding(t, find_structural_bits_in_slice_avx512) 434 }) 435 } 436 } 437 438 func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) { 439 msg := loadCompressed(t, "twitter") 440 441 prev_iter_ends_odd_backslash := uint64(0) 442 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 443 prev_iter_ends_pseudo_pred := uint64(1) 444 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 445 carried := ^uint64(0) 446 position := ^uint64(0) 447 448 indexes := make([]uint32, 0) 449 450 for processed := uint64(0); processed < uint64(len(msg)); { 451 index := indexChan{} 452 index.indexes = &[indexSize]uint32{} 453 454 processed += f(msg[processed:], &prev_iter_ends_odd_backslash, 455 &prev_iter_inside_quote, &error_mask, 456 &prev_iter_ends_pseudo_pred, 457 index.indexes, &index.length, &carried, &position, 0) 458 459 indexes = append(indexes, (*index.indexes)[:index.length]...) 460 } 461 462 // Last 5 expected structural (in reverse order) 463 const expectedStructuralsReversed = `}}":"` 464 const expectedLength = 55263 465 466 if len(indexes) != expectedLength { 467 t.Errorf("TestFindStructuralBitsLoop: got: %d want: %d", len(indexes), expectedLength) 468 } 469 470 pos, j := len(msg)-1, 0 471 for i := len(indexes) - 1; i >= len(indexes)-len(expectedStructuralsReversed); i-- { 472 473 if msg[pos] != expectedStructuralsReversed[j] { 474 t.Errorf("TestFindStructuralBitsLoop: got: %c want: %c", msg[pos], expectedStructuralsReversed[j]) 475 } 476 477 pos -= int(indexes[i]) 478 j++ 479 } 480 } 481 482 func TestFindStructuralBitsLoop(t *testing.T) { 483 if !SupportedCPU() { 484 t.SkipNow() 485 } 486 t.Run("avx2", func(t *testing.T) { 487 testFindStructuralBitsLoop(t, find_structural_bits_in_slice) 488 }) 489 if cpuid.CPU.Has(cpuid.AVX512F) { 490 t.Run("avx512", func(t *testing.T) { 491 testFindStructuralBitsLoop(t, find_structural_bits_in_slice_avx512) 492 }) 493 } 494 } 495 496 func benchmarkFindStructuralBits(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, uint64, *uint64) uint64) { 497 498 const msg = " " 499 b.SetBytes(int64(len(msg))) 500 b.ReportAllocs() 501 b.ResetTimer() 502 503 prev_iter_ends_odd_backslash := uint64(0) 504 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 505 prev_iter_ends_pseudo_pred := uint64(1) 506 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 507 structurals := uint64(0) 508 509 for i := 0; i < b.N; i++ { 510 f([]byte(msg), &prev_iter_ends_odd_backslash, 511 &prev_iter_inside_quote, &error_mask, 512 structurals, 513 &prev_iter_ends_pseudo_pred) 514 } 515 } 516 517 func BenchmarkFindStructuralBits(b *testing.B) { 518 if !SupportedCPU() { 519 b.SkipNow() 520 } 521 b.Run("avx2", func(b *testing.B) { 522 benchmarkFindStructuralBits(b, find_structural_bits) 523 }) 524 if cpuid.CPU.Has(cpuid.AVX512F) { 525 b.Run("avx512", func(b *testing.B) { 526 benchmarkFindStructuralBits(b, find_structural_bits_avx512) 527 }) 528 } 529 } 530 531 func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) { 532 533 msg := loadCompressed(b, "twitter") 534 535 prev_iter_ends_odd_backslash := uint64(0) 536 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 537 prev_iter_ends_pseudo_pred := uint64(1) 538 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 539 carried := ^uint64(0) 540 position := ^uint64(0) 541 542 b.SetBytes(int64(len(msg))) 543 b.ReportAllocs() 544 b.ResetTimer() 545 546 for i := 0; i < b.N; i++ { 547 548 for processed := uint64(0); processed < uint64(len(msg)); { 549 index := indexChan{} 550 index.indexes = &[indexSize]uint32{} 551 552 processed += f(msg[processed:], &prev_iter_ends_odd_backslash, 553 &prev_iter_inside_quote, &error_mask, 554 &prev_iter_ends_pseudo_pred, 555 index.indexes, &index.length, &carried, &position, 0) 556 } 557 } 558 } 559 560 func BenchmarkFindStructuralBitsLoop(b *testing.B) { 561 if !SupportedCPU() { 562 b.SkipNow() 563 } 564 b.Run("avx2", func(b *testing.B) { 565 benchmarkFindStructuralBitsLoop(b, find_structural_bits_in_slice) 566 }) 567 if cpuid.CPU.Has(cpuid.AVX512F) { 568 b.Run("avx512", func(b *testing.B) { 569 benchmarkFindStructuralBitsLoop(b, find_structural_bits_in_slice_avx512) 570 }) 571 } 572 } 573 574 func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) { 575 576 msg := loadCompressed(b, "twitter") 577 cpus := runtime.NumCPU() 578 579 b.SetBytes(int64(len(msg) * cpus)) 580 b.ResetTimer() 581 582 for i := 0; i < b.N; i++ { 583 var wg sync.WaitGroup 584 wg.Add(cpus) 585 for cpu := 0; cpu < cpus; cpu++ { 586 go func() { 587 prev_iter_ends_odd_backslash := uint64(0) 588 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 589 prev_iter_ends_pseudo_pred := uint64(1) 590 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 591 carried := ^uint64(0) 592 position := ^uint64(0) 593 594 for processed := uint64(0); processed < uint64(len(msg)); { 595 index := indexChan{} 596 index.indexes = &[indexSize]uint32{} 597 598 processed += f(msg[processed:], &prev_iter_ends_odd_backslash, 599 &prev_iter_inside_quote, &error_mask, 600 &prev_iter_ends_pseudo_pred, 601 index.indexes, &index.length, &carried, &position, 0) 602 } 603 defer wg.Done() 604 }() 605 } 606 wg.Wait() 607 } 608 } 609 610 func BenchmarkFindStructuralBitsParallelLoop(b *testing.B) { 611 if !SupportedCPU() { 612 b.SkipNow() 613 } 614 615 b.Run("avx2", func(b *testing.B) { 616 benchmarkFindStructuralBitsParallelLoop(b, find_structural_bits_in_slice) 617 }) 618 if cpuid.CPU.Has(cpuid.AVX512F) { 619 b.Run("avx512", func(b *testing.B) { 620 benchmarkFindStructuralBitsParallelLoop(b, find_structural_bits_in_slice_avx512) 621 }) 622 } 623 } 624 625 // find_structural_bits version that calls the individual assembly routines individually 626 func find_structural_bits_multiple_calls(buf []byte, prev_iter_ends_odd_backslash *uint64, 627 prev_iter_inside_quote, error_mask *uint64, 628 structurals uint64, 629 prev_iter_ends_pseudo_pred *uint64) uint64 { 630 quote_bits := uint64(0) 631 whitespace_mask := uint64(0) 632 633 odd_ends := find_odd_backslash_sequences(buf, prev_iter_ends_odd_backslash) 634 635 // detect insides of quote pairs ("quote_mask") and also our quote_bits themselves 636 quote_mask := find_quote_mask_and_bits(buf, odd_ends, prev_iter_inside_quote, "e_bits, error_mask) 637 638 find_whitespace_and_structurals(buf, &whitespace_mask, &structurals) 639 640 // fixup structurals to reflect quotes and add pseudo-structural characters 641 return finalize_structurals(structurals, whitespace_mask, quote_mask, quote_bits, prev_iter_ends_pseudo_pred) 642 } 643 644 func testFindWhitespaceAndStructurals(t *testing.T, f func([]byte, *uint64, *uint64)) { 645 646 testCases := []struct { 647 input string 648 expected_ws uint64 649 expected_strls uint64 650 }{ 651 {`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x0, 0x0}, 652 {` aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x1, 0x0}, 653 {`:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x0, 0x1}, 654 {` :aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x1, 0x2}, 655 {`: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x2, 0x1}, 656 {`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa `, 0x8000000000000000, 0x0}, 657 {`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:`, 0x0, 0x8000000000000000}, 658 {`a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a `, 0xaaaaaaaaaaaaaaaa, 0x0}, 659 {` a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a`, 0x5555555555555555, 0x0}, 660 {`a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:`, 0x0, 0xaaaaaaaaaaaaaaaa}, 661 {`:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a`, 0x0, 0x5555555555555555}, 662 {` `, 0xffffffffffffffff, 0x0}, 663 {`{ `, 0xfffffffffffffffe, 0x1}, 664 {`} `, 0xfffffffffffffffe, 0x1}, 665 {`" `, 0xfffffffffffffffe, 0x0}, 666 {`::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::`, 0x0, 0xffffffffffffffff}, 667 {`{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{`, 0x0, 0xffffffffffffffff}, 668 {`}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}`, 0x0, 0xffffffffffffffff}, 669 {` : `, 0xfffffffffffffffb, 0x4}, 670 {` : `, 0xffffffffffffffef, 0x10}, 671 {` : : : : : :`, 0x7fffefffbff7efbf, 0x8000100040081040}, 672 {demo_json, 0x421000000000000, 0x40440220301}, 673 } 674 675 for i, tc := range testCases { 676 whitespace := uint64(0) 677 structurals := uint64(0) 678 679 f([]byte(tc.input), &whitespace, &structurals) 680 681 if whitespace != tc.expected_ws { 682 t.Errorf("testFindWhitespaceAndStructurals(%d): got: 0x%x want: 0x%x", i, whitespace, tc.expected_ws) 683 } 684 685 if structurals != tc.expected_strls { 686 t.Errorf("testFindWhitespaceAndStructurals(%d): got: 0x%x want: 0x%x", i, structurals, tc.expected_strls) 687 } 688 } 689 } 690 691 func TestFindWhitespaceAndStructurals(t *testing.T) { 692 if !SupportedCPU() { 693 t.SkipNow() 694 } 695 696 t.Run("avx2", func(t *testing.T) { 697 testFindWhitespaceAndStructurals(t, find_whitespace_and_structurals) 698 }) 699 if cpuid.CPU.Has(cpuid.AVX512F) { 700 t.Run("avx512", func(t *testing.T) { 701 testFindWhitespaceAndStructurals(t, find_whitespace_and_structurals_avx512) 702 }) 703 } 704 } 705 706 func TestFlattenBitsIncremental(t *testing.T) { 707 if !SupportedCPU() { 708 t.SkipNow() 709 } 710 711 testCases := []struct { 712 masks []uint64 713 expected []uint32 714 }{ 715 // Single mask 716 {[]uint64{0x11}, []uint32{0x1, 0x4}}, 717 {[]uint64{0x100100100100}, []uint32{0x9, 0xc, 0xc, 0xc}}, 718 {[]uint64{0x100100100300}, []uint32{0x9, 0x1, 0xb, 0xc, 0xc}}, 719 {[]uint64{0x8101010101010101}, []uint32{0x1, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x7}}, 720 {[]uint64{0x4000000000000000}, []uint32{0x3f}}, 721 {[]uint64{0x8000000000000000}, []uint32{0x40}}, 722 {[]uint64{0xf000000000000000}, []uint32{0x3d, 0x1, 0x1, 0x1}}, 723 {[]uint64{0xffffffffffffffff}, []uint32{ 724 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 725 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 726 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 727 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 728 }}, 729 //// 730 //// Multiple masks 731 {[]uint64{0x1, 0x1000}, []uint32{0x1, 0x4c}}, 732 {[]uint64{0x1, 0x4000000000000000}, []uint32{0x1, 0x7e}}, 733 {[]uint64{0x1, 0x8000000000000000}, []uint32{0x1, 0x7f}}, 734 {[]uint64{0x1, 0x0, 0x8000000000000000}, []uint32{0x1, 0xbf}}, 735 {[]uint64{0x1, 0x0, 0x0, 0x8000000000000000}, []uint32{0x1, 0xff}}, 736 {[]uint64{0x100100100100100, 0x100100100100100}, []uint32{0x9, 0xc, 0xc, 0xc, 0xc, 0x10, 0xc, 0xc, 0xc, 0xc}}, 737 {[]uint64{0xffffffffffffffff, 0xffffffffffffffff}, []uint32{ 738 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 739 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 740 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 741 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 742 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 743 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 744 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 745 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 746 }}, 747 } 748 749 for i, tc := range testCases { 750 751 index := indexChan{} 752 index.indexes = &[indexSize]uint32{} 753 carried := 0 754 position := ^uint64(0) 755 756 for _, mask := range tc.masks { 757 flatten_bits_incremental(index.indexes, &index.length, mask, &carried, &position) 758 } 759 760 if index.length != len(tc.expected) { 761 t.Errorf("TestFlattenBitsIncremental(%d): got: %d want: %d", i, index.length, len(tc.expected)) 762 } 763 764 compare := make([]uint32, 0, 1024) 765 for idx := 0; idx < index.length; idx++ { 766 compare = append(compare, index.indexes[idx]) 767 } 768 769 if !reflect.DeepEqual(compare, tc.expected) { 770 t.Errorf("TestFlattenBitsIncremental(%d): got: %v want: %v", i, compare, tc.expected) 771 } 772 } 773 } 774 775 func BenchmarkFlattenBits(b *testing.B) { 776 if !SupportedCPU() { 777 b.SkipNow() 778 } 779 780 msg := loadCompressed(b, "twitter") 781 782 prev_iter_ends_odd_backslash := uint64(0) 783 prev_iter_inside_quote := uint64(0) // either all zeros or all ones 784 prev_iter_ends_pseudo_pred := uint64(1) 785 error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20) 786 structurals := uint64(0) 787 788 structuralsArray := make([]uint64, 0, len(msg)>>6) 789 790 // Collect all structurals into array 791 for i := 0; i < len(msg)-64; i += 64 { 792 find_structural_bits([]byte(msg)[i:], &prev_iter_ends_odd_backslash, 793 &prev_iter_inside_quote, &error_mask, 794 structurals, 795 &prev_iter_ends_pseudo_pred) 796 797 structuralsArray = append(structuralsArray, structurals) 798 } 799 800 b.SetBytes(int64(len(structuralsArray) * 8)) 801 b.ReportAllocs() 802 b.ResetTimer() 803 804 index := indexChan{} 805 index.indexes = &[indexSize]uint32{} 806 carried := 0 807 position := ^uint64(0) 808 809 for i := 0; i < b.N; i++ { 810 for _, structurals := range structuralsArray { 811 index.length = 0 // reset length to prevent overflow 812 flatten_bits_incremental(index.indexes, &index.length, structurals, &carried, &position) 813 } 814 } 815 }