github.com/dgraph-io/simdjson-go@v0.3.0/find_subroutines_amd64_test.go (about)

     1  //+build !noasm
     2  //+build !appengine
     3  //+build gc
     4  
     5  /*
     6   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     7   *
     8   * Licensed under the Apache License, Version 2.0 (the "License");
     9   * you may not use this file except in compliance with the License.
    10   * You may obtain a copy of the License at
    11   *
    12   *     http://www.apache.org/licenses/LICENSE-2.0
    13   *
    14   * Unless required by applicable law or agreed to in writing, software
    15   * distributed under the License is distributed on an "AS IS" BASIS,
    16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    17   * See the License for the specific language governing permissions and
    18   * limitations under the License.
    19   */
    20  
    21  package simdjson
    22  
    23  import (
    24  	"reflect"
    25  	"runtime"
    26  	"strings"
    27  	"sync"
    28  	"testing"
    29  
    30  	"github.com/klauspost/cpuid/v2"
    31  )
    32  
    33  func TestFinalizeStructurals(t *testing.T) {
    34  
    35  	testCases := []struct {
    36  		structurals     uint64
    37  		whitespace      uint64
    38  		quote_mask      uint64
    39  		quote_bits      uint64
    40  		expected_strls  uint64
    41  		expected_pseudo uint64
    42  	}{
    43  		{0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
    44  		{0x1, 0x0, 0x0, 0x0, 0x3, 0x0},
    45  		{0x2, 0x0, 0x0, 0x0, 0x6, 0x0},
    46  		// test to mask off anything inside quotes
    47  		{0x2, 0x0, 0xf, 0x0, 0x0, 0x0},
    48  		// test to add the real quote bits
    49  		{0x8, 0x0, 0x0, 0x10, 0x28, 0x0},
    50  		// whether the previous iteration ended on a whitespace
    51  		{0x0, 0x8000000000000000, 0x0, 0x0, 0x0, 0x1},
    52  		// whether the previous iteration ended on a structural character
    53  		{0x8000000000000000, 0x0, 0x0, 0x0, 0x8000000000000000, 0x1},
    54  		{0xf, 0xf0, 0xf00, 0xf000, 0x1000f, 0x0},
    55  	}
    56  
    57  	for i, tc := range testCases {
    58  		prev_iter_ends_pseudo_pred := uint64(0)
    59  
    60  		structurals := finalize_structurals(tc.structurals, tc.whitespace, tc.quote_mask, tc.quote_bits, &prev_iter_ends_pseudo_pred)
    61  
    62  		if structurals != tc.expected_strls {
    63  			t.Errorf("TestFinalizeStructurals(%d): got: 0x%x want: 0x%x", i, structurals, tc.expected_strls)
    64  		}
    65  
    66  		if prev_iter_ends_pseudo_pred != tc.expected_pseudo {
    67  			t.Errorf("TestFinalizeStructurals(%d): got: 0x%x want: 0x%x", i, prev_iter_ends_pseudo_pred, tc.expected_pseudo)
    68  		}
    69  	}
    70  }
    71  
    72  func testFindNewlineDelimiters(t *testing.T, f func([]byte, uint64) uint64) {
    73  
    74  	want := []uint64{
    75  		0b0000000000000000000000000000000000000000000000000000000000000000,
    76  		0b0000000000000000000000000000000000000000000000000000000000000000,
    77  		0b0000000000000000000000000000000000000000000000000000000000000000,
    78  		0b0000000000000000000000000000000000000000000000000000000000010000,
    79  		0b0000000000000000000000000000000000000000000000000000000000000000,
    80  		0b0000000000000000000000000000000000000000000000000000000000000000,
    81  		0b0000000000000000000000000000000000000000000000000000001000000000,
    82  		0b0000000000000000000000000000000000000000000000000000000000000000,
    83  		0b0000000000000000000000000000000000000000000000000000000000000000,
    84  	}
    85  
    86  	for offset := 0; offset < len(demo_ndjson)-64; offset += 64 {
    87  		mask := f([]byte(demo_ndjson)[offset:], 0)
    88  		if mask != want[offset>>6] {
    89  			t.Errorf("testFindNewlineDelimiters: got: %064b want: %064b", mask, want[offset>>6])
    90  		}
    91  	}
    92  }
    93  
    94  func TestFindNewlineDelimiters(t *testing.T) {
    95  	t.Run("avx2", func(t *testing.T) {
    96  		testFindNewlineDelimiters(t, _find_newline_delimiters)
    97  	})
    98  	if cpuid.CPU.Has(cpuid.AVX512F) {
    99  		t.Run("avx512", func(t *testing.T) {
   100  			testFindNewlineDelimiters(t, _find_newline_delimiters_avx512)
   101  		})
   102  	}
   103  }
   104  
   105  func testExcludeNewlineDelimitersWithinQuotes(t *testing.T, f func([]byte, uint64) uint64) {
   106  
   107  	input := []byte(`  "-------------------------------------"                       `)
   108  	input[10] = 0x0a // within quoted string, so should be ignored
   109  	input[50] = 0x0a // outside quoted string, so should be found
   110  
   111  	prev_iter_inside_quote, quote_bits, error_mask := uint64(0), uint64(0), uint64(0)
   112  
   113  	odd_ends := uint64(0)
   114  	quotemask := find_quote_mask_and_bits(input, odd_ends, &prev_iter_inside_quote, &quote_bits, &error_mask)
   115  
   116  	mask := f(input, quotemask)
   117  	want := uint64(1 << 50)
   118  
   119  	if mask != want {
   120  		t.Errorf("testExcludeNewlineDelimitersWithinQuotes: got: %064b want: %064b", mask, want)
   121  	}
   122  }
   123  
   124  func TestExcludeNewlineDelimitersWithinQuotes(t *testing.T) {
   125  	t.Run("avx2", func(t *testing.T) {
   126  		testExcludeNewlineDelimitersWithinQuotes(t, _find_newline_delimiters)
   127  	})
   128  	if cpuid.CPU.Has(cpuid.AVX512F) {
   129  		t.Run("avx512", func(t *testing.T) {
   130  			testExcludeNewlineDelimitersWithinQuotes(t, _find_newline_delimiters_avx512)
   131  		})
   132  	}
   133  
   134  }
   135  
   136  func testFindOddBackslashSequences(t *testing.T, f func([]byte, *uint64) uint64) {
   137  
   138  	testCases := []struct {
   139  		prev_ends_odd      uint64
   140  		input              string
   141  		expected           uint64
   142  		ends_odd_backslash uint64
   143  	}{
   144  		{0, `                                                                `, 0x0, 0},
   145  		{0, `\"                                                              `, 0x2, 0},
   146  		{0, `  \"                                                            `, 0x8, 0},
   147  		{0, `        \"                                                      `, 0x200, 0},
   148  		{0, `                           \"                                   `, 0x10000000, 0},
   149  		{0, `                               \"                               `, 0x100000000, 0},
   150  		{0, `                                                              \"`, 0x8000000000000000, 0},
   151  		{0, `                                                               \`, 0x0, 1},
   152  		{0, `\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"`, 0xaaaaaaaaaaaaaaaa, 0},
   153  		{0, `"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\`, 0x5555555555555554, 1},
   154  		{1, `                                                                `, 0x1, 0},
   155  		{1, `\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"`, 0xaaaaaaaaaaaaaaa8, 0},
   156  		{1, `"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\`, 0x5555555555555555, 1},
   157  	}
   158  
   159  	for i, tc := range testCases {
   160  		prev_iter_ends_odd_backslash := tc.prev_ends_odd
   161  		mask := f([]byte(tc.input), &prev_iter_ends_odd_backslash)
   162  
   163  		if mask != tc.expected {
   164  			t.Errorf("testFindOddBackslashSequences(%d): got: 0x%x want: 0x%x", i, mask, tc.expected)
   165  		}
   166  
   167  		if prev_iter_ends_odd_backslash != tc.ends_odd_backslash {
   168  			t.Errorf("testFindOddBackslashSequences(%d): got: %v want: %v", i, prev_iter_ends_odd_backslash, tc.ends_odd_backslash)
   169  		}
   170  	}
   171  
   172  	// prepend test string with longer space, making sure shift to next 256-bit word is fine
   173  	for i := uint(1); i <= 128; i++ {
   174  		test := strings.Repeat(" ", int(i-1)) + `\"` + strings.Repeat(" ", 62+64)
   175  
   176  		prev_iter_ends_odd_backslash := uint64(0)
   177  		mask_lo := f([]byte(test), &prev_iter_ends_odd_backslash)
   178  		mask_hi := f([]byte(test[64:]), &prev_iter_ends_odd_backslash)
   179  
   180  		if i < 64 {
   181  			if mask_lo != 1<<i || mask_hi != 0 {
   182  				t.Errorf("testFindOddBackslashSequences(%d): got: lo = 0x%x; hi = 0x%x  want: 0x%x 0x0", i, mask_lo, mask_hi, 1<<i)
   183  			}
   184  		} else {
   185  			if mask_lo != 0 || mask_hi != 1<<(i-64) {
   186  				t.Errorf("testFindOddBackslashSequences(%d): got: lo = 0x%x; hi = 0x%x  want:  0x0 0x%x", i, mask_lo, mask_hi, 1<<(i-64))
   187  			}
   188  		}
   189  	}
   190  }
   191  
   192  func TestFindOddBackslashSequences(t *testing.T) {
   193  	t.Run("avx2", func(t *testing.T) {
   194  		testFindOddBackslashSequences(t, find_odd_backslash_sequences)
   195  	})
   196  	if cpuid.CPU.Has(cpuid.AVX512F) {
   197  		t.Run("avx512", func(t *testing.T) {
   198  			testFindOddBackslashSequences(t, find_odd_backslash_sequences_avx512)
   199  		})
   200  	}
   201  }
   202  
   203  func testFindQuoteMaskAndBits(t *testing.T, f func([]byte, uint64, *uint64, *uint64, *uint64) uint64) {
   204  
   205  	testCases := []struct {
   206  		inputOE      uint64 // odd_ends
   207  		input        string
   208  		expected     uint64
   209  		expectedQB   uint64 // quote_bits
   210  		expectedPIIQ uint64 // prev_iter_inside_quote
   211  		expectedEM   uint64 // error_mask
   212  	}{
   213  		{0x0, `  ""                                                            `, 0x4, 0xc, 0, 0},
   214  		{0x0, `  "-"                                                           `, 0xc, 0x14, 0, 0},
   215  		{0x0, `  "--"                                                          `, 0x1c, 0x24, 0, 0},
   216  		{0x0, `  "---"                                                         `, 0x3c, 0x44, 0, 0},
   217  		{0x0, `  "-------------"                                               `, 0xfffc, 0x10004, 0, 0},
   218  		{0x0, `  "---------------------------------------"                     `, 0x3fffffffffc, 0x40000000004, 0, 0},
   219  		{0x0, `"--------------------------------------------------------------"`, 0x7fffffffffffffff, 0x8000000000000001, 0, 0},
   220  
   221  		// quote is not closed --> prev_iter_inside_quote should be set
   222  		{0x0, `                                                            "---`, 0xf000000000000000, 0x1000000000000000, ^uint64(0), 0},
   223  		{0x0, `                                                            "", `, 0x1000000000000000, 0x3000000000000000, 0, 0},
   224  		{0x0, `                                                            "-",`, 0x3000000000000000, 0x5000000000000000, 0, 0},
   225  		{0x0, `                                                            "--"`, 0x7000000000000000, 0x9000000000000000, 0, 0},
   226  		{0x0, `                                                            "---`, 0xf000000000000000, 0x1000000000000000, ^uint64(0), 0},
   227  
   228  		// test previous mask ending in backslash
   229  		{0x1, `"                                                               `, 0x0, 0x0, 0x0, 0x0},
   230  		{0x1, `"""                                                             `, 0x2, 0x6, 0x0, 0x0},
   231  		{0x0, `"                                                               `, 0xffffffffffffffff, 0x1, ^uint64(0), 0x0},
   232  		{0x0, `"""                                                             `, 0xfffffffffffffffd, 0x7, ^uint64(0), 0x0},
   233  
   234  		// test invalid chars (< 0x20) that are enclosed in quotes
   235  		{0x0, `"` + string([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}) + ` "                             `, 0x3ffffffff, 0x400000001, 0, 0x1fffffffe},
   236  		{0x0, `"` + string([]byte{0, 32, 1, 32, 2, 32, 3, 32, 4, 32, 5, 32, 6, 32, 7, 32, 8, 32, 9, 32, 10, 32, 11, 32, 12, 32, 13, 32, 14, 32, 15, 32, 16, 32, 17, 32, 18, 32, 19, 32, 20, 32, 21, 32, 22, 32, 23, 32, 24, 32, 25, 32, 26, 32, 27, 32, 28, 32, 29, 32, 31}) + ` "`, 0x7fffffffffffffff, 0x8000000000000001, 0, 0x2aaaaaaaaaaaaaaa},
   237  		{0x0, `" ` + string([]byte{0, 32, 1, 32, 2, 32, 3, 32, 4, 32, 5, 32, 6, 32, 7, 32, 8, 32, 9, 32, 10, 32, 11, 32, 12, 32, 13, 32, 14, 32, 15, 32, 16, 32, 17, 32, 18, 32, 19, 32, 20, 32, 21, 32, 22, 32, 23, 32, 24, 32, 25, 32, 26, 32, 27, 32, 28, 32, 29, 32, 31}) + `"`, 0x7fffffffffffffff, 0x8000000000000001, 0, 0x5555555555555554},
   238  	}
   239  
   240  	for i, tc := range testCases {
   241  
   242  		prev_iter_inside_quote, quote_bits, error_mask := uint64(0), uint64(0), uint64(0)
   243  
   244  		mask := f([]byte(tc.input), tc.inputOE, &prev_iter_inside_quote, &quote_bits, &error_mask)
   245  
   246  		if mask != tc.expected {
   247  			t.Errorf("testFindQuoteMaskAndBits(%d): got: 0x%x want: 0x%x", i, mask, tc.expected)
   248  		}
   249  
   250  		if quote_bits != tc.expectedQB {
   251  			t.Errorf("testFindQuoteMaskAndBits(%d): got quote_bits: 0x%x want: 0x%x", i, quote_bits, tc.expectedQB)
   252  		}
   253  
   254  		if prev_iter_inside_quote != tc.expectedPIIQ {
   255  			t.Errorf("testFindQuoteMaskAndBits(%d): got prev_iter_inside_quote: 0x%x want: 0x%x", i, prev_iter_inside_quote, tc.expectedPIIQ)
   256  		}
   257  
   258  		if error_mask != tc.expectedEM {
   259  			t.Errorf("testFindQuoteMaskAndBits(%d): got error_mask: 0x%x want: 0x%x", i, error_mask, tc.expectedEM)
   260  		}
   261  	}
   262  
   263  	testCasesPIIQ := []struct {
   264  		inputPIIQ    uint64
   265  		input        string
   266  		expectedPIIQ uint64
   267  	}{
   268  		// prev_iter_inside_quote state remains unchanged
   269  		{uint64(0), `----------------------------------------------------------------`, uint64(0)},
   270  		{^uint64(0), `----------------------------------------------------------------`, ^uint64(0)},
   271  
   272  		// prev_iter_inside_quote state remains flips
   273  		{uint64(0), `---------------------------"------------------------------------`, ^uint64(0)},
   274  		{^uint64(0), `---------------------------"------------------------------------`, uint64(0)},
   275  
   276  		// prev_iter_inside_quote state remains flips twice (thus unchanged)
   277  		{uint64(0), `----------------"------------------------"----------------------`, uint64(0)},
   278  		{^uint64(0), `----------------"------------------------"----------------------`, ^uint64(0)},
   279  	}
   280  
   281  	for i, tc := range testCasesPIIQ {
   282  
   283  		prev_iter_inside_quote, quote_bits, error_mask := tc.inputPIIQ, uint64(0), uint64(0)
   284  
   285  		f([]byte(tc.input), 0, &prev_iter_inside_quote, &quote_bits, &error_mask)
   286  
   287  		if prev_iter_inside_quote != tc.expectedPIIQ {
   288  			t.Errorf("testFindQuoteMaskAndBits(%d): got prev_iter_inside_quote: 0x%x want: 0x%x", i, prev_iter_inside_quote, tc.expectedPIIQ)
   289  		}
   290  	}
   291  }
   292  
   293  func TestFindQuoteMaskAndBits(t *testing.T) {
   294  	t.Run("avx2", func(t *testing.T) {
   295  		testFindQuoteMaskAndBits(t, find_quote_mask_and_bits)
   296  	})
   297  	if cpuid.CPU.Has(cpuid.AVX512F) {
   298  		t.Run("avx512", func(t *testing.T) {
   299  			testFindQuoteMaskAndBits(t, find_quote_mask_and_bits_avx512)
   300  		})
   301  	}
   302  }
   303  
   304  func testFindStructuralBits(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, uint64, *uint64) uint64) {
   305  
   306  	testCases := []struct {
   307  		input string
   308  	}{
   309  		{`{"Image":{"Width":800,"Height":600,"Title":"View from 15th Floor`},
   310  		{`","Thumbnail":{"Url":"http://www.example.com/image/481989943","H`},
   311  		{`eight":125,"Width":100},"Animated":false,"IDs":[116,943,234,3879`},
   312  	}
   313  
   314  	prev_iter_ends_odd_backslash := uint64(0)
   315  	prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   316  	prev_iter_ends_pseudo_pred := uint64(1)
   317  	error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   318  	structurals := uint64(0)
   319  
   320  	// Declare same variables for 'multiple_calls' version
   321  	prev_iter_ends_odd_backslash_MC := uint64(0)
   322  	prev_iter_inside_quote_MC := uint64(0) // either all zeros or all ones
   323  	prev_iter_ends_pseudo_pred_MC := uint64(1)
   324  	error_mask_MC := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   325  	structurals_MC := uint64(0)
   326  
   327  	for i, tc := range testCases {
   328  
   329  		// Call assembly routines as a single method
   330  		structurals := f([]byte(tc.input), &prev_iter_ends_odd_backslash,
   331  			&prev_iter_inside_quote, &error_mask,
   332  			structurals,
   333  			&prev_iter_ends_pseudo_pred)
   334  
   335  		// Call assembly routines individually
   336  		structurals_MC := find_structural_bits_multiple_calls([]byte(tc.input), &prev_iter_ends_odd_backslash_MC,
   337  			&prev_iter_inside_quote_MC, &error_mask_MC,
   338  			structurals_MC,
   339  			&prev_iter_ends_pseudo_pred_MC)
   340  
   341  		// And compare the results
   342  		if structurals != structurals_MC {
   343  			t.Errorf("TestFindStructuralBits(%d): got: 0x%x want: 0x%x", i, structurals, structurals_MC)
   344  		}
   345  	}
   346  }
   347  
   348  func TestFindStructuralBits(t *testing.T) {
   349  	t.Run("avx2", func(t *testing.T) {
   350  		testFindStructuralBits(t, find_structural_bits)
   351  	})
   352  	if cpuid.CPU.Has(cpuid.AVX512F) {
   353  		t.Run("avx512", func(t *testing.T) {
   354  			testFindStructuralBits(t, find_structural_bits_avx512)
   355  		})
   356  	}
   357  }
   358  
   359  func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
   360  
   361  	// Test whitespace padding (for partial load of last 64 bytes) with
   362  	// string full of structural characters
   363  	msg := `::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::`
   364  
   365  	for l := len(msg); l >= 0; l-- {
   366  
   367  		prev_iter_ends_odd_backslash := uint64(0)
   368  		prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   369  		prev_iter_ends_pseudo_pred := uint64(1)
   370  		error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   371  		carried := ^uint64(0)
   372  		position := ^uint64(0)
   373  
   374  		index := indexChan{}
   375  		index.indexes = &[indexSize]uint32{}
   376  
   377  		processed := find_structural_bits_in_slice([]byte(msg[:l]), &prev_iter_ends_odd_backslash,
   378  			&prev_iter_inside_quote, &error_mask,
   379  			&prev_iter_ends_pseudo_pred,
   380  			index.indexes, &index.length, &carried, &position, 0)
   381  
   382  		if processed != uint64(l) {
   383  			t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, processed, l)
   384  		}
   385  		if index.length != l {
   386  			t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, index.length, l)
   387  		}
   388  
   389  		// Compute offset of last (structural) character and verify it points to the end of the message
   390  		lastChar := uint64(0)
   391  		for i := 0; i < index.length; i++ {
   392  			lastChar += uint64(index.indexes[i])
   393  		}
   394  		if l > 0 {
   395  			if lastChar != uint64(l-1) {
   396  				t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, lastChar, uint64(l-1))
   397  			}
   398  		} else {
   399  			if lastChar != uint64(l-1)-carried {
   400  				t.Errorf("testFindStructuralBitsWhitespacePadding(%d): got: %d want: %d", l, lastChar, uint64(l-1)-carried)
   401  			}
   402  		}
   403  	}
   404  }
   405  
   406  func TestFindStructuralBitsWhitespacePadding(t *testing.T) {
   407  	t.Run("avx2", func(t *testing.T) {
   408  		testFindStructuralBitsWhitespacePadding(t, find_structural_bits_in_slice)
   409  	})
   410  	if cpuid.CPU.Has(cpuid.AVX512F) {
   411  		t.Run("avx512", func(t *testing.T) {
   412  			testFindStructuralBitsWhitespacePadding(t, find_structural_bits_in_slice_avx512)
   413  		})
   414  	}
   415  }
   416  
   417  func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
   418  	msg := loadCompressed(t, "twitter")
   419  
   420  	prev_iter_ends_odd_backslash := uint64(0)
   421  	prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   422  	prev_iter_ends_pseudo_pred := uint64(1)
   423  	error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   424  	carried := ^uint64(0)
   425  	position := ^uint64(0)
   426  
   427  	indexes := make([]uint32, 0)
   428  
   429  	for processed := uint64(0); processed < uint64(len(msg)); {
   430  		index := indexChan{}
   431  		index.indexes = &[indexSize]uint32{}
   432  
   433  		processed += f(msg[processed:], &prev_iter_ends_odd_backslash,
   434  			&prev_iter_inside_quote, &error_mask,
   435  			&prev_iter_ends_pseudo_pred,
   436  			index.indexes, &index.length, &carried, &position, 0)
   437  
   438  		indexes = append(indexes, (*index.indexes)[:index.length]...)
   439  	}
   440  
   441  	// Last 5 expected structural (in reverse order)
   442  	const expectedStructuralsReversed = `}}":"`
   443  	const expectedLength = 55263
   444  
   445  	if len(indexes) != expectedLength {
   446  		t.Errorf("TestFindStructuralBitsLoop: got: %d want: %d", len(indexes), expectedLength)
   447  	}
   448  
   449  	pos, j := len(msg)-1, 0
   450  	for i := len(indexes) - 1; i >= len(indexes)-len(expectedStructuralsReversed); i-- {
   451  
   452  		if msg[pos] != expectedStructuralsReversed[j] {
   453  			t.Errorf("TestFindStructuralBitsLoop: got: %c want: %c", msg[pos], expectedStructuralsReversed[j])
   454  		}
   455  
   456  		pos -= int(indexes[i])
   457  		j++
   458  	}
   459  }
   460  
   461  func TestFindStructuralBitsLoop(t *testing.T) {
   462  	t.Run("avx2", func(t *testing.T) {
   463  		testFindStructuralBitsLoop(t, find_structural_bits_in_slice)
   464  	})
   465  	if cpuid.CPU.Has(cpuid.AVX512F) {
   466  		t.Run("avx512", func(t *testing.T) {
   467  			testFindStructuralBitsLoop(t, find_structural_bits_in_slice_avx512)
   468  		})
   469  	}
   470  }
   471  
   472  func benchmarkFindStructuralBits(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, uint64, *uint64) uint64) {
   473  
   474  	const msg = "                                                                "
   475  	b.SetBytes(int64(len(msg)))
   476  	b.ReportAllocs()
   477  	b.ResetTimer()
   478  
   479  	prev_iter_ends_odd_backslash := uint64(0)
   480  	prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   481  	prev_iter_ends_pseudo_pred := uint64(1)
   482  	error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   483  	structurals := uint64(0)
   484  
   485  	for i := 0; i < b.N; i++ {
   486  		f([]byte(msg), &prev_iter_ends_odd_backslash,
   487  			&prev_iter_inside_quote, &error_mask,
   488  			structurals,
   489  			&prev_iter_ends_pseudo_pred)
   490  	}
   491  }
   492  
   493  func BenchmarkFindStructuralBits(b *testing.B) {
   494  	b.Run("avx2", func(b *testing.B) {
   495  		benchmarkFindStructuralBits(b, find_structural_bits)
   496  	})
   497  	if cpuid.CPU.Has(cpuid.AVX512F) {
   498  		b.Run("avx512", func(b *testing.B) {
   499  			benchmarkFindStructuralBits(b, find_structural_bits_avx512)
   500  		})
   501  	}
   502  }
   503  
   504  func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
   505  
   506  	msg := loadCompressed(b, "twitter")
   507  
   508  	prev_iter_ends_odd_backslash := uint64(0)
   509  	prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   510  	prev_iter_ends_pseudo_pred := uint64(1)
   511  	error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   512  	carried := ^uint64(0)
   513  	position := ^uint64(0)
   514  
   515  	b.SetBytes(int64(len(msg)))
   516  	b.ReportAllocs()
   517  	b.ResetTimer()
   518  
   519  	for i := 0; i < b.N; i++ {
   520  
   521  		for processed := uint64(0); processed < uint64(len(msg)); {
   522  			index := indexChan{}
   523  			index.indexes = &[indexSize]uint32{}
   524  
   525  			processed += f(msg[processed:], &prev_iter_ends_odd_backslash,
   526  				&prev_iter_inside_quote, &error_mask,
   527  				&prev_iter_ends_pseudo_pred,
   528  				index.indexes, &index.length, &carried, &position, 0)
   529  		}
   530  	}
   531  }
   532  
   533  func BenchmarkFindStructuralBitsLoop(b *testing.B) {
   534  	b.Run("avx2", func(b *testing.B) {
   535  		benchmarkFindStructuralBitsLoop(b, find_structural_bits_in_slice)
   536  	})
   537  	if cpuid.CPU.Has(cpuid.AVX512F) {
   538  		b.Run("avx512", func(b *testing.B) {
   539  			benchmarkFindStructuralBitsLoop(b, find_structural_bits_in_slice_avx512)
   540  		})
   541  	}
   542  }
   543  
   544  func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
   545  
   546  	msg := loadCompressed(b, "twitter")
   547  	cpus := runtime.NumCPU()
   548  
   549  	b.SetBytes(int64(len(msg) * cpus))
   550  	b.ResetTimer()
   551  
   552  	for i := 0; i < b.N; i++ {
   553  		var wg sync.WaitGroup
   554  		wg.Add(cpus)
   555  		for cpu := 0; cpu < cpus; cpu++ {
   556  			go func() {
   557  				prev_iter_ends_odd_backslash := uint64(0)
   558  				prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   559  				prev_iter_ends_pseudo_pred := uint64(1)
   560  				error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   561  				carried := ^uint64(0)
   562  				position := ^uint64(0)
   563  
   564  				for processed := uint64(0); processed < uint64(len(msg)); {
   565  					index := indexChan{}
   566  					index.indexes = &[indexSize]uint32{}
   567  
   568  					processed += f(msg[processed:], &prev_iter_ends_odd_backslash,
   569  						&prev_iter_inside_quote, &error_mask,
   570  						&prev_iter_ends_pseudo_pred,
   571  						index.indexes, &index.length, &carried, &position, 0)
   572  				}
   573  				defer wg.Done()
   574  			}()
   575  		}
   576  		wg.Wait()
   577  	}
   578  }
   579  
   580  func BenchmarkFindStructuralBitsParallelLoop(b *testing.B) {
   581  	b.Run("avx2", func(b *testing.B) {
   582  		benchmarkFindStructuralBitsParallelLoop(b, find_structural_bits_in_slice)
   583  	})
   584  	if cpuid.CPU.Has(cpuid.AVX512F) {
   585  		b.Run("avx512", func(b *testing.B) {
   586  			benchmarkFindStructuralBitsParallelLoop(b, find_structural_bits_in_slice_avx512)
   587  		})
   588  	}
   589  }
   590  
   591  // find_structural_bits version that calls the individual assembly routines individually
   592  func find_structural_bits_multiple_calls(buf []byte, prev_iter_ends_odd_backslash *uint64,
   593  	prev_iter_inside_quote, error_mask *uint64,
   594  	structurals uint64,
   595  	prev_iter_ends_pseudo_pred *uint64) uint64 {
   596  	quote_bits := uint64(0)
   597  	whitespace_mask := uint64(0)
   598  
   599  	odd_ends := find_odd_backslash_sequences(buf, prev_iter_ends_odd_backslash)
   600  
   601  	// detect insides of quote pairs ("quote_mask") and also our quote_bits themselves
   602  	quote_mask := find_quote_mask_and_bits(buf, odd_ends, prev_iter_inside_quote, &quote_bits, error_mask)
   603  
   604  	find_whitespace_and_structurals(buf, &whitespace_mask, &structurals)
   605  
   606  	// fixup structurals to reflect quotes and add pseudo-structural characters
   607  	return finalize_structurals(structurals, whitespace_mask, quote_mask, quote_bits, prev_iter_ends_pseudo_pred)
   608  }
   609  
   610  func testFindWhitespaceAndStructurals(t *testing.T, f func([]byte, *uint64, *uint64)) {
   611  
   612  	testCases := []struct {
   613  		input          string
   614  		expected_ws    uint64
   615  		expected_strls uint64
   616  	}{
   617  		{`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x0, 0x0},
   618  		{` aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x1, 0x0},
   619  		{`:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x0, 0x1},
   620  		{` :aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x1, 0x2},
   621  		{`: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, 0x2, 0x1},
   622  		{`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa `, 0x8000000000000000, 0x0},
   623  		{`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:`, 0x0, 0x8000000000000000},
   624  		{`a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a `, 0xaaaaaaaaaaaaaaaa, 0x0},
   625  		{` a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a`, 0x5555555555555555, 0x0},
   626  		{`a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:`, 0x0, 0xaaaaaaaaaaaaaaaa},
   627  		{`:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a`, 0x0, 0x5555555555555555},
   628  		{`                                                                `, 0xffffffffffffffff, 0x0},
   629  		{`{                                                               `, 0xfffffffffffffffe, 0x1},
   630  		{`}                                                               `, 0xfffffffffffffffe, 0x1},
   631  		{`"                                                               `, 0xfffffffffffffffe, 0x0},
   632  		{`::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::`, 0x0, 0xffffffffffffffff},
   633  		{`{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{`, 0x0, 0xffffffffffffffff},
   634  		{`}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}`, 0x0, 0xffffffffffffffff},
   635  		{`  :                                                             `, 0xfffffffffffffffb, 0x4},
   636  		{`    :                                                           `, 0xffffffffffffffef, 0x10},
   637  		{`      :     :      :          :             :                  :`, 0x7fffefffbff7efbf, 0x8000100040081040},
   638  		{demo_json, 0x421000000000000, 0x40440220301},
   639  	}
   640  
   641  	for i, tc := range testCases {
   642  		whitespace := uint64(0)
   643  		structurals := uint64(0)
   644  
   645  		f([]byte(tc.input), &whitespace, &structurals)
   646  
   647  		if whitespace != tc.expected_ws {
   648  			t.Errorf("testFindWhitespaceAndStructurals(%d): got: 0x%x want: 0x%x", i, whitespace, tc.expected_ws)
   649  		}
   650  
   651  		if structurals != tc.expected_strls {
   652  			t.Errorf("testFindWhitespaceAndStructurals(%d): got: 0x%x want: 0x%x", i, structurals, tc.expected_strls)
   653  		}
   654  	}
   655  }
   656  
   657  func TestFindWhitespaceAndStructurals(t *testing.T) {
   658  	t.Run("avx2", func(t *testing.T) {
   659  		testFindWhitespaceAndStructurals(t, find_whitespace_and_structurals)
   660  	})
   661  	if cpuid.CPU.Has(cpuid.AVX512F) {
   662  		t.Run("avx512", func(t *testing.T) {
   663  			testFindWhitespaceAndStructurals(t, find_whitespace_and_structurals_avx512)
   664  		})
   665  	}
   666  }
   667  
   668  func TestFlattenBitsIncremental(t *testing.T) {
   669  
   670  	testCases := []struct {
   671  		masks    []uint64
   672  		expected []uint32
   673  	}{
   674  		// Single mask
   675  		{[]uint64{0x11}, []uint32{0x1, 0x4}},
   676  		{[]uint64{0x100100100100}, []uint32{0x9, 0xc, 0xc, 0xc}},
   677  		{[]uint64{0x100100100300}, []uint32{0x9, 0x1, 0xb, 0xc, 0xc}},
   678  		{[]uint64{0x8101010101010101}, []uint32{0x1, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x7}},
   679  		{[]uint64{0x4000000000000000}, []uint32{0x3f}},
   680  		{[]uint64{0x8000000000000000}, []uint32{0x40}},
   681  		{[]uint64{0xf000000000000000}, []uint32{0x3d, 0x1, 0x1, 0x1}},
   682  		{[]uint64{0xffffffffffffffff}, []uint32{
   683  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   684  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   685  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   686  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   687  		}},
   688  		////
   689  		//// Multiple masks
   690  		{[]uint64{0x1, 0x1000}, []uint32{0x1, 0x4c}},
   691  		{[]uint64{0x1, 0x4000000000000000}, []uint32{0x1, 0x7e}},
   692  		{[]uint64{0x1, 0x8000000000000000}, []uint32{0x1, 0x7f}},
   693  		{[]uint64{0x1, 0x0, 0x8000000000000000}, []uint32{0x1, 0xbf}},
   694  		{[]uint64{0x1, 0x0, 0x0, 0x8000000000000000}, []uint32{0x1, 0xff}},
   695  		{[]uint64{0x100100100100100, 0x100100100100100}, []uint32{0x9, 0xc, 0xc, 0xc, 0xc, 0x10, 0xc, 0xc, 0xc, 0xc}},
   696  		{[]uint64{0xffffffffffffffff, 0xffffffffffffffff}, []uint32{
   697  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   698  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   699  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   700  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   701  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   702  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   703  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   704  			0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
   705  		}},
   706  	}
   707  
   708  	for i, tc := range testCases {
   709  
   710  		index := indexChan{}
   711  		index.indexes = &[indexSize]uint32{}
   712  		carried := 0
   713  		position := ^uint64(0)
   714  
   715  		for _, mask := range tc.masks {
   716  			flatten_bits_incremental(index.indexes, &index.length, mask, &carried, &position)
   717  		}
   718  
   719  		if index.length != len(tc.expected) {
   720  			t.Errorf("TestFlattenBitsIncremental(%d): got: %d want: %d", i, index.length, len(tc.expected))
   721  		}
   722  
   723  		compare := make([]uint32, 0, 1024)
   724  		for idx := 0; idx < index.length; idx++ {
   725  			compare = append(compare, index.indexes[idx])
   726  		}
   727  
   728  		if !reflect.DeepEqual(compare, tc.expected) {
   729  			t.Errorf("TestFlattenBitsIncremental(%d): got: %v want: %v", i, compare, tc.expected)
   730  		}
   731  	}
   732  }
   733  
   734  func BenchmarkFlattenBits(b *testing.B) {
   735  
   736  	msg := loadCompressed(b, "twitter")
   737  
   738  	prev_iter_ends_odd_backslash := uint64(0)
   739  	prev_iter_inside_quote := uint64(0) // either all zeros or all ones
   740  	prev_iter_ends_pseudo_pred := uint64(1)
   741  	error_mask := uint64(0) // for unescaped characters within strings (ASCII code points < 0x20)
   742  	structurals := uint64(0)
   743  
   744  	structuralsArray := make([]uint64, 0, len(msg)>>6)
   745  
   746  	// Collect all structurals into array
   747  	for i := 0; i < len(msg)-64; i += 64 {
   748  		find_structural_bits([]byte(msg)[i:], &prev_iter_ends_odd_backslash,
   749  			&prev_iter_inside_quote, &error_mask,
   750  			structurals,
   751  			&prev_iter_ends_pseudo_pred)
   752  
   753  		structuralsArray = append(structuralsArray, structurals)
   754  	}
   755  
   756  	b.SetBytes(int64(len(structuralsArray) * 8))
   757  	b.ReportAllocs()
   758  	b.ResetTimer()
   759  
   760  	index := indexChan{}
   761  	index.indexes = &[indexSize]uint32{}
   762  	carried := 0
   763  	position := ^uint64(0)
   764  
   765  	for i := 0; i < b.N; i++ {
   766  		for _, structurals := range structuralsArray {
   767  			index.length = 0 // reset length to prevent overflow
   768  			flatten_bits_incremental(index.indexes, &index.length, structurals, &carried, &position)
   769  		}
   770  	}
   771  }