github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/scanner/bigtable_index_reader_test.go (about)

     1  package scanner
     2  
     3  import (
     4  	"testing"
     5  
     6  	"cloud.google.com/go/bigtable"
     7  	"github.com/stretchr/testify/require"
     8  
     9  	"github.com/cortexproject/cortex/pkg/chunk"
    10  )
    11  
    12  func TestParseRowKey(t *testing.T) {
    13  	tcs := map[string]struct {
    14  		row   bigtable.Row
    15  		table string
    16  
    17  		expectedEntries []chunk.IndexEntry
    18  		expectedError   string
    19  	}{
    20  		"newStorageClientV1 format": {
    21  			row: map[string][]bigtable.ReadItem{
    22  				"f": {
    23  					{Row: "testUser:d18500:test_metric\u0000eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Column: "f:c", Value: []byte("-")},
    24  				},
    25  			},
    26  			table: "test",
    27  			expectedEntries: []chunk.IndexEntry{
    28  				{
    29  					TableName:  "test",
    30  					HashValue:  "testUser:d18500:test_metric",
    31  					RangeValue: []byte("eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\x00\x00\x007\x00"),
    32  					Value:      []byte("-"),
    33  				},
    34  			},
    35  		},
    36  		// 2a) newStorageClientColumnKey, WITHOUT key distribution
    37  		//    - RowKey = entry.HashValue
    38  		//    - Column: entry.RangeValue (in family "f")
    39  		//    - Value: entry.Value
    40  		"newStorageClientColumnKey without key distribution": {
    41  			row: map[string][]bigtable.ReadItem{
    42  				"f": {
    43  					{Row: "testUser:d18500:test_metric", Column: "f:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Value: []byte("-")},
    44  				},
    45  			},
    46  			table: "test",
    47  			expectedEntries: []chunk.IndexEntry{
    48  				{
    49  					TableName:  "test",
    50  					HashValue:  "testUser:d18500:test_metric",
    51  					RangeValue: []byte("eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\x00\x00\x007\x00"),
    52  					Value:      []byte("-"),
    53  				},
    54  			},
    55  		},
    56  		"newStorageClientColumnKey without key distribution, multiple columns": {
    57  			row: map[string][]bigtable.ReadItem{
    58  				"f": {
    59  					{Row: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:00a4cb80\u0000\u0000chunkID_1\u00003\u0000", Value: []byte("")},
    60  					{Row: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:05265c00\x00\x00chunkID_2\x003\x00", Value: []byte("")},
    61  					{Row: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:0036ee80\x00\x00chunkID_2\x003\x00", Value: []byte("")},
    62  				},
    63  			},
    64  			table: "test",
    65  			expectedEntries: []chunk.IndexEntry{
    66  				{
    67  					TableName:  "test",
    68  					HashValue:  "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
    69  					RangeValue: []byte("0036ee80\x00\x00chunkID_2\x003\x00"),
    70  					Value:      []byte(""),
    71  				},
    72  				{
    73  					TableName:  "test",
    74  					HashValue:  "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
    75  					RangeValue: []byte("00a4cb80\x00\x00chunkID_1\x003\x00"),
    76  					Value:      []byte(""),
    77  				},
    78  				{
    79  					TableName:  "test",
    80  					HashValue:  "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
    81  					RangeValue: []byte("05265c00\x00\x00chunkID_2\x003\x00"),
    82  					Value:      []byte(""),
    83  				},
    84  			},
    85  		},
    86  
    87  		// 2b) newStorageClientColumnKey, WITH key distribution
    88  		//    - RowKey: hashPrefix(entry.HashValue) + "-" + entry.HashValue, where hashPrefix is 64-bit FNV64a hash, encoded as little-endian hex value
    89  		//    - Column: entry.RangeValue (in family "f")
    90  		//    - Value: entry.Value
    91  		"newStorageClientColumnKey with key distribution, multiple columns": {
    92  			row: map[string][]bigtable.ReadItem{
    93  				"f": {
    94  					{Row: "15820f698f0f8d81-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:00a4cb80\u0000\u0000chunkID_1\u00003\u0000", Value: []byte("")},
    95  					{Row: "15820f698f0f8d81-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:05265c00\x00\x00chunkID_2\x003\x00", Value: []byte("")},
    96  					{Row: "15820f698f0f8d81-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:0036ee80\x00\x00chunkID_2\x003\x00", Value: []byte("")},
    97  				},
    98  			},
    99  			table: "test",
   100  			expectedEntries: []chunk.IndexEntry{
   101  				{
   102  					TableName:  "test",
   103  					HashValue:  "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
   104  					RangeValue: []byte("0036ee80\x00\x00chunkID_2\x003\x00"),
   105  					Value:      []byte(""),
   106  				},
   107  				{
   108  					TableName:  "test",
   109  					HashValue:  "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
   110  					RangeValue: []byte("00a4cb80\x00\x00chunkID_1\x003\x00"),
   111  					Value:      []byte(""),
   112  				},
   113  				{
   114  					TableName:  "test",
   115  					HashValue:  "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
   116  					RangeValue: []byte("05265c00\x00\x00chunkID_2\x003\x00"),
   117  					Value:      []byte(""),
   118  				},
   119  			},
   120  		},
   121  		"different row keys": {
   122  			row: map[string][]bigtable.ReadItem{
   123  				"f": {
   124  					{Row: "a", Column: "f:c", Value: []byte("-")},
   125  					{Row: "b", Column: "f:c", Value: []byte("-")},
   126  				},
   127  			},
   128  			table:         "test",
   129  			expectedError: "rowkey mismatch: \"b\", \"a\"",
   130  		},
   131  
   132  		"newStorageClientV1, invalid column": {
   133  			row: map[string][]bigtable.ReadItem{
   134  				"f": {
   135  					{Row: "testUser:d18500:test_metric\u0000eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Column: "wrong", Value: []byte("-")},
   136  				},
   137  			},
   138  			table:         "test",
   139  			expectedError: "found rangeValue in RowKey, but column is not 'f:c': \"wrong\"",
   140  		},
   141  
   142  		"newStorageClientColumnKey, invalid column family": {
   143  			row: map[string][]bigtable.ReadItem{
   144  				"f": {
   145  					{Row: "testUser:d18500:test_metric", Column: "family:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Value: []byte("-")},
   146  				},
   147  			},
   148  			table:         "test",
   149  			expectedError: "invalid column prefix: \"family:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\\x00\\x00\\x007\\x00\"",
   150  		},
   151  		"newStorageClientColumnKey, invalid hash (hash ignored, not stripped)": {
   152  			row: map[string][]bigtable.ReadItem{
   153  				"f": {
   154  					{Row: "1234567890123456-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:00a4cb80\u0000\u0000chunkID_1\u00003\u0000", Value: []byte("")},
   155  				},
   156  			},
   157  			expectedEntries: []chunk.IndexEntry{
   158  				{
   159  					TableName:  "test",
   160  					HashValue:  "1234567890123456-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
   161  					RangeValue: []byte("00a4cb80\u0000\u0000chunkID_1\u00003\u0000"),
   162  					Value:      []byte(""),
   163  				},
   164  			},
   165  			table: "test",
   166  		},
   167  	}
   168  
   169  	for name, tc := range tcs {
   170  		t.Run(name, func(t *testing.T) {
   171  			entries, err := parseRowKey(tc.row, tc.table)
   172  
   173  			if tc.expectedError != "" {
   174  				require.EqualError(t, err, tc.expectedError)
   175  				require.Nil(t, entries)
   176  			} else {
   177  				require.NoError(t, err)
   178  				require.Equal(t, tc.expectedEntries, entries)
   179  			}
   180  		})
   181  	}
   182  }