github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/chunk/local/boltdb_index_client_test.go (about) 1 package local 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "testing" 10 11 "github.com/stretchr/testify/require" 12 "go.etcd.io/bbolt" 13 14 "github.com/cortexproject/cortex/pkg/chunk" 15 ) 16 17 var ( 18 testKey = []byte("test-key") 19 testValue = []byte("test-value") 20 ) 21 22 func setupDB(t *testing.T, boltdbIndexClient *BoltIndexClient, dbname string) { 23 db, err := boltdbIndexClient.GetDB(dbname, DBOperationWrite) 24 require.NoError(t, err) 25 26 err = db.Update(func(tx *bbolt.Tx) error { 27 b, err := tx.CreateBucketIfNotExists(bucketName) 28 if err != nil { 29 return err 30 } 31 32 return b.Put(testKey, testValue) 33 }) 34 require.NoError(t, err) 35 } 36 37 func TestBoltDBReload(t *testing.T) { 38 dirname, err := ioutil.TempDir(os.TempDir(), "boltdb") 39 require.NoError(t, err) 40 41 defer require.NoError(t, os.RemoveAll(dirname)) 42 43 boltdbIndexClient, err := NewBoltDBIndexClient(BoltDBConfig{ 44 Directory: dirname, 45 }) 46 require.NoError(t, err) 47 48 defer boltdbIndexClient.Stop() 49 50 testDb1 := "test1" 51 testDb2 := "test2" 52 53 setupDB(t, boltdbIndexClient, testDb1) 54 setupDB(t, boltdbIndexClient, testDb2) 55 56 boltdbIndexClient.reload() 57 require.Equal(t, 2, len(boltdbIndexClient.dbs), "There should be 2 boltdbs open") 58 59 require.NoError(t, os.Remove(filepath.Join(dirname, testDb1))) 60 61 droppedDb, err := boltdbIndexClient.GetDB(testDb1, DBOperationRead) 62 require.NoError(t, err) 63 64 valueFromDb := []byte{} 65 _ = droppedDb.View(func(tx *bbolt.Tx) error { 66 b := tx.Bucket(bucketName) 67 valueFromDb = b.Get(testKey) 68 return nil 69 }) 70 require.Equal(t, testValue, valueFromDb, "should match value from db") 71 72 boltdbIndexClient.reload() 73 74 require.Equal(t, 1, len(boltdbIndexClient.dbs), "There should be 1 boltdb open") 75 76 _, err = boltdbIndexClient.GetDB(testDb1, DBOperationRead) 77 require.Equal(t, ErrUnexistentBoltDB, err) 78 } 79 80 func TestBoltDB_GetDB(t *testing.T) { 81 dirname, err := ioutil.TempDir(os.TempDir(), "boltdb") 82 require.NoError(t, err) 83 84 defer require.NoError(t, os.RemoveAll(dirname)) 85 86 boltdbIndexClient, err := NewBoltDBIndexClient(BoltDBConfig{ 87 Directory: dirname, 88 }) 89 require.NoError(t, err) 90 91 // setup a db to already exist 92 testDb1 := "test1" 93 setupDB(t, boltdbIndexClient, testDb1) 94 95 // check whether an existing db can be fetched for reading 96 _, err = boltdbIndexClient.GetDB(testDb1, DBOperationRead) 97 require.NoError(t, err) 98 99 // check whether read operation throws ErrUnexistentBoltDB error for db which does not exists 100 unexistentDb := "unexistent-db" 101 102 _, err = boltdbIndexClient.GetDB(unexistentDb, DBOperationRead) 103 require.Equal(t, ErrUnexistentBoltDB, err) 104 105 // check whether write operation sets up a new db for writing 106 db, err := boltdbIndexClient.GetDB(unexistentDb, DBOperationWrite) 107 require.NoError(t, err) 108 require.NotEqual(t, nil, db) 109 110 // recreate index client to check whether we can read already created test1 db without writing first 111 boltdbIndexClient.Stop() 112 boltdbIndexClient, err = NewBoltDBIndexClient(BoltDBConfig{ 113 Directory: dirname, 114 }) 115 require.NoError(t, err) 116 defer boltdbIndexClient.Stop() 117 118 _, err = boltdbIndexClient.GetDB(testDb1, DBOperationRead) 119 require.NoError(t, err) 120 } 121 122 func Test_CreateTable_BoltdbRW(t *testing.T) { 123 tableName := "test" 124 dirname, err := ioutil.TempDir(os.TempDir(), "boltdb") 125 require.NoError(t, err) 126 127 indexClient, err := NewBoltDBIndexClient(BoltDBConfig{ 128 Directory: dirname, 129 }) 130 require.NoError(t, err) 131 132 tableClient, err := NewTableClient(dirname) 133 require.NoError(t, err) 134 135 err = tableClient.CreateTable(context.Background(), chunk.TableDesc{ 136 Name: tableName, 137 }) 138 require.NoError(t, err) 139 140 batch := indexClient.NewWriteBatch() 141 batch.Add(tableName, fmt.Sprintf("hash%s", "test"), []byte(fmt.Sprintf("range%s", "value")), nil) 142 143 err = indexClient.BatchWrite(context.Background(), batch) 144 require.NoError(t, err) 145 146 // try to create the same file which is already existing 147 err = tableClient.CreateTable(context.Background(), chunk.TableDesc{ 148 Name: tableName, 149 }) 150 require.NoError(t, err) 151 152 // make sure file content is not modified 153 entry := chunk.IndexQuery{ 154 TableName: tableName, 155 HashValue: fmt.Sprintf("hash%s", "test"), 156 } 157 var have []chunk.IndexEntry 158 err = indexClient.query(context.Background(), entry, func(_ chunk.IndexQuery, read chunk.ReadBatch) bool { 159 iter := read.Iterator() 160 for iter.Next() { 161 have = append(have, chunk.IndexEntry{ 162 RangeValue: iter.RangeValue(), 163 }) 164 } 165 return true 166 }) 167 require.NoError(t, err) 168 require.Equal(t, []chunk.IndexEntry{ 169 {RangeValue: []byte(fmt.Sprintf("range%s", "value"))}, 170 }, have) 171 172 } 173 174 func TestBoltDB_Writes(t *testing.T) { 175 dirname, err := ioutil.TempDir(os.TempDir(), "boltdb") 176 require.NoError(t, err) 177 178 defer func() { 179 require.NoError(t, os.RemoveAll(dirname)) 180 }() 181 182 for i, tc := range []struct { 183 name string 184 initialPuts []string 185 testPuts []string 186 testDeletes []string 187 err error 188 valuesAfterWrites []string 189 }{ 190 { 191 name: "just puts", 192 testPuts: []string{"1", "2"}, 193 valuesAfterWrites: []string{"1", "2"}, 194 }, 195 { 196 name: "just deletes", 197 initialPuts: []string{"1", "2", "3", "4"}, 198 testDeletes: []string{"1", "2"}, 199 valuesAfterWrites: []string{"3", "4"}, 200 }, 201 { 202 name: "both puts and deletes", 203 initialPuts: []string{"1", "2", "3", "4"}, 204 testPuts: []string{"5", "6"}, 205 testDeletes: []string{"1", "2"}, 206 valuesAfterWrites: []string{"3", "4", "5", "6"}, 207 }, 208 { 209 name: "deletes without initial writes", 210 testDeletes: []string{"1", "2"}, 211 err: fmt.Errorf("bucket %s not found in table 3", bucketName), 212 }, 213 } { 214 t.Run(tc.name, func(t *testing.T) { 215 tableName := fmt.Sprint(i) 216 217 indexClient, err := NewBoltDBIndexClient(BoltDBConfig{ 218 Directory: dirname, 219 }) 220 require.NoError(t, err) 221 222 defer func() { 223 indexClient.Stop() 224 }() 225 226 // doing initial writes if there are any 227 if len(tc.initialPuts) != 0 { 228 batch := indexClient.NewWriteBatch() 229 for _, put := range tc.initialPuts { 230 batch.Add(tableName, "hash", []byte(put), []byte(put)) 231 } 232 233 require.NoError(t, indexClient.BatchWrite(context.Background(), batch)) 234 } 235 236 // doing writes with testPuts and testDeletes 237 batch := indexClient.NewWriteBatch() 238 for _, put := range tc.testPuts { 239 batch.Add(tableName, "hash", []byte(put), []byte(put)) 240 } 241 for _, put := range tc.testDeletes { 242 batch.Delete(tableName, "hash", []byte(put)) 243 } 244 245 require.Equal(t, tc.err, indexClient.BatchWrite(context.Background(), batch)) 246 247 // verifying test writes by querying 248 var have []chunk.IndexEntry 249 err = indexClient.query(context.Background(), chunk.IndexQuery{ 250 TableName: tableName, 251 HashValue: "hash", 252 }, func(_ chunk.IndexQuery, read chunk.ReadBatch) bool { 253 iter := read.Iterator() 254 for iter.Next() { 255 have = append(have, chunk.IndexEntry{ 256 RangeValue: iter.RangeValue(), 257 Value: iter.Value(), 258 }) 259 } 260 return true 261 }) 262 263 require.NoError(t, err) 264 require.Len(t, have, len(tc.valuesAfterWrites)) 265 266 for i, value := range tc.valuesAfterWrites { 267 require.Equal(t, chunk.IndexEntry{ 268 RangeValue: []byte(value), 269 Value: []byte(value), 270 }, have[i]) 271 } 272 }) 273 } 274 }