github.com/muhammedhassanm/blockchain@v0.0.0-20200120143007-697261defd4d/sawtooth-core-master/validator/tests/test_indexing/tests.py (about) 1 # Copyright 2017 Intel Corporation 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ------------------------------------------------------------------------------ 15 import os 16 import shutil 17 import tempfile 18 import unittest 19 import struct 20 21 from sawtooth_validator.database.indexed_database import IndexedDatabase 22 23 24 class IndexedDatabaseTest(unittest.TestCase): 25 def __init__(self, test_name): 26 super().__init__(test_name) 27 self._temp_dir = None 28 29 def setUp(self): 30 self._temp_dir = tempfile.mkdtemp() 31 32 def tearDown(self): 33 shutil.rmtree(self._temp_dir) 34 35 def test_count(self): 36 """Test that a database with three records, plus an index will 37 return the correct count of primary key/values, using `len`. 38 """ 39 db = IndexedDatabase( 40 os.path.join(self._temp_dir, 'test_db'), 41 _serialize_tuple, 42 _deserialize_tuple, 43 indexes={'name': lambda tup: [tup[1].encode()]}, 44 flag='c', 45 _size=1024**2) 46 47 db.put('1', (1, "foo", "bar")) 48 db.put('2', (2, "alice", "Alice's data")) 49 db.put('3', (3, "bob", "Bob's data")) 50 51 self.assertEqual(3, len(db)) 52 53 self.assertEqual(3, db.count()) 54 self.assertEqual(3, db.count(index="name")) 55 56 def test_contains(self): 57 """Given a database with three records and an index, test the 58 following: 59 - a primary key will return True for `contains_key` and `in` 60 - a non-existent key will return False for both of those methods 61 - an index key will return True for `contains_key`, using the index 62 """ 63 db = IndexedDatabase( 64 os.path.join(self._temp_dir, 'test_db'), 65 _serialize_tuple, 66 _deserialize_tuple, 67 indexes={'name': lambda tup: [tup[1].encode()]}, 68 flag='c', 69 _size=1024**2) 70 71 db.put('1', (1, "foo", "bar")) 72 db.put('2', (2, "alice", "Alice's data")) 73 db.put('3', (3, "bob", "Bob's data")) 74 75 self.assertTrue('1' in db) 76 self.assertTrue(db.contains_key('1')) 77 self.assertFalse('4' in db) 78 self.assertFalse(db.contains_key('4')) 79 80 self.assertTrue(db.contains_key('alice', index='name')) 81 self.assertFalse(db.contains_key('charlie', index='name')) 82 83 def test_get_multi(self): 84 """Given a database with three records and an index, test that it can 85 return multiple values from a set of keys. 86 87 Show that: 88 - it returns all existing values in the list 89 - ignores keys that do not exist (i.e. no error is thrown) 90 - it works with index keys in the same manor 91 """ 92 db = IndexedDatabase( 93 os.path.join(self._temp_dir, 'test_db'), 94 _serialize_tuple, 95 _deserialize_tuple, 96 indexes={'name': lambda tup: [tup[1].encode()]}, 97 flag='c', 98 _size=1024**2) 99 100 db.put('1', (1, "foo", "bar")) 101 db.put('2', (2, "alice", "Alice's data")) 102 db.put('3', (3, "bob", "Bob's data")) 103 104 # get multi via the primary key 105 self.assertEqual([ 106 ('3', (3, "bob", "Bob's data")), 107 ('2', (2, "alice", "Alice's data"))], 108 db.get_multi(['3', '2'])) 109 110 # Ignore unknown primary keys 111 self.assertEqual([ 112 ('3', (3, "bob", "Bob's data"))], 113 db.get_multi(['3', '4'])) 114 115 # Get multi via an index 116 self.assertEqual([ 117 ('1', (1, "foo", "bar")), 118 ('3', (3, "bob", "Bob's data"))], 119 db.get_multi(['foo', 'bob'], index='name')) 120 121 # Ignore unknown index keys 122 self.assertEqual([ 123 ('3', (3, "bob", "Bob's data"))], 124 db.get_multi(['bar', 'bob'], index='name')) 125 126 def test_indexing(self): 127 """Test basic indexing around name 128 """ 129 db = IndexedDatabase( 130 os.path.join(self._temp_dir, 'test_db'), 131 _serialize_tuple, 132 _deserialize_tuple, 133 indexes={'name': lambda tup: [tup[1].encode()]}, 134 flag='c', 135 _size=1024**2) 136 137 db.put('1', (1, "foo", "bar")) 138 db.put('2', (2, "alice", "Alice's data")) 139 db.put('3', (3, "bob", "Bob's data")) 140 141 self.assertEqual((1, "foo", "bar"), db.get('1')) 142 143 self.assertEqual( 144 (2, "alice", "Alice's data"), 145 db.get('alice', index='name')) 146 147 def test_iteration(self): 148 """Test iteration on over the items in a database, using the natural 149 order of the primary keys. 150 """ 151 db = IndexedDatabase( 152 os.path.join(self._temp_dir, 'test_db'), 153 _serialize_tuple, 154 _deserialize_tuple, 155 indexes={'name': lambda tup: [tup[1].encode()]}, 156 flag='c', 157 _size=1024**2) 158 159 db.put('1', (1, "foo", "bar")) 160 db.put('2', (2, "alice", "Alice's data")) 161 db.put('3', (3, "bob", "Bob's data")) 162 163 with db.cursor() as curs: 164 ordered_values = list(curs.iter()) 165 166 self.assertEqual( 167 [(1, "foo", "bar"), 168 (2, "alice", "Alice's data"), 169 (3, "bob", "Bob's data")], 170 ordered_values) 171 172 def test_reverse_iteration(self): 173 """Test reverse iteration over the items in a database, using the 174 reverse natural order of the primary keys. 175 """ 176 db = IndexedDatabase(os.path.join(self._temp_dir, 'test_db'), 177 _serialize_tuple, _deserialize_tuple, 178 indexes={ 179 'name': lambda tup: [tup[1].encode()] 180 }, 181 flag='c', 182 _size=1024**2) 183 184 db.put('1', (1, "foo", "bar")) 185 db.put('2', (2, "alice", "Alice's data")) 186 db.put('3', (3, "bob", "Bob's data")) 187 188 with db.cursor() as curs: 189 ordered_values = list(curs.iter_rev()) 190 191 self.assertEqual( 192 [(3, "bob", "Bob's data"), 193 (2, "alice", "Alice's data"), 194 (1, "foo", "bar")], 195 ordered_values) 196 197 def test_iteration_with_concurrent_mods(self): 198 """Given a database with three items, and a cursor on the primary keys, 199 test that a concurrent update will: 200 - not change the results 201 - not interrupt the iteration with an error 202 """ 203 db = IndexedDatabase( 204 os.path.join(self._temp_dir, 'test_db'), 205 _serialize_tuple, 206 _deserialize_tuple, 207 indexes={'name': lambda tup: [tup[1].encode()]}, 208 flag='c', 209 _size=1024**2) 210 211 db.put('1', (1, "foo", "bar")) 212 db.put('2', (2, "alice", "Alice's data")) 213 db.put('3', (3, "bob", "Bob's data")) 214 215 with db.cursor() as curs: 216 iterator = curs.iter() 217 218 self.assertEqual(1, next(iterator)[0]) 219 220 db.put('4', (4, 'charlie', "Charlie's data")) 221 222 self.assertEqual(2, next(iterator)[0]) 223 self.assertEqual(3, next(iterator)[0]) 224 225 with self.assertRaises(StopIteration): 226 next(iterator) 227 228 def test_first(self): 229 """Given a database with three items and a cursor on the primary keys, 230 test that the cursor can be properly position to the first element in 231 the natural order of the keys 232 """ 233 db = IndexedDatabase( 234 os.path.join(self._temp_dir, 'test_db'), 235 _serialize_tuple, 236 _deserialize_tuple, 237 indexes={'name': lambda tup: [tup[1].encode()]}, 238 flag='c', 239 _size=1024**2) 240 241 db.put('1', (1, "alice", "Alice's data")) 242 db.put('2', (2, "bob", "Bob's data")) 243 db.put('3', (3, 'charlie', "Charlie's data")) 244 245 with db.cursor() as curs: 246 # Start from the end 247 last = next(curs.iter_rev()) 248 # Read forward again 249 iterator = curs.iter() 250 forward = next(iterator) 251 252 self.assertEqual(last, forward) 253 254 # Check the iterator is exhausted from there 255 with self.assertRaises(StopIteration): 256 next(iterator) 257 258 # reset to first element 259 curs.first() 260 new_iter = curs.iter() 261 262 # verify that we'll see the first element again 263 first = next(new_iter) 264 self.assertEqual(1, first[0]) 265 266 def test_last(self): 267 """Given a database with three items and a cursor on the primary keys, 268 test that the cursor can be properly position to the last element in 269 the natural order of the keys 270 """ 271 db = IndexedDatabase( 272 os.path.join(self._temp_dir, 'test_db'), 273 _serialize_tuple, 274 _deserialize_tuple, 275 indexes={'name': lambda tup: [tup[1].encode()]}, 276 flag='c', 277 _size=1024**2) 278 279 db.put('1', (1, "alice", "Alice's data")) 280 db.put('2', (2, "bob", "Bob's data")) 281 db.put('3', (3, 'charlie', "Charlie's data")) 282 283 with db.cursor() as curs: 284 # Start from the beginning 285 first = next(curs.iter()) 286 # Read backward again 287 iterator = curs.iter_rev() 288 backward = next(iterator) 289 290 self.assertEqual(first, backward) 291 292 # Check the iterator is exhausted from there 293 with self.assertRaises(StopIteration): 294 next(iterator) 295 296 # reset to first element 297 curs.last() 298 new_iter = curs.iter_rev() 299 300 # verify that we'll see the first element again 301 last = next(new_iter) 302 self.assertEqual(3, last[0]) 303 304 def test_seek(self): 305 """Given a database with three items and a cursor on the primary keys, 306 test that the cursor can be properly position to an arbitrary element 307 in the natural order of the keys 308 """ 309 db = IndexedDatabase( 310 os.path.join(self._temp_dir, 'test_db'), 311 _serialize_tuple, 312 _deserialize_tuple, 313 indexes={'name': lambda tup: [tup[1].encode()]}, 314 flag='c', 315 _size=1024**2) 316 317 db.put('1', (1, "alice", "Alice's data")) 318 db.put('2', (2, "bob", "Bob's data")) 319 db.put('3', (3, 'charlie', "Charlie's data")) 320 321 with db.cursor() as curs: 322 curs.seek('2') 323 self.assertEqual(2, curs.value()[0]) 324 325 self.assertEqual('2', curs.key()) 326 327 iterator = curs.iter() 328 self.assertEqual(2, next(iterator)[0]) 329 330 def test_index_empty_db(self): 331 """Given an empty database, show that the cursor will return a value of 332 None for the first position. 333 """ 334 db = IndexedDatabase( 335 os.path.join(self._temp_dir, 'test_db'), 336 _serialize_tuple, 337 _deserialize_tuple, 338 indexes={'name': lambda tup: [tup[1].encode()]}, 339 flag='c', 340 _size=1024**2) 341 342 with db.cursor(index='name') as curs: 343 curs.first() 344 self.assertIsNone(curs.value()) 345 346 with db.cursor() as curs: 347 curs.first() 348 self.assertIsNone(curs.value()) 349 350 def test_index_iteration(self): 351 """Test iteration over the items in a database, using the natural order 352 of the index keys. 353 """ 354 db = IndexedDatabase( 355 os.path.join(self._temp_dir, 'test_db'), 356 _serialize_tuple, 357 _deserialize_tuple, 358 indexes={'name': lambda tup: [tup[1].encode()]}, 359 flag='c', 360 _size=1024**2) 361 362 db.put('1', (1, "foo", "bar")) 363 db.put('2', (2, "alice", "Alice's data")) 364 db.put('3', (3, "bob", "Bob's data")) 365 366 with db.cursor(index='name') as curs: 367 ordered_values = list(curs.iter()) 368 369 self.assertEqual( 370 [(2, "alice", "Alice's data"), 371 (3, "bob", "Bob's data"), 372 (1, "foo", "bar")], 373 ordered_values) 374 375 def test_index_reverse_iteration(self): 376 """Test reverse iteration over the items in a database, using the 377 reverse natural order of the index keys. 378 """ 379 db = IndexedDatabase( 380 os.path.join(self._temp_dir, 'test_db'), 381 _serialize_tuple, 382 _deserialize_tuple, 383 indexes={'name': lambda tup: [tup[1].encode()]}, 384 flag='c', 385 _size=1024**2) 386 387 db.put('1', (1, "foo", "bar")) 388 db.put('2', (2, "alice", "Alice's data")) 389 db.put('3', (3, "bob", "Bob's data")) 390 391 with db.cursor(index='name') as curs: 392 ordered_values = list(curs.iter_rev()) 393 394 self.assertEqual( 395 [(1, "foo", "bar"), 396 (3, "bob", "Bob's data"), 397 (2, "alice", "Alice's data")], 398 ordered_values) 399 400 def test_index_iteration_with_concurrent_mods(self): 401 """Given a database with three items, and a cursor on the index keys, 402 test that a concurrent update will: 403 - not change the results 404 - not interrupt the iteration with an error 405 """ 406 db = IndexedDatabase( 407 os.path.join(self._temp_dir, 'test_db'), 408 _serialize_tuple, 409 _deserialize_tuple, 410 indexes={'name': lambda tup: [tup[1].encode()]}, 411 flag='c', 412 _size=1024**2) 413 414 db.put('1', (1, "foo", "bar")) 415 db.put('2', (2, "alice", "Alice's data")) 416 db.put('3', (3, "bob", "Bob's data")) 417 418 with db.cursor(index='name') as curs: 419 iterator = curs.iter() 420 421 self.assertEqual(2, next(iterator)[0]) 422 423 db.put('4', (4, 'charlie', "Charlie's data")) 424 425 self.assertEqual(3, next(iterator)[0]) 426 self.assertEqual(1, next(iterator)[0]) 427 428 with self.assertRaises(StopIteration): 429 next(iterator) 430 431 def test_integer_indexing(self): 432 """Test that a database can be indexed using integer keys. 433 """ 434 int_index_config = { 435 'key_fn': lambda tup: [struct.pack('I', tup[0])], 436 'integerkey': True 437 } 438 db = IndexedDatabase( 439 os.path.join(self._temp_dir, 'test_db'), 440 _serialize_tuple, 441 _deserialize_tuple, 442 indexes={'age': int_index_config}, 443 flag='c', 444 _size=1024**2) 445 446 db.put('1', (1, "foo", "bar")) 447 db.put('2', (30, "alice", "Alice's data")) 448 db.put('3', (20, "bob", "Bob's data")) 449 db.put('4', (12, "charlie", "Charlie's data")) 450 451 self.assertEqual( 452 [1, 12, 20, 30], 453 [struct.unpack('I', k.encode())[0] for k in db.keys(index='age')]) 454 with db.cursor(index='age') as curs: 455 self.assertEqual([ 456 (1, "foo", "bar"), 457 (12, "charlie", "Charlie's data"), 458 (20, "bob", "Bob's data"), 459 (30, "alice", "Alice's data")], 460 list(curs.iter())) 461 462 def test_hex_ordered_indexing(self): 463 """Test that an index that uses hex-encoded keys will properly order 464 the keys (i.e. the natural order of the keys is in numerical order). 465 """ 466 467 def to_hex(num): 468 return "{0:#0{1}x}".format(num, 18) 469 470 def age_index_key_fn(tup): 471 return [to_hex(tup[0]).encode()] 472 473 db = IndexedDatabase( 474 os.path.join(self._temp_dir, 'test_db'), 475 _serialize_tuple, 476 _deserialize_tuple, 477 indexes={'age': age_index_key_fn}, 478 flag='c', 479 _size=1024**2) 480 481 entry_count = 100 482 for i in range(1, entry_count): 483 db.put(str(entry_count - i), (i, "foo" + str(i), "data")) 484 485 self.assertEqual( 486 [to_hex(i) for i in range(1, entry_count)], 487 list(db.keys(index='age'))) 488 489 def test_delete(self): 490 """Test that items are deleted, including their index references. 491 """ 492 db = IndexedDatabase( 493 os.path.join(self._temp_dir, 'test_db'), 494 _serialize_tuple, 495 _deserialize_tuple, 496 indexes={'name': lambda tup: [tup[1].encode()]}, 497 flag='c', 498 _size=1024**2) 499 500 db.put('1', (1, "foo", "bar")) 501 db.put('2', (2, "alice", "Alice's data")) 502 db.put('3', (3, "bob", "Bob's data")) 503 504 with db.cursor(index='name') as curs: 505 ordered_values = list(curs.iter()) 506 507 self.assertEqual( 508 [(2, "alice", "Alice's data"), 509 (3, "bob", "Bob's data"), 510 (1, "foo", "bar")], 511 ordered_values) 512 513 db.delete('3') 514 515 with db.cursor(index='name') as curs: 516 ordered_values = list(curs.iter()) 517 518 self.assertEqual( 519 [(2, "alice", "Alice's data"), 520 (1, "foo", "bar")], 521 ordered_values) 522 523 def test_update(self): 524 """Test that a database will commit both inserts and deletes using the 525 update method. 526 """ 527 db = IndexedDatabase( 528 os.path.join(self._temp_dir, 'test_db'), 529 _serialize_tuple, 530 _deserialize_tuple, 531 indexes={'name': lambda tup: [tup[1].encode()]}, 532 flag='c', 533 _size=1024**2) 534 535 db.put('1', (1, "foo", "bar")) 536 db.put('2', (2, "alice", "Alice's data")) 537 db.put('3', (3, "bob", "Bob's data")) 538 539 db.update([('4', (4, 'charlie', "Charlie's data"))], ['1']) 540 541 self.assertEqual(['2', '3', '4'], db.keys()) 542 543 def test_update_replace_index(self): 544 """Test that update will properly update insert records that have 545 the same index value of a deleted record. 546 - insert items should be added 547 - inserted items index should be correct 548 - deleted items should be removed 549 """ 550 db = IndexedDatabase( 551 os.path.join(self._temp_dir, 'test_db'), 552 _serialize_tuple, 553 _deserialize_tuple, 554 indexes={'name': lambda tup: [tup[1].encode()]}, 555 flag='c', 556 _size=1024**2) 557 db.put('1', (1, "foo", "bar")) 558 db.put('2', (2, "alice", "Alice's data")) 559 db.put('3', (3, "bob", "Bob's data")) 560 561 db.update([('4', (4, 'foo', "foo's data"))], ['1']) 562 563 self.assertEqual(['2', '3', '4'], db.keys()) 564 self.assertEqual( 565 (4, 'foo', "foo's data"), 566 db.get('foo', index='name')) 567 568 569 def _serialize_tuple(tup): 570 return "{}-{}-{}".format(*tup).encode() 571 572 573 def _deserialize_tuple(bytestring): 574 (rec_id, name, data) = tuple(bytestring.decode().split('-')) 575 return (int(rec_id), name, data)