bitbucket.org/lygo/lygo_ext_dbbolt@v0.1.3/test/lygo_db_bolt_test.go (about) 1 package test 2 3 import ( 4 "bitbucket.org/lygo/lygo_commons/lygo_io" 5 "bitbucket.org/lygo/lygo_commons/lygo_mu" 6 "bitbucket.org/lygo/lygo_commons/lygo_paths" 7 "bitbucket.org/lygo/lygo_commons/lygo_reflect" 8 "bitbucket.org/lygo/lygo_commons/lygo_rnd" 9 "bitbucket.org/lygo/lygo_commons/lygo_stopwatch" 10 "bitbucket.org/lygo/lygo_commons/lygo_strings" 11 "bitbucket.org/lygo/lygo_ext_dbbolt" 12 "encoding/json" 13 "fmt" 14 "sync" 15 "testing" 16 "time" 17 ) 18 19 type Entity struct { 20 Key string `json:"_key"` 21 Age int `json:"age"` 22 } 23 24 type Token struct { 25 Key string `json:"_key"` 26 Exp int64 `json:"exp"` 27 } 28 29 func TestDatabase(t *testing.T) { 30 31 drop_on_exit := false 32 33 config, err := getConfig() 34 if nil != err { 35 t.Error(err) 36 t.Fail() 37 } 38 39 db := lygo_ext_dbbolt.NewBoltDatabase(config) 40 err = db.Open() 41 if nil != err { 42 t.Error(err) 43 t.Fail() 44 } 45 defer db.Close() 46 47 coll, err := db.Collection("my-coll", false) 48 if nil != coll { 49 t.Error("COLLECTION SHOULD BE NULL") 50 t.Fail() 51 } 52 fmt.Println("Test OK:", err) 53 54 coll, err = db.Collection("my-coll", true) 55 if nil != err { 56 t.Error(err) 57 t.Fail() 58 } 59 60 // insert item 61 item := &map[string]interface{}{ 62 "_key": "1", 63 "name": "Mario", 64 "age": 22, 65 } 66 err = coll.Upsert(item) 67 if nil != err { 68 t.Error(err) 69 t.Fail() 70 return 71 } 72 item2 := &map[string]interface{}{ 73 "_key": "2", 74 "name": "Giorgio", 75 "age": 22, 76 } 77 err = coll.Upsert(item2) 78 if nil != err { 79 t.Error(err) 80 t.Fail() 81 return 82 } 83 item3 := &map[string]interface{}{ 84 "_key": "3", 85 "name": "Mirko", 86 "age": 45, 87 } 88 err = coll.Upsert(item3) 89 if nil != err { 90 t.Error(err) 91 t.Fail() 92 return 93 } 94 count, err := coll.CountByFieldValue("age", 22) 95 fmt.Println(count) 96 97 item_des, err := coll.Get("1") 98 if nil != err { 99 t.Error(err) 100 t.Fail() 101 } 102 fmt.Println(item_des) 103 104 data, err := coll.GetByFieldValue("age", 22) 105 fmt.Println(data) 106 107 // remove collection 108 err = coll.Drop() 109 if nil != err { 110 t.Error(err) 111 t.Fail() 112 } 113 coll, err = db.Collection("my-coll", false) 114 if nil != coll { 115 t.Error("COLLECTION SHOULD BE NULL") 116 t.Fail() 117 } 118 119 // remove database 120 if drop_on_exit { 121 err = db.Drop() 122 if nil != err { 123 t.Error(err) 124 t.Fail() 125 } 126 } 127 } 128 129 func TestQuery(t *testing.T) { 130 item := &map[string]interface{}{ 131 "_key": "1", 132 "name": "Mario", 133 "age": 22, 134 } 135 136 query, err := lygo_ext_dbbolt.NewQueryFromFile("./query.json") 137 if nil != err { 138 t.Error(err) 139 t.FailNow() 140 } 141 142 match := query.MatchFilter(item) 143 if !match { 144 t.Error("Query do not match") 145 t.FailNow() 146 } 147 148 } 149 150 func TestFatData(t *testing.T) { 151 152 drop_on_exit := true 153 154 config, err := getConfig() 155 if nil != err { 156 t.Error(err) 157 t.Fail() 158 } 159 160 db := lygo_ext_dbbolt.NewBoltDatabase(config) 161 err = db.Open() 162 size, _ := db.Size() 163 fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size))) 164 if nil != err { 165 t.Error(err) 166 t.Fail() 167 } 168 defer db.Close() 169 170 coll, err := db.Collection("big-coll", true) 171 if nil != err { 172 t.Error(err) 173 t.Fail() 174 } 175 176 arrayData := generateArray() 177 178 watch := lygo_stopwatch.New() 179 watch.Start() 180 181 //-- START LOOP TO ADD RECORDS --// 182 cur, _ := coll.Count() 183 cur++ 184 for i := cur; i < cur+100; i++ { 185 item := &map[string]interface{}{ 186 "_key": lygo_strings.Format("%s", i), 187 "name": "NAME " + lygo_strings.Format("%s", i), 188 "age": i, 189 "x": arrayData, 190 "y": arrayData, 191 } 192 err = coll.Upsert(item) 193 if nil != err { 194 t.Error(err) 195 t.Fail() 196 break 197 } 198 } 199 watch.Stop() 200 fmt.Println("ELAPSED FOR CREATION: ", watch.Seconds(), "seconds") 201 202 watch.Start() 203 count, err := coll.Count() 204 if nil != err { 205 t.Error(err) 206 t.Fail() 207 return 208 } 209 watch.Stop() 210 fmt.Println("RECORDS: ", count) 211 fmt.Println("ELAPSED FOR COUNT: ", watch.Seconds(), "seconds") 212 213 watch.Start() 214 query, err := lygo_ext_dbbolt.NewQueryFromFile("./query.json") 215 if nil != err { 216 t.Error(err) 217 t.FailNow() 218 } 219 query.Conditions[0].Filters[0].Value = 1 220 //var entity Entity 221 //data, err := coll.Find(query, &entity) 222 // data, err := coll.Find(query) 223 data := make([]map[string]interface{}, 0) 224 err = coll.ForEach(func(k, v []byte) bool { 225 key := string(k) 226 fmt.Println("KEY", key) 227 var e Entity 228 _ = json.Unmarshal(v, &e) 229 if query.MatchFilter(e) { 230 var m map[string]interface{} 231 _ = json.Unmarshal(v, &m) 232 data = append(data, m) 233 } 234 return false // continue 235 }) 236 237 if nil != err { 238 t.Error(err) 239 t.FailNow() 240 } 241 watch.Stop() 242 fmt.Println("QUERY: ", len(data)) 243 fmt.Println("ELAPSED FOR QUERY: ", watch.Seconds(), "seconds") 244 if len(data) > 0 { 245 for _, item := range data { 246 fmt.Println("AGE:", lygo_reflect.Get(item, "age")) 247 } 248 } 249 250 size, _ = db.Size() 251 fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size))) 252 253 // remove database 254 if drop_on_exit { 255 err = db.Drop() 256 if nil != err { 257 t.Error(err) 258 t.Fail() 259 } 260 } 261 } 262 263 // This test is for a JWT like application. 264 // Is BBolt good to store tokens? 265 func TestBigThinData(t *testing.T) { 266 267 dropOnExit := false 268 maxRecords := 1000 269 270 filename := "./db/big_thin.dat" 271 _ = lygo_paths.Mkdir(filename) 272 config := lygo_ext_dbbolt.NewBoltConfig() 273 config.Name = filename 274 watch := lygo_stopwatch.New() 275 var wg sync.WaitGroup 276 277 // OPEN 278 watch.Start() 279 db := lygo_ext_dbbolt.NewBoltDatabase(config) 280 err := db.Open() 281 size, _ := db.Size() 282 fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size))) 283 if nil != err { 284 t.Error(err) 285 t.Fail() 286 } 287 watch.Stop() 288 fmt.Println("ELAPSED FOR OPEN: ", watch.Seconds(), "seconds") 289 290 defer db.Close() 291 292 coll, err := db.Collection("big-coll", true) 293 if nil != err { 294 t.Error(err) 295 t.Fail() 296 } 297 298 if n, _ := coll.Count(); n == 0 { 299 //-- START LOOP TO ADD RECORDS --// 300 watch.Start() 301 for i := 0; i < maxRecords; i++ { 302 wg.Add(1) 303 go func() { 304 defer wg.Done() 305 item := &map[string]interface{}{ 306 "_key": lygo_rnd.Uuid(), 307 "name": "NAME " + lygo_strings.Format("%s", i), 308 "exp": time.Now().Add(15 * time.Second).Unix(), 309 } 310 err = coll.Upsert(item) 311 if nil != err { 312 t.Error(err) 313 t.Fail() 314 } 315 }() 316 } 317 wg.Wait() 318 watch.Stop() 319 fmt.Println("ELAPSED FOR CREATION: ", watch.Seconds(), "seconds") 320 } 321 322 // COUNT 323 watch.Start() 324 count, err := coll.Count() 325 if nil != err { 326 t.Error(err) 327 t.Fail() 328 return 329 } 330 watch.Stop() 331 fmt.Println("RECORDS: ", count) 332 fmt.Println("ELAPSED FOR COUNT: ", watch.Seconds(), "seconds") 333 334 // GET EXPIRED 335 watch.Start() 336 expired := make([]string, 0) 337 err = coll.ForEach(func(k, v []byte) bool { 338 key := string(k) 339 var e Token 340 _ = json.Unmarshal(v, &e) 341 if time.Now().Unix()-e.Exp > 0 { 342 expired = append(expired, key) 343 } 344 return false // continue 345 }) 346 if nil != err { 347 t.Error(err) 348 t.FailNow() 349 } 350 watch.Stop() 351 fmt.Println("EXPIRED: ", len(expired)) 352 fmt.Println("ELAPSED FOR QUERY: ", watch.Seconds(), "seconds") 353 354 size, _ = db.Size() 355 fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size))) 356 357 // remove expired 358 fmt.Println("REMOVING:", len(expired)) 359 if len(expired) > 0 { 360 watch.Start() 361 for _, key := range expired { 362 wg.Add(1) 363 go func(key string) { 364 err = coll.Remove(key) 365 if nil != err { 366 fmt.Println("ERROR REMOVING:", err) 367 } 368 wg.Done() 369 }(key) 370 } 371 wg.Wait() 372 watch.Stop() 373 fmt.Println("ELAPSED FOR REMOVE: ", watch.Seconds(), "seconds") 374 } 375 376 size, _ = db.Size() 377 fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size))) 378 379 count, _ = coll.Count() 380 fmt.Println("RECORDS: ", count) 381 382 // remove database 383 if dropOnExit { 384 err = db.Drop() 385 if nil != err { 386 t.Error(err) 387 t.Fail() 388 } 389 } 390 } 391 392 func TestExpireData(t *testing.T) { 393 dropOnExit := false 394 maxRecords := 1000 395 396 filename := "./db/expiring.dat" 397 _ = lygo_paths.Mkdir(filename) 398 config := lygo_ext_dbbolt.NewBoltConfig() 399 config.Name = filename 400 watch := lygo_stopwatch.New() 401 402 // OPEN 403 watch.Start() 404 db := lygo_ext_dbbolt.NewBoltDatabase(config) 405 err := db.Open() 406 size, _ := db.Size() 407 fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size))) 408 if nil != err { 409 t.Error(err) 410 t.Fail() 411 } 412 watch.Stop() 413 fmt.Println("ELAPSED FOR OPEN: ", watch.Seconds(), "seconds") 414 415 defer db.Close() 416 417 coll, err := db.Collection("big-coll", true) 418 if nil != err { 419 t.Error(err) 420 t.Fail() 421 } 422 423 coll.EnableExpire(true) 424 425 // ADD 426 if n, _ := coll.Count(); n == 0 { 427 //-- START LOOP TO ADD RECORDS --// 428 watch.Start() 429 for i := 0; i < maxRecords; i++ { 430 item := map[string]interface{}{ 431 "_key": lygo_rnd.Uuid(), 432 "name": "NAME " + lygo_strings.Format("%s", i), 433 } 434 item[lygo_ext_dbbolt.FieldExpire] = time.Now().Add(5 * time.Second).Unix() 435 err = coll.Upsert(item) 436 if nil != err { 437 t.Error(err) 438 t.Fail() 439 } 440 } 441 watch.Stop() 442 fmt.Println("ELAPSED FOR ADD: ", watch.Seconds(), "seconds") 443 } 444 445 for { 446 time.Sleep(2 * time.Second) 447 n, _ := coll.Count() 448 if n == 0 { 449 break 450 } else { 451 fmt.Println(n) 452 } 453 } 454 455 count, _ := coll.Count() 456 fmt.Println("RECORDS: ", count) 457 458 // remove database 459 if dropOnExit { 460 err = db.Drop() 461 if nil != err { 462 t.Error(err) 463 t.Fail() 464 } 465 } 466 } 467 468 //---------------------------------------------------------------------------------------------------------------------- 469 // p r i v a t e 470 //---------------------------------------------------------------------------------------------------------------------- 471 472 func getConfig() (*lygo_ext_dbbolt.BoltConfig, error) { 473 text_cfg, err := lygo_io.ReadTextFromFile("./config.json") 474 if nil != err { 475 return nil, err 476 } 477 config := lygo_ext_dbbolt.NewBoltConfig() 478 err = config.Parse(text_cfg) 479 480 return config, err 481 } 482 483 func generateArray() []float64 { 484 size := 250 * 60 * 10 // 10 minutes of 250Hz data sequence 485 data := make([]float64, size) 486 for i := 0; i < size; i++ { 487 val := float64(i * 2) 488 data[i] = val 489 } 490 return data 491 }