bitbucket.org/lygo/lygo_ext_dbbolt@v0.1.3/test/lygo_db_bolt_test.go (about)

     1  package test
     2  
     3  import (
     4  	"bitbucket.org/lygo/lygo_commons/lygo_io"
     5  	"bitbucket.org/lygo/lygo_commons/lygo_mu"
     6  	"bitbucket.org/lygo/lygo_commons/lygo_paths"
     7  	"bitbucket.org/lygo/lygo_commons/lygo_reflect"
     8  	"bitbucket.org/lygo/lygo_commons/lygo_rnd"
     9  	"bitbucket.org/lygo/lygo_commons/lygo_stopwatch"
    10  	"bitbucket.org/lygo/lygo_commons/lygo_strings"
    11  	"bitbucket.org/lygo/lygo_ext_dbbolt"
    12  	"encoding/json"
    13  	"fmt"
    14  	"sync"
    15  	"testing"
    16  	"time"
    17  )
    18  
    19  type Entity struct {
    20  	Key string `json:"_key"`
    21  	Age int    `json:"age"`
    22  }
    23  
    24  type Token struct {
    25  	Key string `json:"_key"`
    26  	Exp int64  `json:"exp"`
    27  }
    28  
    29  func TestDatabase(t *testing.T) {
    30  
    31  	drop_on_exit := false
    32  
    33  	config, err := getConfig()
    34  	if nil != err {
    35  		t.Error(err)
    36  		t.Fail()
    37  	}
    38  
    39  	db := lygo_ext_dbbolt.NewBoltDatabase(config)
    40  	err = db.Open()
    41  	if nil != err {
    42  		t.Error(err)
    43  		t.Fail()
    44  	}
    45  	defer db.Close()
    46  
    47  	coll, err := db.Collection("my-coll", false)
    48  	if nil != coll {
    49  		t.Error("COLLECTION SHOULD BE NULL")
    50  		t.Fail()
    51  	}
    52  	fmt.Println("Test OK:", err)
    53  
    54  	coll, err = db.Collection("my-coll", true)
    55  	if nil != err {
    56  		t.Error(err)
    57  		t.Fail()
    58  	}
    59  
    60  	// insert item
    61  	item := &map[string]interface{}{
    62  		"_key": "1",
    63  		"name": "Mario",
    64  		"age":  22,
    65  	}
    66  	err = coll.Upsert(item)
    67  	if nil != err {
    68  		t.Error(err)
    69  		t.Fail()
    70  		return
    71  	}
    72  	item2 := &map[string]interface{}{
    73  		"_key": "2",
    74  		"name": "Giorgio",
    75  		"age":  22,
    76  	}
    77  	err = coll.Upsert(item2)
    78  	if nil != err {
    79  		t.Error(err)
    80  		t.Fail()
    81  		return
    82  	}
    83  	item3 := &map[string]interface{}{
    84  		"_key": "3",
    85  		"name": "Mirko",
    86  		"age":  45,
    87  	}
    88  	err = coll.Upsert(item3)
    89  	if nil != err {
    90  		t.Error(err)
    91  		t.Fail()
    92  		return
    93  	}
    94  	count, err := coll.CountByFieldValue("age", 22)
    95  	fmt.Println(count)
    96  
    97  	item_des, err := coll.Get("1")
    98  	if nil != err {
    99  		t.Error(err)
   100  		t.Fail()
   101  	}
   102  	fmt.Println(item_des)
   103  
   104  	data, err := coll.GetByFieldValue("age", 22)
   105  	fmt.Println(data)
   106  
   107  	// remove collection
   108  	err = coll.Drop()
   109  	if nil != err {
   110  		t.Error(err)
   111  		t.Fail()
   112  	}
   113  	coll, err = db.Collection("my-coll", false)
   114  	if nil != coll {
   115  		t.Error("COLLECTION SHOULD BE NULL")
   116  		t.Fail()
   117  	}
   118  
   119  	// remove database
   120  	if drop_on_exit {
   121  		err = db.Drop()
   122  		if nil != err {
   123  			t.Error(err)
   124  			t.Fail()
   125  		}
   126  	}
   127  }
   128  
   129  func TestQuery(t *testing.T) {
   130  	item := &map[string]interface{}{
   131  		"_key": "1",
   132  		"name": "Mario",
   133  		"age":  22,
   134  	}
   135  
   136  	query, err := lygo_ext_dbbolt.NewQueryFromFile("./query.json")
   137  	if nil != err {
   138  		t.Error(err)
   139  		t.FailNow()
   140  	}
   141  
   142  	match := query.MatchFilter(item)
   143  	if !match {
   144  		t.Error("Query do not match")
   145  		t.FailNow()
   146  	}
   147  
   148  }
   149  
   150  func TestFatData(t *testing.T) {
   151  
   152  	drop_on_exit := true
   153  
   154  	config, err := getConfig()
   155  	if nil != err {
   156  		t.Error(err)
   157  		t.Fail()
   158  	}
   159  
   160  	db := lygo_ext_dbbolt.NewBoltDatabase(config)
   161  	err = db.Open()
   162  	size, _ := db.Size()
   163  	fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size)))
   164  	if nil != err {
   165  		t.Error(err)
   166  		t.Fail()
   167  	}
   168  	defer db.Close()
   169  
   170  	coll, err := db.Collection("big-coll", true)
   171  	if nil != err {
   172  		t.Error(err)
   173  		t.Fail()
   174  	}
   175  
   176  	arrayData := generateArray()
   177  
   178  	watch := lygo_stopwatch.New()
   179  	watch.Start()
   180  
   181  	//-- START LOOP TO ADD RECORDS --//
   182  	cur, _ := coll.Count()
   183  	cur++
   184  	for i := cur; i < cur+100; i++ {
   185  		item := &map[string]interface{}{
   186  			"_key": lygo_strings.Format("%s", i),
   187  			"name": "NAME " + lygo_strings.Format("%s", i),
   188  			"age":  i,
   189  			"x":    arrayData,
   190  			"y":    arrayData,
   191  		}
   192  		err = coll.Upsert(item)
   193  		if nil != err {
   194  			t.Error(err)
   195  			t.Fail()
   196  			break
   197  		}
   198  	}
   199  	watch.Stop()
   200  	fmt.Println("ELAPSED FOR CREATION: ", watch.Seconds(), "seconds")
   201  
   202  	watch.Start()
   203  	count, err := coll.Count()
   204  	if nil != err {
   205  		t.Error(err)
   206  		t.Fail()
   207  		return
   208  	}
   209  	watch.Stop()
   210  	fmt.Println("RECORDS: ", count)
   211  	fmt.Println("ELAPSED FOR COUNT: ", watch.Seconds(), "seconds")
   212  
   213  	watch.Start()
   214  	query, err := lygo_ext_dbbolt.NewQueryFromFile("./query.json")
   215  	if nil != err {
   216  		t.Error(err)
   217  		t.FailNow()
   218  	}
   219  	query.Conditions[0].Filters[0].Value = 1
   220  	//var entity Entity
   221  	//data, err := coll.Find(query, &entity)
   222  	// data, err := coll.Find(query)
   223  	data := make([]map[string]interface{}, 0)
   224  	err = coll.ForEach(func(k, v []byte) bool {
   225  		key := string(k)
   226  		fmt.Println("KEY", key)
   227  		var e Entity
   228  		_ = json.Unmarshal(v, &e)
   229  		if query.MatchFilter(e) {
   230  			var m map[string]interface{}
   231  			_ = json.Unmarshal(v, &m)
   232  			data = append(data, m)
   233  		}
   234  		return false // continue
   235  	})
   236  
   237  	if nil != err {
   238  		t.Error(err)
   239  		t.FailNow()
   240  	}
   241  	watch.Stop()
   242  	fmt.Println("QUERY: ", len(data))
   243  	fmt.Println("ELAPSED FOR QUERY: ", watch.Seconds(), "seconds")
   244  	if len(data) > 0 {
   245  		for _, item := range data {
   246  			fmt.Println("AGE:", lygo_reflect.Get(item, "age"))
   247  		}
   248  	}
   249  
   250  	size, _ = db.Size()
   251  	fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size)))
   252  
   253  	// remove database
   254  	if drop_on_exit {
   255  		err = db.Drop()
   256  		if nil != err {
   257  			t.Error(err)
   258  			t.Fail()
   259  		}
   260  	}
   261  }
   262  
   263  // This test is for a JWT like application.
   264  // Is BBolt good to store tokens?
   265  func TestBigThinData(t *testing.T) {
   266  
   267  	dropOnExit := false
   268  	maxRecords := 1000
   269  
   270  	filename := "./db/big_thin.dat"
   271  	_ = lygo_paths.Mkdir(filename)
   272  	config := lygo_ext_dbbolt.NewBoltConfig()
   273  	config.Name = filename
   274  	watch := lygo_stopwatch.New()
   275  	var wg sync.WaitGroup
   276  
   277  	// OPEN
   278  	watch.Start()
   279  	db := lygo_ext_dbbolt.NewBoltDatabase(config)
   280  	err := db.Open()
   281  	size, _ := db.Size()
   282  	fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size)))
   283  	if nil != err {
   284  		t.Error(err)
   285  		t.Fail()
   286  	}
   287  	watch.Stop()
   288  	fmt.Println("ELAPSED FOR OPEN: ", watch.Seconds(), "seconds")
   289  
   290  	defer db.Close()
   291  
   292  	coll, err := db.Collection("big-coll", true)
   293  	if nil != err {
   294  		t.Error(err)
   295  		t.Fail()
   296  	}
   297  
   298  	if n, _ := coll.Count(); n == 0 {
   299  		//-- START LOOP TO ADD RECORDS --//
   300  		watch.Start()
   301  		for i := 0; i < maxRecords; i++ {
   302  			wg.Add(1)
   303  			go func() {
   304  				defer wg.Done()
   305  				item := &map[string]interface{}{
   306  					"_key": lygo_rnd.Uuid(),
   307  					"name": "NAME " + lygo_strings.Format("%s", i),
   308  					"exp":  time.Now().Add(15 * time.Second).Unix(),
   309  				}
   310  				err = coll.Upsert(item)
   311  				if nil != err {
   312  					t.Error(err)
   313  					t.Fail()
   314  				}
   315  			}()
   316  		}
   317  		wg.Wait()
   318  		watch.Stop()
   319  		fmt.Println("ELAPSED FOR CREATION: ", watch.Seconds(), "seconds")
   320  	}
   321  
   322  	// COUNT
   323  	watch.Start()
   324  	count, err := coll.Count()
   325  	if nil != err {
   326  		t.Error(err)
   327  		t.Fail()
   328  		return
   329  	}
   330  	watch.Stop()
   331  	fmt.Println("RECORDS: ", count)
   332  	fmt.Println("ELAPSED FOR COUNT: ", watch.Seconds(), "seconds")
   333  
   334  	// GET EXPIRED
   335  	watch.Start()
   336  	expired := make([]string, 0)
   337  	err = coll.ForEach(func(k, v []byte) bool {
   338  		key := string(k)
   339  		var e Token
   340  		_ = json.Unmarshal(v, &e)
   341  		if time.Now().Unix()-e.Exp > 0 {
   342  			expired = append(expired, key)
   343  		}
   344  		return false // continue
   345  	})
   346  	if nil != err {
   347  		t.Error(err)
   348  		t.FailNow()
   349  	}
   350  	watch.Stop()
   351  	fmt.Println("EXPIRED: ", len(expired))
   352  	fmt.Println("ELAPSED FOR QUERY: ", watch.Seconds(), "seconds")
   353  
   354  	size, _ = db.Size()
   355  	fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size)))
   356  
   357  	// remove expired
   358  	fmt.Println("REMOVING:", len(expired))
   359  	if len(expired) > 0 {
   360  		watch.Start()
   361  		for _, key := range expired {
   362  			wg.Add(1)
   363  			go func(key string) {
   364  				err = coll.Remove(key)
   365  				if nil != err {
   366  					fmt.Println("ERROR REMOVING:", err)
   367  				}
   368  				wg.Done()
   369  			}(key)
   370  		}
   371  		wg.Wait()
   372  		watch.Stop()
   373  		fmt.Println("ELAPSED FOR REMOVE: ", watch.Seconds(), "seconds")
   374  	}
   375  
   376  	size, _ = db.Size()
   377  	fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size)))
   378  
   379  	count, _ = coll.Count()
   380  	fmt.Println("RECORDS: ", count)
   381  
   382  	// remove database
   383  	if dropOnExit {
   384  		err = db.Drop()
   385  		if nil != err {
   386  			t.Error(err)
   387  			t.Fail()
   388  		}
   389  	}
   390  }
   391  
   392  func TestExpireData(t *testing.T) {
   393  	dropOnExit := false
   394  	maxRecords := 1000
   395  
   396  	filename := "./db/expiring.dat"
   397  	_ = lygo_paths.Mkdir(filename)
   398  	config := lygo_ext_dbbolt.NewBoltConfig()
   399  	config.Name = filename
   400  	watch := lygo_stopwatch.New()
   401  
   402  	// OPEN
   403  	watch.Start()
   404  	db := lygo_ext_dbbolt.NewBoltDatabase(config)
   405  	err := db.Open()
   406  	size, _ := db.Size()
   407  	fmt.Println("FILE SIZE: ", lygo_mu.FmtBytes(uint64(size)))
   408  	if nil != err {
   409  		t.Error(err)
   410  		t.Fail()
   411  	}
   412  	watch.Stop()
   413  	fmt.Println("ELAPSED FOR OPEN: ", watch.Seconds(), "seconds")
   414  
   415  	defer db.Close()
   416  
   417  	coll, err := db.Collection("big-coll", true)
   418  	if nil != err {
   419  		t.Error(err)
   420  		t.Fail()
   421  	}
   422  
   423  	coll.EnableExpire(true)
   424  
   425  	// ADD
   426  	if n, _ := coll.Count(); n == 0 {
   427  		//-- START LOOP TO ADD RECORDS --//
   428  		watch.Start()
   429  		for i := 0; i < maxRecords; i++ {
   430  			item := map[string]interface{}{
   431  				"_key":    lygo_rnd.Uuid(),
   432  				"name":    "NAME " + lygo_strings.Format("%s", i),
   433  			}
   434  			item[lygo_ext_dbbolt.FieldExpire] = time.Now().Add(5 * time.Second).Unix()
   435  			err = coll.Upsert(item)
   436  			if nil != err {
   437  				t.Error(err)
   438  				t.Fail()
   439  			}
   440  		}
   441  		watch.Stop()
   442  		fmt.Println("ELAPSED FOR ADD: ", watch.Seconds(), "seconds")
   443  	}
   444  
   445  	for {
   446  		time.Sleep(2 * time.Second)
   447  		n, _ := coll.Count()
   448  		if n == 0 {
   449  			break
   450  		} else {
   451  			fmt.Println(n)
   452  		}
   453  	}
   454  
   455  	count, _ := coll.Count()
   456  	fmt.Println("RECORDS: ", count)
   457  
   458  	// remove database
   459  	if dropOnExit {
   460  		err = db.Drop()
   461  		if nil != err {
   462  			t.Error(err)
   463  			t.Fail()
   464  		}
   465  	}
   466  }
   467  
   468  //----------------------------------------------------------------------------------------------------------------------
   469  //	p r i v a t e
   470  //----------------------------------------------------------------------------------------------------------------------
   471  
   472  func getConfig() (*lygo_ext_dbbolt.BoltConfig, error) {
   473  	text_cfg, err := lygo_io.ReadTextFromFile("./config.json")
   474  	if nil != err {
   475  		return nil, err
   476  	}
   477  	config := lygo_ext_dbbolt.NewBoltConfig()
   478  	err = config.Parse(text_cfg)
   479  
   480  	return config, err
   481  }
   482  
   483  func generateArray() []float64 {
   484  	size := 250 * 60 * 10 // 10 minutes of 250Hz data sequence
   485  	data := make([]float64, size)
   486  	for i := 0; i < size; i++ {
   487  		val := float64(i * 2)
   488  		data[i] = val
   489  	}
   490  	return data
   491  }