github.com/vuuihc/gocedar@v0.1.0/mmap_test.go (about)

     1  package cedar
     2  
     3  import (
     4  	"bufio"
     5  	"log"
     6  	"os"
     7  	"path"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/go-ego/cedar"
    12  	"github.com/stretchr/testify/require"
    13  )
    14  
    15  const (
    16  	sourceFile = "/mnt/cbs/data/song_name_500w" // a file contains one key per line
    17  	dumpPath   = "./dump_path"                  // a path to the dumped file
    18  )
    19  
    20  func TestMDumpAndLoad(t *testing.T) {
    21  	if _, err := os.Stat(dumpPath); err != nil {
    22  		require.NoError(t, os.MkdirAll(dumpPath, 0644))
    23  	}
    24  
    25  	sf, err := os.Open(sourceFile)
    26  	require.NoError(t, err)
    27  	defer func() {
    28  		require.NoError(t, sf.Close())
    29  	}()
    30  
    31  	reader := bufio.NewReader(sf)
    32  
    33  	cd := cedar.New()
    34  	t1 := time.Now()
    35  	var index int
    36  	lines := make([][]byte, 0)
    37  	for {
    38  		line, err := reader.ReadBytes('\n')
    39  		if err != nil {
    40  			log.Printf("ReadBytes fail, err %v\n", err)
    41  			break
    42  		}
    43  		err = cd.Insert(line, index)
    44  		lines = append(lines, line)
    45  		if err != nil {
    46  			log.Printf("Insert fail, line %s, err %v\n", line, err)
    47  			break
    48  		}
    49  		index++
    50  	}
    51  	log.Printf("go-gse/cedar insert %d keys cost %v", index, time.Since(t1))
    52  
    53  	t1 = time.Now()
    54  	for _, w := range lines {
    55  		_, err = cd.Get(w)
    56  		require.NotEqual(t, cedar.ErrNoPath, err)
    57  	}
    58  	log.Printf("go-gse/cedar Search cost %v", time.Since(t1))
    59  
    60  	t1 = time.Now()
    61  	err = cd.SaveToFile(path.Join(dumpPath, "trie_gob"), "gob")
    62  	require.NoError(t, err)
    63  	log.Printf("go-gse/cedar SaveToFile with gob cost %v", time.Since(t1))
    64  
    65  	gocedar := New(&Options{
    66  		UseMMap:  true,
    67  		MMapPath: dumpPath,
    68  	})
    69  	t1 = time.Now()
    70  	index = 0
    71  	for i, line := range lines {
    72  		index++
    73  		err = gocedar.Insert(line, i)
    74  		require.NoError(t, err)
    75  	}
    76  
    77  	log.Printf("gocedar insert %d keys cost %v", index, time.Since(t1))
    78  
    79  	t1 = time.Now()
    80  	gocedar.Close()
    81  	log.Printf("gocedar close cost %v", time.Since(t1))
    82  
    83  }
    84  func TestMLoadFromGob(t *testing.T) {
    85  	_, err := os.Stat(dumpPath)
    86  	require.NoError(t, err)
    87  
    88  	t1 := time.Now()
    89  	jcd := cedar.New()
    90  	err = jcd.LoadFromFile(path.Join(dumpPath, "trie_gob"), "gob")
    91  	require.NoError(t, err)
    92  	log.Printf("go-gse/cedar LoadFromFile with gob cost %v", time.Since(t1))
    93  
    94  	sf, err := os.Open(sourceFile)
    95  	require.NoError(t, err)
    96  	defer sf.Close() // nolint
    97  	reader := bufio.NewReader(sf)
    98  
    99  	t1 = time.Now()
   100  	var index int
   101  	var maxCost time.Duration
   102  	for {
   103  		line, err := reader.ReadBytes('\n')
   104  		if err != nil {
   105  			log.Printf("ReadBytes fail, err %v\n", err)
   106  			break
   107  		}
   108  		t2 := time.Now()
   109  		v, err := jcd.Get(line)
   110  		require.NoError(t, err)
   111  		require.Equal(t, index, v)
   112  		index++
   113  		if time.Since(t2) > maxCost {
   114  			maxCost = time.Since(t2)
   115  		}
   116  	}
   117  	log.Printf("gob cedar Search cost %v, maxCost %v", time.Since(t1), maxCost)
   118  }
   119  
   120  func TestMLoadFromMMap(t *testing.T) {
   121  	sf, err := os.Open(sourceFile)
   122  	require.NoError(t, err)
   123  	defer func() {
   124  		require.NoError(t, sf.Close())
   125  	}()
   126  	reader := bufio.NewReader(sf)
   127  
   128  	t1 := time.Now()
   129  	ngocedar := New(&Options{
   130  		UseMMap:  true,
   131  		MMapPath: dumpPath,
   132  	})
   133  	log.Printf("gocedar load from mmap cost %v", time.Since(t1))
   134  
   135  	t1 = time.Now()
   136  	var index int
   137  	var maxCost time.Duration
   138  	for {
   139  		line, err := reader.ReadBytes('\n')
   140  		if err != nil {
   141  			log.Printf("ReadBytes fail, err %v\n", err)
   142  			break
   143  		}
   144  		t2 := time.Now()
   145  		v, err := ngocedar.Get(line)
   146  		require.NoError(t, err)
   147  		require.Equal(t, index, v)
   148  		index++
   149  		if maxCost < time.Since(t2) {
   150  			maxCost = time.Since(t2)
   151  		}
   152  	}
   153  	log.Printf("gocedar Search cost %v, maxSingleCost %v", time.Since(t1), maxCost)
   154  }