github.com/scottcagno/storage@v1.8.0/pkg/lsmtree/commitlog_test.go (about)

     1  package lsmtree
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  )
     7  
     8  const baseDir = "commit-log-testing"
     9  const count = 500
    10  
    11  func TestCommitLogWriteAndReadAt(t *testing.T) {
    12  
    13  	// offsets for later
    14  	var offsets []int64
    15  
    16  	fmt.Println("opening commit log...")
    17  	// open commit log
    18  	c, err := openCommitLog(baseDir, false)
    19  	if err != nil {
    20  		t.Errorf("open: %v\n", err)
    21  	}
    22  
    23  	fmt.Println("writing entries...")
    24  	// write entries
    25  	for i := 0; i < count; i++ {
    26  		// make entry
    27  		e := &Entry{
    28  			Key:   makeData("key", i),
    29  			Value: makeData("value", i),
    30  		}
    31  		// add checksum
    32  		e.CRC = checksum(append(e.Key, e.Value...))
    33  
    34  		// write entry
    35  		offset, err := c.put(e)
    36  		if err != nil {
    37  			t.Errorf("put: %v\n", err)
    38  		}
    39  
    40  		// add offset to set
    41  		offsets = append(offsets, offset)
    42  	}
    43  
    44  	fmt.Println("closing commit log...")
    45  	// close commit log
    46  	err = c.close()
    47  	if err != nil {
    48  		t.Errorf("close: %v\n", err)
    49  	}
    50  
    51  	fmt.Println("opening commit log, again...")
    52  	// open commit log
    53  	c, err = openCommitLog(baseDir, false)
    54  	if err != nil {
    55  		t.Errorf("open: %v\n", err)
    56  	}
    57  
    58  	fmt.Println("reading entries at...")
    59  	// read entries at
    60  	for i := range offsets {
    61  		e, err := c.get(offsets[i])
    62  		if err != nil {
    63  			t.Errorf("reading entry at (%d): %v\n", err, offsets[i])
    64  		}
    65  		fmt.Printf("offset: %d, %s\n", offsets[i], e)
    66  	}
    67  
    68  	fmt.Println("closing commit log, again...")
    69  	// close commit log
    70  	err = c.close()
    71  	if err != nil {
    72  		t.Errorf("close: %v\n", err)
    73  	}
    74  }
    75  
    76  func TestCommitLogAppend(t *testing.T) {
    77  
    78  	// offsets for later
    79  	var offsets []int64
    80  
    81  	fmt.Println("opening commit log...")
    82  	// open commit log
    83  	c, err := openCommitLog(baseDir, false)
    84  	if err != nil {
    85  		t.Errorf("open: %v\n", err)
    86  	}
    87  
    88  	fmt.Println("writing entries...")
    89  	// write entries
    90  	for i := count; i < count*2; i++ {
    91  		// make entry
    92  		e := &Entry{
    93  			Key:   makeData("key", i),
    94  			Value: makeData("value", i),
    95  		}
    96  		// add checksum
    97  		e.CRC = checksum(append(e.Key, e.Value...))
    98  
    99  		// write entry
   100  		offset, err := c.put(e)
   101  		if err != nil {
   102  			t.Errorf("put: %v\n", err)
   103  		}
   104  
   105  		// add offset to set
   106  		offsets = append(offsets, offset)
   107  	}
   108  
   109  	fmt.Println("closing commit log...")
   110  	// close commit log
   111  	err = c.close()
   112  	if err != nil {
   113  		t.Errorf("close: %v\n", err)
   114  	}
   115  
   116  	fmt.Println("opening commit log, again...")
   117  	// open commit log
   118  	c, err = openCommitLog(baseDir, false)
   119  	if err != nil {
   120  		t.Errorf("open: %v\n", err)
   121  	}
   122  
   123  	fmt.Println("reading entries at...")
   124  	// read entries at
   125  	for i := range offsets {
   126  		e, err := c.get(offsets[i])
   127  		if err != nil {
   128  			t.Errorf("reading entry at (%d): %v\n", err, offsets[i])
   129  		}
   130  		fmt.Printf("offset: %d, %s\n", offsets[i], e)
   131  	}
   132  
   133  	fmt.Println("closing commit log, again...")
   134  	// close commit log
   135  	err = c.close()
   136  	if err != nil {
   137  		t.Errorf("close: %v\n", err)
   138  	}
   139  }
   140  
   141  func TestCommitLogReset(t *testing.T) {
   142  
   143  	// offsets for later
   144  	var offsets []int64
   145  
   146  	fmt.Println("opening commit log...")
   147  	// open commit log
   148  	c, err := openCommitLog(baseDir, false)
   149  	if err != nil {
   150  		t.Errorf("open: %v\n", err)
   151  	}
   152  
   153  	fmt.Println("cycling...")
   154  	// cycling
   155  	err = c.cycle()
   156  	if err != nil {
   157  		t.Errorf("cycling: %v\n", err)
   158  	}
   159  
   160  	fmt.Println("writing entries...")
   161  	// write entries
   162  	for i := count / 2; i < count; i++ {
   163  		// make entry
   164  		e := &Entry{
   165  			Key:   makeData("key", i),
   166  			Value: makeData("value", i),
   167  		}
   168  		// add checksum
   169  		e.CRC = checksum(append(e.Key, e.Value...))
   170  
   171  		// write entry
   172  		offset, err := c.put(e)
   173  		if err != nil {
   174  			t.Errorf("put: %v\n", err)
   175  		}
   176  
   177  		// add offset to set
   178  		offsets = append(offsets, offset)
   179  	}
   180  
   181  	fmt.Println("closing commit log...")
   182  	// close commit log
   183  	err = c.close()
   184  	if err != nil {
   185  		t.Errorf("close: %v\n", err)
   186  	}
   187  
   188  	fmt.Println("opening commit log, again...")
   189  	// open commit log
   190  	c, err = openCommitLog(baseDir, false)
   191  	if err != nil {
   192  		t.Errorf("open: %v\n", err)
   193  	}
   194  
   195  	fmt.Println("reading entries at...")
   196  	// read entries at
   197  	for i := range offsets {
   198  		e, err := c.get(offsets[i])
   199  		if err != nil {
   200  			t.Errorf("reading entry at (%d): %v\n", err, offsets[i])
   201  		}
   202  		fmt.Printf("offset: %d, %s\n", offsets[i], e)
   203  	}
   204  
   205  	fmt.Println("cycling again...")
   206  	// cycling
   207  	err = c.cycle()
   208  	if err != nil {
   209  		t.Errorf("cycling: %v\n", err)
   210  	}
   211  
   212  	fmt.Println("closing commit log, again...")
   213  	// close commit log
   214  	err = c.close()
   215  	if err != nil {
   216  		t.Errorf("close: %v\n", err)
   217  	}
   218  }
   219  
   220  func TestCommitLogTestLargeLoad(t *testing.T) {
   221  
   222  	// offsets for later
   223  	var offsets []int64
   224  
   225  	fmt.Println("opening commit log...")
   226  	// open commit log
   227  	c, err := openCommitLog(baseDir, false)
   228  	if err != nil {
   229  		t.Errorf("open: %v\n", err)
   230  	}
   231  
   232  	fmt.Println("cycling...")
   233  	// cycling
   234  	err = c.cycle()
   235  	if err != nil {
   236  		t.Errorf("cycling: %v\n", err)
   237  	}
   238  
   239  	fmt.Println("syncing...")
   240  	// syncing
   241  	err = c.sync()
   242  	if err != nil {
   243  		t.Errorf("syncing: %v\n", err)
   244  	}
   245  
   246  	fmt.Println("writing entries...")
   247  	// write entries
   248  	for i := 0; i < count*count; i++ {
   249  		// make entry
   250  		e := &Entry{
   251  			Key:   makeData("key", i),
   252  			Value: makeData("value", i),
   253  		}
   254  		// add checksum
   255  		e.CRC = checksum(append(e.Key, e.Value...))
   256  
   257  		// write entry
   258  		offset, err := c.put(e)
   259  		if err != nil {
   260  			t.Errorf("put: %v\n", err)
   261  		}
   262  
   263  		// add offset to set
   264  		offsets = append(offsets, offset)
   265  	}
   266  
   267  	fmt.Println("closing commit log...")
   268  	// close commit log
   269  	err = c.close()
   270  	if err != nil {
   271  		t.Errorf("close: %v\n", err)
   272  	}
   273  
   274  	fmt.Println("opening commit log, again...")
   275  	// open commit log
   276  	c, err = openCommitLog(baseDir, false)
   277  	if err != nil {
   278  		t.Errorf("open: %v\n", err)
   279  	}
   280  
   281  	fmt.Println("reading entries at...")
   282  	// read entries at
   283  	j := 0
   284  	for i := range offsets {
   285  		e, err := c.get(offsets[i])
   286  		if err != nil {
   287  			t.Errorf("reading entry at (%d): %v\n", err, offsets[i])
   288  		}
   289  		if i%500 == 0 {
   290  			fmt.Printf("offset: %d, %s\n", offsets[i], e)
   291  		}
   292  		j++
   293  	}
   294  
   295  	fmt.Printf(">>> read %d entries\n", j)
   296  
   297  	fmt.Println("closing commit log, again...")
   298  	// close commit log
   299  	err = c.close()
   300  	if err != nil {
   301  		t.Errorf("close: %v\n", err)
   302  	}
   303  }
   304  
   305  func TestCommitLogScanNext(t *testing.T) {
   306  
   307  	fmt.Println("opening commit log...")
   308  	// open commit log
   309  	c, err := openCommitLog(baseDir, false)
   310  	if err != nil {
   311  		t.Errorf("open: %v\n", err)
   312  	}
   313  
   314  	fmt.Println("writing entries...")
   315  	// write entries
   316  	for i := count / 2; i < count; i++ {
   317  		// make entry
   318  		e := &Entry{
   319  			Key:   makeData("key", i),
   320  			Value: makeData("value", i),
   321  		}
   322  		// add checksum
   323  		e.CRC = checksum(append(e.Key, e.Value...))
   324  
   325  		// write entry
   326  		_, err := c.put(e)
   327  		if err != nil {
   328  			t.Errorf("put: %v\n", err)
   329  		}
   330  	}
   331  
   332  	fmt.Println("closing commit log...")
   333  	// close commit log
   334  	err = c.close()
   335  	if err != nil {
   336  		t.Errorf("close: %v\n", err)
   337  	}
   338  
   339  	fmt.Println("opening commit log, again...")
   340  	// open commit log
   341  	c, err = openCommitLog(baseDir, false)
   342  	if err != nil {
   343  		t.Errorf("open: %v\n", err)
   344  	}
   345  
   346  	fmt.Println("reading entries using scan next...")
   347  	err = c.scan(func(e *Entry) bool {
   348  		fmt.Printf("%s\n", e)
   349  		return true
   350  	})
   351  	if err != nil {
   352  		t.Errorf("next: %v\n", err)
   353  	}
   354  
   355  	fmt.Println("cycling...")
   356  	// cycling
   357  	err = c.cycle()
   358  	if err != nil {
   359  		t.Errorf("cycling: %v\n", err)
   360  	}
   361  
   362  	fmt.Println("closing commit log")
   363  	// close commit log
   364  	err = c.close()
   365  	if err != nil {
   366  		t.Errorf("close: %v\n", err)
   367  	}
   368  }