github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/stats_test.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package nbs
    23  
    24  import (
    25  	"context"
    26  	"io/ioutil"
    27  	"os"
    28  	"testing"
    29  
    30  	"github.com/stretchr/testify/assert"
    31  	"github.com/stretchr/testify/require"
    32  
    33  	"github.com/dolthub/dolt/go/store/chunks"
    34  	"github.com/dolthub/dolt/go/store/constants"
    35  	"github.com/dolthub/dolt/go/store/hash"
    36  )
    37  
    38  func TestStats(t *testing.T) {
    39  	assert := assert.New(t)
    40  
    41  	stats := func(store *NomsBlockStore) Stats {
    42  		return store.Stats().(Stats)
    43  	}
    44  
    45  	dir, err := ioutil.TempDir("", "")
    46  	require.NoError(t, err)
    47  	store, err := NewLocalStore(context.Background(), constants.FormatDefaultString, dir, testMemTableSize)
    48  	require.NoError(t, err)
    49  
    50  	assert.EqualValues(1, stats(store).OpenLatency.Samples())
    51  
    52  	// Opening a new store will still incur some read IO, to discover that the manifest doesn't exist
    53  	assert.EqualValues(1, stats(store).ReadManifestLatency.Samples())
    54  
    55  	i1, i2, i3, i4, i5 := []byte("abc"), []byte("def"), []byte("ghi"), []byte("jkl"), []byte("mno")
    56  
    57  	c1, c2, c3, c4, c5 := chunks.NewChunk(i1), chunks.NewChunk(i2), chunks.NewChunk(i3), chunks.NewChunk(i4), chunks.NewChunk(i5)
    58  
    59  	// These just go to mem table, only operation stats
    60  	err = store.Put(context.Background(), c1)
    61  	require.NoError(t, err)
    62  	err = store.Put(context.Background(), c2)
    63  	require.NoError(t, err)
    64  	err = store.Put(context.Background(), c3)
    65  	require.NoError(t, err)
    66  	assert.Equal(uint64(3), stats(store).PutLatency.Samples())
    67  	assert.Equal(uint64(0), stats(store).PersistLatency.Samples())
    68  
    69  	assert.True(store.Has(context.Background(), c1.Hash()))
    70  	assert.True(store.Has(context.Background(), c2.Hash()))
    71  	assert.True(store.Has(context.Background(), c3.Hash()))
    72  	assert.Equal(uint64(3), stats(store).HasLatency.Samples())
    73  	assert.Equal(uint64(3), stats(store).AddressesPerHas.Sum())
    74  
    75  	c, err := store.Get(context.Background(), c1.Hash())
    76  	require.NoError(t, err)
    77  	assert.False(c.IsEmpty())
    78  	c, err = store.Get(context.Background(), c2.Hash())
    79  	require.NoError(t, err)
    80  	assert.False(c.IsEmpty())
    81  	c, err = store.Get(context.Background(), c3.Hash())
    82  	require.NoError(t, err)
    83  	assert.False(c.IsEmpty())
    84  	assert.Equal(uint64(3), stats(store).GetLatency.Samples())
    85  	assert.Equal(uint64(0), stats(store).FileReadLatency.Samples())
    86  	assert.Equal(uint64(3), stats(store).ChunksPerGet.Sum())
    87  
    88  	h, err := store.Root(context.Background())
    89  	require.NoError(t, err)
    90  	_, err = store.Commit(context.Background(), h, h)
    91  	require.NoError(t, err)
    92  
    93  	// Commit will update the manifest
    94  	assert.EqualValues(1, stats(store).WriteManifestLatency.Samples())
    95  	assert.EqualValues(1, stats(store).CommitLatency.Samples())
    96  
    97  	// Now we have write IO
    98  	assert.Equal(uint64(1), stats(store).PersistLatency.Samples())
    99  	assert.Equal(uint64(3), stats(store).ChunksPerPersist.Sum())
   100  	assert.Equal(uint64(131), stats(store).BytesPerPersist.Sum())
   101  
   102  	// Now some gets that will incur read IO
   103  	_, err = store.Get(context.Background(), c1.Hash())
   104  	require.NoError(t, err)
   105  	_, err = store.Get(context.Background(), c2.Hash())
   106  	require.NoError(t, err)
   107  	_, err = store.Get(context.Background(), c3.Hash())
   108  	require.NoError(t, err)
   109  	assert.Equal(uint64(3), stats(store).FileReadLatency.Samples())
   110  	assert.Equal(uint64(27), stats(store).FileBytesPerRead.Sum())
   111  
   112  	// Try A GetMany
   113  	chnx := make([]chunks.Chunk, 3)
   114  	chnx[0] = c1
   115  	chnx[1] = c2
   116  	chnx[2] = c3
   117  	hashes := make(hash.HashSlice, len(chnx))
   118  	for i, c := range chnx {
   119  		hashes[i] = c.Hash()
   120  	}
   121  	chunkChan := make(chan *chunks.Chunk, 3)
   122  	err = store.GetMany(context.Background(), hashes.HashSet(), func(c *chunks.Chunk) { chunkChan <- c })
   123  	require.NoError(t, err)
   124  	assert.Equal(uint64(4), stats(store).FileReadLatency.Samples())
   125  	assert.Equal(uint64(54), stats(store).FileBytesPerRead.Sum())
   126  
   127  	// Force a conjoin
   128  	store.c = inlineConjoiner{2}
   129  	err = store.Put(context.Background(), c4)
   130  	require.NoError(t, err)
   131  	h, err = store.Root(context.Background())
   132  	require.NoError(t, err)
   133  	_, err = store.Commit(context.Background(), h, h)
   134  	require.NoError(t, err)
   135  
   136  	err = store.Put(context.Background(), c5)
   137  	require.NoError(t, err)
   138  	h, err = store.Root(context.Background())
   139  	require.NoError(t, err)
   140  	_, err = store.Commit(context.Background(), h, h)
   141  	require.NoError(t, err)
   142  
   143  	assert.Equal(uint64(1), stats(store).ConjoinLatency.Samples())
   144  	// TODO: Once random conjoin hack is out, test other conjoin stats
   145  
   146  	defer store.Close()
   147  	defer os.RemoveAll(dir)
   148  }