github.com/authzed/spicedb@v1.32.1-0.20240520085336-ebda56537386/internal/datastore/proxy/schemacaching/estimatedsize_test.go (about)

     1  package schemacaching
     2  
     3  import (
     4  	"context"
     5  	"os"
     6  	"path"
     7  	"path/filepath"
     8  	"runtime"
     9  	"runtime/debug"
    10  	"strings"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/stretchr/testify/require"
    15  
    16  	"github.com/authzed/spicedb/internal/datastore/memdb"
    17  	core "github.com/authzed/spicedb/pkg/proto/core/v1"
    18  	"github.com/authzed/spicedb/pkg/validationfile"
    19  )
    20  
    21  const retryCount = 5
    22  
    23  func TestEstimatedDefinitionSizes(t *testing.T) {
    24  	// Load all consistency and benchmark YAMLs to get a set of sample namespace
    25  	// definitions for testing.
    26  	_, filename, _, _ := runtime.Caller(0)
    27  	integrationTestDirectoryPath := path.Join(path.Dir(filename), "../../../services/integrationtesting")
    28  
    29  	consistencyTestFiles := []string{}
    30  	err := filepath.Walk(integrationTestDirectoryPath, func(path string, info os.FileInfo, err error) error {
    31  		if info == nil || info.IsDir() {
    32  			return nil
    33  		}
    34  
    35  		if strings.HasSuffix(info.Name(), ".yaml") {
    36  			consistencyTestFiles = append(consistencyTestFiles, path)
    37  		}
    38  
    39  		return nil
    40  	})
    41  	require.NoError(t, err)
    42  	require.NotEqual(t, 0, len(consistencyTestFiles))
    43  
    44  	for _, filePath := range consistencyTestFiles {
    45  		filePath := filePath
    46  		t.Run(path.Base(filePath), func(t *testing.T) {
    47  			ds, err := memdb.NewMemdbDatastore(0, 1*time.Second, memdb.DisableGC)
    48  			require.NoError(t, err)
    49  
    50  			fullyResolved, _, err := validationfile.PopulateFromFiles(context.Background(), ds, []string{filePath})
    51  			require.NoError(t, err)
    52  
    53  			for _, nsDef := range fullyResolved.NamespaceDefinitions {
    54  				nsDef := nsDef
    55  				t.Run("namespace "+nsDef.Name, func(t *testing.T) {
    56  					serialized, _ := nsDef.MarshalVT()
    57  					sizevt := nsDef.SizeVT()
    58  					estimated := estimatedNamespaceDefinitionSize(sizevt)
    59  
    60  					succeeded := false
    61  					var used uint64
    62  					for i := 0; i < retryCount; i++ {
    63  						runtime.GC()
    64  						debug.FreeOSMemory()
    65  
    66  						// Calculate the memory used for deserializing the namespace definition.
    67  						var m1, m2 runtime.MemStats
    68  						runtime.ReadMemStats(&m1)
    69  
    70  						var def core.NamespaceDefinition
    71  						require.NoError(t, def.UnmarshalVT(serialized))
    72  
    73  						runtime.ReadMemStats(&m2)
    74  						used := m2.TotalAlloc - m1.TotalAlloc
    75  
    76  						// Ensure the memory used is less than the SizeVT * the multiplier.
    77  						if used <= uint64(estimated) {
    78  							succeeded = true
    79  							break
    80  						}
    81  					}
    82  
    83  					require.True(t, succeeded, "found size %d, for with SizeVT: %d", used, sizevt)
    84  				})
    85  			}
    86  
    87  			for _, caveatDef := range fullyResolved.CaveatDefinitions {
    88  				caveatDef := caveatDef
    89  				t.Run("caveat "+caveatDef.Name, func(t *testing.T) {
    90  					serialized, _ := caveatDef.MarshalVT()
    91  					sizevt := caveatDef.SizeVT()
    92  					estimated := estimatedCaveatDefinitionSize(sizevt)
    93  
    94  					succeeded := false
    95  					var used uint64
    96  					for i := 0; i < retryCount; i++ {
    97  						runtime.GC()
    98  						debug.FreeOSMemory()
    99  
   100  						// Calculate the memory used for deserializing the caveat definition.
   101  						var m1, m2 runtime.MemStats
   102  						runtime.ReadMemStats(&m1)
   103  
   104  						var def core.CaveatDefinition
   105  						require.NoError(t, def.UnmarshalVT(serialized))
   106  
   107  						runtime.ReadMemStats(&m2)
   108  						used := m2.TotalAlloc - m1.TotalAlloc
   109  
   110  						// Ensure the memory used is less than the SizeVT * the multiplier.
   111  						if used <= uint64(estimated) {
   112  							succeeded = true
   113  							break
   114  						}
   115  					}
   116  
   117  					require.True(t, succeeded, "found size %d, for with SizeVT: %d", used, sizevt)
   118  				})
   119  			}
   120  		})
   121  	}
   122  }