github.com/weaviate/weaviate@v1.24.6/usecases/scaler/scaler_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package scaler
    13  
    14  import (
    15  	"context"
    16  	"os"
    17  	"path"
    18  	"strconv"
    19  	"testing"
    20  
    21  	"github.com/stretchr/testify/assert"
    22  	"github.com/weaviate/weaviate/entities/backup"
    23  	"github.com/weaviate/weaviate/usecases/sharding"
    24  )
    25  
    26  func TestScalerScale(t *testing.T) {
    27  	ctx := context.Background()
    28  	t.Run("NoShardingState", func(t *testing.T) {
    29  		f := newFakeFactory()
    30  		f.ShardingState.M = nil
    31  		scaler := f.Scaler("")
    32  		old := sharding.Config{}
    33  		_, err := scaler.Scale(ctx, "C", old, 1, 2)
    34  		assert.NotNil(t, err)
    35  		assert.Contains(t, err.Error(), "no sharding state")
    36  	})
    37  	t.Run("SameReplicationFactor", func(t *testing.T) {
    38  		scaler := newFakeFactory().Scaler("")
    39  		old := sharding.Config{}
    40  		_, err := scaler.Scale(ctx, "C", old, 2, 2)
    41  		assert.Nil(t, err)
    42  	})
    43  	t.Run("ScaleInNotSupported", func(t *testing.T) {
    44  		scaler := newFakeFactory().Scaler("")
    45  		old := sharding.Config{}
    46  		_, err := scaler.Scale(ctx, "C", old, 2, 1)
    47  		assert.NotNil(t, err)
    48  		assert.Contains(t, err.Error(), "not supported")
    49  	})
    50  }
    51  
    52  func TestScalerScaleOut(t *testing.T) {
    53  	var (
    54  		dataDir = t.TempDir()
    55  		ctx     = context.Background()
    56  		cls     = "C"
    57  		old     = sharding.Config{}
    58  		bak     = backup.ClassDescriptor{
    59  			Name: "C",
    60  			Shards: []*backup.ShardDescriptor{
    61  				{
    62  					Name: "S1", Files: []string{"f1"},
    63  					PropLengthTrackerPath: "f4",
    64  					ShardVersionPath:      "f4",
    65  					DocIDCounterPath:      "f4",
    66  				},
    67  			},
    68  		}
    69  	)
    70  	for i := 1; i < 5; i++ {
    71  		file, err := os.Create(path.Join(dataDir, "f"+strconv.Itoa(i)))
    72  		assert.Nil(t, err)
    73  		file.Close()
    74  	}
    75  	t.Run("UnresolvedName", func(t *testing.T) {
    76  		f := newFakeFactory()
    77  		delete(f.NodeHostMap, "N3")
    78  		scaler := f.Scaler(dataDir)
    79  		_, err := scaler.Scale(ctx, "C", old, 1, 3)
    80  		assert.ErrorIs(t, err, ErrUnresolvedName)
    81  	})
    82  	t.Run("GetLocalShards", func(t *testing.T) {
    83  		f := newFakeFactory()
    84  		f.Source.On("ShardsBackup", anyVal, anyVal, cls, []string{"S1"}).Return(bak, errAny)
    85  
    86  		// shard doesn't exist locally
    87  		f.Client.On("IncreaseReplicationFactor", anyVal, "H3", cls, anyVal, anyVal).Return(nil)
    88  		f.Client.On("IncreaseReplicationFactor", anyVal, "H4", cls, anyVal, anyVal).Return(nil)
    89  
    90  		f.Source.On("ReleaseBackup", anyVal, anyVal, "C").Return(nil)
    91  		scaler := f.Scaler(dataDir)
    92  		_, err := scaler.Scale(ctx, "C", old, 1, 3)
    93  		assert.ErrorIs(t, err, errAny)
    94  	})
    95  
    96  	t.Run("IncreaseFactor", func(t *testing.T) {
    97  		f := newFakeFactory()
    98  		f.Source.On("ShardsBackup", anyVal, anyVal, cls, []string{"S1"}).Return(bak, nil)
    99  		// sync update to remote node N2
   100  		f.Client.On("CreateShard", anyVal, "H2", cls, "S1").Return(nil)
   101  		f.Client.On("PutFile", anyVal, "H2", cls, "S1", "f1", anyVal).Return(nil)
   102  		f.Client.On("PutFile", anyVal, "H2", cls, "S1", "f4", anyVal).Return(nil)
   103  		f.Client.On("ReInitShard", anyVal, "H2", cls, "S1").Return(nil)
   104  
   105  		// sync update to remote node N3
   106  		f.Client.On("CreateShard", anyVal, "H3", cls, "S1").Return(nil)
   107  		f.Client.On("PutFile", anyVal, "H3", cls, "S1", "f1", anyVal).Return(nil)
   108  		f.Client.On("PutFile", anyVal, "H3", cls, "S1", "f4", anyVal).Return(nil)
   109  		f.Client.On("ReInitShard", anyVal, "H3", cls, "S1").Return(nil)
   110  
   111  		// shard doesn't exist locally
   112  		f.Client.On("IncreaseReplicationFactor", anyVal, "H3", cls, anyVal, anyVal).Return(errAny)
   113  		f.Client.On("IncreaseReplicationFactor", anyVal, "H4", cls, anyVal, anyVal).Return(nil)
   114  
   115  		f.Source.On("ReleaseBackup", anyVal, anyVal, "C").Return(nil)
   116  		scaler := f.Scaler(dataDir)
   117  		_, err := scaler.Scale(ctx, "C", old, 1, 3)
   118  		assert.ErrorIs(t, err, errAny)
   119  	})
   120  
   121  	t.Run("Success", func(t *testing.T) {
   122  		f := newFakeFactory()
   123  		f.Source.On("ShardsBackup", anyVal, anyVal, cls, []string{"S1"}).Return(bak, nil)
   124  		// sync update to remote node N2
   125  		f.Client.On("CreateShard", anyVal, "H2", cls, "S1").Return(nil)
   126  		f.Client.On("PutFile", anyVal, "H2", cls, "S1", "f1", anyVal).Return(nil)
   127  		f.Client.On("PutFile", anyVal, "H2", cls, "S1", "f4", anyVal).Return(nil)
   128  		f.Client.On("ReInitShard", anyVal, "H2", cls, "S1").Return(nil)
   129  
   130  		// sync update to remote node N3
   131  		f.Client.On("CreateShard", anyVal, "H3", cls, "S1").Return(nil)
   132  		f.Client.On("PutFile", anyVal, "H3", cls, "S1", "f1", anyVal).Return(nil)
   133  		f.Client.On("PutFile", anyVal, "H3", cls, "S1", "f4", anyVal).Return(nil)
   134  		f.Client.On("ReInitShard", anyVal, "H3", cls, "S1").Return(nil)
   135  
   136  		// shard doesn't exist locally
   137  		f.Client.On("IncreaseReplicationFactor", anyVal, "H3", cls, anyVal, anyVal).Return(nil)
   138  		f.Client.On("IncreaseReplicationFactor", anyVal, "H4", cls, anyVal, anyVal).Return(nil)
   139  
   140  		f.Source.On("ReleaseBackup", anyVal, anyVal, "C").Return(nil)
   141  		scaler := f.Scaler(dataDir)
   142  		_, err := scaler.Scale(ctx, "C", old, 1, 3)
   143  		assert.Nil(t, err)
   144  	})
   145  
   146  	t.Run("ReleaseBackupAsync", func(t *testing.T) {
   147  		f := newFakeFactory()
   148  		f.Source.On("ShardsBackup", anyVal, anyVal, cls, []string{"S1"}).Return(bak, nil)
   149  		// sync update to remote node N2
   150  		f.Client.On("CreateShard", anyVal, "H2", cls, "S1").Return(nil)
   151  		f.Client.On("PutFile", anyVal, "H2", cls, "S1", "f1", anyVal).Return(nil)
   152  		f.Client.On("PutFile", anyVal, "H2", cls, "S1", "f4", anyVal).Return(nil)
   153  		f.Client.On("ReInitShard", anyVal, "H2", cls, "S1").Return(nil)
   154  
   155  		// sync update to remote node N3
   156  		f.Client.On("CreateShard", anyVal, "H3", cls, "S1").Return(nil)
   157  		f.Client.On("PutFile", anyVal, "H3", cls, "S1", "f1", anyVal).Return(nil)
   158  		f.Client.On("PutFile", anyVal, "H3", cls, "S1", "f4", anyVal).Return(nil)
   159  		f.Client.On("ReInitShard", anyVal, "H3", cls, "S1").Return(nil)
   160  
   161  		// shard doesn't exist locally
   162  		f.Client.On("IncreaseReplicationFactor", anyVal, "H3", cls, anyVal, anyVal).Return(nil)
   163  		f.Client.On("IncreaseReplicationFactor", anyVal, "H4", cls, anyVal, anyVal).Return(nil)
   164  
   165  		f.Source.On("ReleaseBackup", anyVal, anyVal, "C").Return(errAny)
   166  		scaler := f.Scaler(dataDir)
   167  		_, err := scaler.Scale(ctx, "C", old, 1, 3)
   168  		assert.Nil(t, err)
   169  	})
   170  }