github.com/matrixorigin/matrixone@v1.2.0/pkg/hakeeper/checkers/logservice/parse_test.go (about)

     1  // Copyright 2021 - 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package logservice
    16  
    17  import (
    18  	"fmt"
    19  	"sort"
    20  	"testing"
    21  	"time"
    22  
    23  	"github.com/matrixorigin/matrixone/pkg/hakeeper"
    24  	pb "github.com/matrixorigin/matrixone/pkg/pb/logservice"
    25  	"github.com/matrixorigin/matrixone/pkg/pb/metadata"
    26  	"github.com/stretchr/testify/assert"
    27  )
    28  
    29  func TestFixedLogShardInfo(t *testing.T) {
    30  	cases := []struct {
    31  		desc string
    32  
    33  		record        metadata.LogShardRecord
    34  		info          pb.LogShardInfo
    35  		expiredStores []string
    36  
    37  		expected *fixingShard
    38  	}{
    39  		{
    40  			desc: "normal case",
    41  			record: metadata.LogShardRecord{
    42  				ShardID:          1,
    43  				NumberOfReplicas: 3,
    44  			},
    45  			info: pb.LogShardInfo{
    46  				ShardID:  1,
    47  				Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
    48  			},
    49  			expected: &fixingShard{
    50  				shardID:  1,
    51  				replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
    52  				toAdd:    0,
    53  			},
    54  		},
    55  		{
    56  			desc: "shard 1 has 2 replicas, which expected to be 3",
    57  			record: metadata.LogShardRecord{
    58  				ShardID:          1,
    59  				NumberOfReplicas: 3,
    60  			},
    61  			info: pb.LogShardInfo{
    62  				ShardID:  1,
    63  				Replicas: map[uint64]string{1: "a", 2: "b"},
    64  			},
    65  			expected: &fixingShard{
    66  				shardID:  1,
    67  				replicas: map[uint64]string{1: "a", 2: "b"},
    68  				toAdd:    1,
    69  			},
    70  		},
    71  		{
    72  			desc: "shard 1 has 3 replicas, which expected to be 1, and leader is the last to be removed",
    73  			record: metadata.LogShardRecord{
    74  				ShardID:          1,
    75  				NumberOfReplicas: 1,
    76  			},
    77  			info: pb.LogShardInfo{
    78  				ShardID:  1,
    79  				Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
    80  				LeaderID: 2,
    81  			},
    82  			expected: &fixingShard{
    83  				shardID:  1,
    84  				replicas: map[uint64]string{2: "b"},
    85  				toAdd:    0,
    86  			},
    87  		},
    88  	}
    89  
    90  	for _, c := range cases {
    91  		output := fixedLogShardInfo(c.record, c.info, c.expiredStores)
    92  		assert.Equal(t, c.expected, output)
    93  	}
    94  }
    95  
    96  func TestCollectStats(t *testing.T) {
    97  	cases := []struct {
    98  		desc     string
    99  		cluster  pb.ClusterInfo
   100  		infos    pb.LogState
   101  		expired  []string
   102  		expected *stats
   103  	}{
   104  		{
   105  			desc: "Normal case",
   106  			cluster: pb.ClusterInfo{
   107  				TNShards: nil,
   108  				LogShards: []metadata.LogShardRecord{{
   109  					ShardID:          1,
   110  					NumberOfReplicas: 3}}},
   111  			infos: pb.LogState{
   112  				Shards: map[uint64]pb.LogShardInfo{
   113  					1: {
   114  						ShardID:  1,
   115  						Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   116  						Epoch:    1,
   117  						LeaderID: 0,
   118  						Term:     0,
   119  					}},
   120  				Stores: map[string]pb.LogStoreInfo{
   121  					"a": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   122  						Replicas: []pb.LogReplicaInfo{{
   123  							LogShardInfo: pb.LogShardInfo{
   124  								ShardID:  1,
   125  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   126  								Epoch:    1,
   127  								LeaderID: 0,
   128  								Term:     0,
   129  							}}}},
   130  					"b": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   131  						Replicas: []pb.LogReplicaInfo{{
   132  							LogShardInfo: pb.LogShardInfo{
   133  								ShardID:  1,
   134  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   135  								Epoch:    1,
   136  								LeaderID: 0,
   137  								Term:     0,
   138  							}}}},
   139  					"c": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   140  						Replicas: []pb.LogReplicaInfo{{
   141  							LogShardInfo: pb.LogShardInfo{
   142  								ShardID:  1,
   143  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   144  								Epoch:    1,
   145  								LeaderID: 0,
   146  								Term:     0}}}},
   147  				},
   148  			},
   149  			expired:  []string{},
   150  			expected: &stats{toRemove: map[uint64][]replica{}, toAdd: map[uint64]uint32{}}},
   151  		{
   152  			desc: "Shard 1 has only 2 replicas, which is expected as 3.",
   153  			cluster: pb.ClusterInfo{
   154  				TNShards: nil,
   155  				LogShards: []metadata.LogShardRecord{{
   156  					ShardID:          1,
   157  					NumberOfReplicas: 3}}},
   158  			infos: pb.LogState{
   159  				Shards: map[uint64]pb.LogShardInfo{1: {
   160  					ShardID:  1,
   161  					Replicas: map[uint64]string{1: "a", 2: "b"},
   162  					Epoch:    1,
   163  					LeaderID: 0,
   164  					Term:     0}},
   165  				Stores: map[string]pb.LogStoreInfo{
   166  					"a": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   167  						Replicas: []pb.LogReplicaInfo{{
   168  							LogShardInfo: pb.LogShardInfo{
   169  								ShardID:  1,
   170  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   171  								Epoch:    1,
   172  								LeaderID: 0,
   173  								Term:     0}}}},
   174  					"b": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   175  						Replicas: []pb.LogReplicaInfo{{
   176  							LogShardInfo: pb.LogShardInfo{
   177  								ShardID:  1,
   178  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   179  								Epoch:    1,
   180  								LeaderID: 0,
   181  								Term:     0}}}},
   182  				},
   183  			},
   184  			expired:  []string{},
   185  			expected: &stats{toRemove: map[uint64][]replica{}, toAdd: map[uint64]uint32{1: 1}}},
   186  		{
   187  			desc: "replica on Store c is not started.",
   188  			cluster: pb.ClusterInfo{
   189  				LogShards: []metadata.LogShardRecord{{
   190  					ShardID:          1,
   191  					NumberOfReplicas: 3}}},
   192  			infos: pb.LogState{
   193  				Shards: map[uint64]pb.LogShardInfo{
   194  					1: {
   195  						ShardID:  1,
   196  						Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   197  						Epoch:    1,
   198  						LeaderID: 0,
   199  						Term:     0}},
   200  				Stores: map[string]pb.LogStoreInfo{
   201  					"a": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   202  						Replicas: []pb.LogReplicaInfo{{
   203  							LogShardInfo: pb.LogShardInfo{
   204  								ShardID:  1,
   205  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   206  								Epoch:    1,
   207  								LeaderID: 0,
   208  								Term:     0}}}},
   209  					"b": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   210  						Replicas: []pb.LogReplicaInfo{{
   211  							LogShardInfo: pb.LogShardInfo{
   212  								ShardID:  1,
   213  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   214  								Epoch:    1,
   215  								LeaderID: 0,
   216  								Term:     0}}}},
   217  					"c": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   218  						Replicas: []pb.LogReplicaInfo{}},
   219  				},
   220  			},
   221  			expected: &stats{toStart: []replica{{"c", 1, 0, 3}},
   222  				toRemove: map[uint64][]replica{}, toAdd: map[uint64]uint32{}}},
   223  		{
   224  			desc: "replica on Store d is a zombie.",
   225  			cluster: pb.ClusterInfo{
   226  				LogShards: []metadata.LogShardRecord{{
   227  					ShardID:          1,
   228  					NumberOfReplicas: 3}}},
   229  			infos: pb.LogState{
   230  				Shards: map[uint64]pb.LogShardInfo{
   231  					1: {
   232  						ShardID:  1,
   233  						Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   234  						Epoch:    1,
   235  						LeaderID: 0,
   236  						Term:     0}},
   237  				Stores: map[string]pb.LogStoreInfo{
   238  					"a": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   239  						Replicas: []pb.LogReplicaInfo{{
   240  							LogShardInfo: pb.LogShardInfo{
   241  								ShardID:  1,
   242  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   243  								Epoch:    1,
   244  								LeaderID: 0,
   245  								Term:     0}}}},
   246  					"b": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   247  						Replicas: []pb.LogReplicaInfo{{
   248  							LogShardInfo: pb.LogShardInfo{
   249  								ShardID:  1,
   250  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   251  								Epoch:    1,
   252  								LeaderID: 0,
   253  								Term:     0}}}},
   254  					"c": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   255  						Replicas: []pb.LogReplicaInfo{{
   256  							LogShardInfo: pb.LogShardInfo{
   257  								ShardID:  1,
   258  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   259  								Epoch:    1,
   260  								LeaderID: 0,
   261  								Term:     0}}}},
   262  					"d": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   263  						Replicas: []pb.LogReplicaInfo{{
   264  							LogShardInfo: pb.LogShardInfo{
   265  								ShardID:  1,
   266  								Replicas: map[uint64]string{1: "a", 2: "b", 4: "d"},
   267  								Epoch:    0,
   268  								LeaderID: 0,
   269  								Term:     0}}}},
   270  				},
   271  			},
   272  			expected: &stats{zombies: []replica{{"d", 1, 0, 0}},
   273  				toRemove: map[uint64][]replica{}, toAdd: map[uint64]uint32{}}},
   274  		{
   275  			desc: "do not remove replica d if it is in LogShardInfo.Replicas, despite it's epoch is small.",
   276  			cluster: pb.ClusterInfo{
   277  				TNShards: nil,
   278  				LogShards: []metadata.LogShardRecord{{
   279  					ShardID:          1,
   280  					NumberOfReplicas: 4}}},
   281  			infos: pb.LogState{
   282  				Shards: map[uint64]pb.LogShardInfo{
   283  					1: {ShardID: 1,
   284  						Replicas: map[uint64]string{1: "a", 2: "b", 3: "c", 4: "d"},
   285  						Epoch:    1}},
   286  				Stores: map[string]pb.LogStoreInfo{
   287  					"a": {Replicas: []pb.LogReplicaInfo{{
   288  						LogShardInfo: pb.LogShardInfo{
   289  							ShardID:  1,
   290  							Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   291  							Epoch:    1},
   292  						ReplicaID: 1,
   293  					}}},
   294  					"b": {Replicas: []pb.LogReplicaInfo{{
   295  						LogShardInfo: pb.LogShardInfo{
   296  							ShardID:  1,
   297  							Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   298  							Epoch:    1},
   299  						ReplicaID: 2,
   300  					}}},
   301  					"c": {Replicas: []pb.LogReplicaInfo{{
   302  						LogShardInfo: pb.LogShardInfo{
   303  							ShardID:  1,
   304  							Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   305  							Epoch:    1},
   306  						ReplicaID: 3,
   307  					}}},
   308  					"d": {Replicas: []pb.LogReplicaInfo{{
   309  						LogShardInfo: pb.LogShardInfo{
   310  							ShardID:  1,
   311  							Replicas: map[uint64]string{1: "a", 2: "b", 4: "d"}},
   312  						ReplicaID: 4,
   313  					}}},
   314  				},
   315  			},
   316  			expected: &stats{
   317  				toRemove: map[uint64][]replica{},
   318  				toAdd:    map[uint64]uint32{}}},
   319  		{
   320  			desc: "Shard 1 has 4 replicas, which is expected as 3.",
   321  			cluster: pb.ClusterInfo{
   322  				TNShards: nil,
   323  				LogShards: []metadata.LogShardRecord{{
   324  					ShardID:          1,
   325  					NumberOfReplicas: 3}}},
   326  			infos: pb.LogState{
   327  				Shards: map[uint64]pb.LogShardInfo{
   328  					1: {
   329  						ShardID:  1,
   330  						Replicas: map[uint64]string{1: "a", 2: "b", 3: "c", 4: "d"},
   331  						Epoch:    1,
   332  						LeaderID: 0,
   333  						Term:     0}},
   334  				Stores: map[string]pb.LogStoreInfo{
   335  					"a": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   336  						Replicas: []pb.LogReplicaInfo{{
   337  							LogShardInfo: pb.LogShardInfo{
   338  								ShardID:  1,
   339  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c", 4: "d"},
   340  								Epoch:    1,
   341  								LeaderID: 0,
   342  								Term:     0}}}},
   343  					"b": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   344  						Replicas: []pb.LogReplicaInfo{{
   345  							LogShardInfo: pb.LogShardInfo{
   346  								ShardID:  1,
   347  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c", 4: "d"},
   348  								Epoch:    1,
   349  								LeaderID: 0,
   350  								Term:     0}}}},
   351  					"c": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   352  						Replicas: []pb.LogReplicaInfo{{
   353  							LogShardInfo: pb.LogShardInfo{
   354  								ShardID:  1,
   355  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c", 4: "d"},
   356  								Epoch:    1,
   357  								LeaderID: 0,
   358  								Term:     0}}}},
   359  					"d": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   360  						Replicas: []pb.LogReplicaInfo{{
   361  							LogShardInfo: pb.LogShardInfo{
   362  								ShardID:  1,
   363  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c", 4: "d"},
   364  								Epoch:    1,
   365  								LeaderID: 0,
   366  								Term:     0}}}},
   367  				},
   368  			},
   369  			expected: &stats{toRemove: map[uint64][]replica{1: {{"a", 1, 0, 1}}},
   370  				toAdd: map[uint64]uint32{},
   371  			},
   372  		},
   373  		{
   374  			desc: "Store a is expired",
   375  			cluster: pb.ClusterInfo{
   376  				LogShards: []metadata.LogShardRecord{{
   377  					ShardID:          1,
   378  					NumberOfReplicas: 3}}},
   379  			infos: pb.LogState{
   380  				Shards: map[uint64]pb.LogShardInfo{
   381  					1: {
   382  						ShardID:  1,
   383  						Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   384  						Epoch:    1,
   385  						LeaderID: 0,
   386  						Term:     0}},
   387  				Stores: map[string]pb.LogStoreInfo{
   388  					"a": {Tick: 0, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   389  						Replicas: []pb.LogReplicaInfo{{
   390  							LogShardInfo: pb.LogShardInfo{
   391  								ShardID:  1,
   392  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   393  								Epoch:    1,
   394  								LeaderID: 0,
   395  								Term:     0}}}},
   396  					"b": {Tick: 999999999, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   397  						Replicas: []pb.LogReplicaInfo{{
   398  							LogShardInfo: pb.LogShardInfo{
   399  								ShardID:  1,
   400  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   401  								Epoch:    1,
   402  								LeaderID: 0,
   403  								Term:     0}}}},
   404  					"c": {Tick: 999999999, RaftAddress: "", ServiceAddress: "", GossipAddress: "",
   405  						Replicas: []pb.LogReplicaInfo{{
   406  							LogShardInfo: pb.LogShardInfo{
   407  								ShardID:  1,
   408  								Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   409  								Epoch:    1,
   410  								LeaderID: 0,
   411  								Term:     0}}}}},
   412  			},
   413  			expired: []string{"a"},
   414  			expected: &stats{
   415  				toRemove: map[uint64][]replica{1: {{uuid: "a", shardID: 1, replicaID: 1}}},
   416  				toAdd:    map[uint64]uint32{},
   417  			},
   418  		},
   419  	}
   420  
   421  	for i, c := range cases {
   422  		fmt.Printf("case %v: %s\n", i, c.desc)
   423  		stat := parseLogShards(c.cluster, c.infos, c.expired)
   424  		assert.Equal(t, c.expected, stat)
   425  	}
   426  }
   427  
   428  func TestCollectStore(t *testing.T) {
   429  	cases := []struct {
   430  		desc     string
   431  		cluster  pb.ClusterInfo
   432  		infos    pb.LogState
   433  		tick     uint64
   434  		expected []string
   435  	}{
   436  		{
   437  			desc: "no expired stores",
   438  			cluster: pb.ClusterInfo{
   439  				TNShards:  nil,
   440  				LogShards: []metadata.LogShardRecord{{ShardID: 1, NumberOfReplicas: 3}},
   441  			},
   442  			infos: pb.LogState{
   443  				Shards: map[uint64]pb.LogShardInfo{1: {
   444  					ShardID:  1,
   445  					Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   446  					Epoch:    1,
   447  				}},
   448  				Stores: map[string]pb.LogStoreInfo{
   449  					"a": {Tick: 0},
   450  					"b": {Tick: 0},
   451  					"c": {Tick: 0},
   452  				},
   453  			},
   454  			tick:     0,
   455  			expected: []string{},
   456  		},
   457  		{
   458  			desc: "store b expired",
   459  			cluster: pb.ClusterInfo{
   460  				TNShards:  nil,
   461  				LogShards: []metadata.LogShardRecord{{ShardID: 1, NumberOfReplicas: 3}},
   462  			},
   463  			infos: pb.LogState{
   464  				Shards: map[uint64]pb.LogShardInfo{1: {
   465  					ShardID:  1,
   466  					Replicas: map[uint64]string{1: "a", 2: "b", 3: "c"},
   467  					Epoch:    1,
   468  				}},
   469  				Stores: map[string]pb.LogStoreInfo{
   470  					"a": {Tick: expiredTick + 1},
   471  					"b": {Tick: 0},
   472  					"c": {Tick: expiredTick + 1},
   473  				},
   474  			},
   475  			tick:     expiredTick + 1,
   476  			expected: []string{"b"},
   477  		},
   478  	}
   479  	for i, c := range cases {
   480  		fmt.Printf("case %v: %s\n", i, c.desc)
   481  		cfg := hakeeper.Config{}
   482  		cfg.Fill()
   483  		working, expired := parseLogStores(cfg, c.infos, c.tick)
   484  		sort.Slice(working, func(i, j int) bool {
   485  			return working[i] < working[j]
   486  		})
   487  		sort.Slice(expired, func(i, j int) bool {
   488  			return expired[i] < expired[j]
   489  		})
   490  		assert.Equal(t, c.expected, expired)
   491  		cfg1 := hakeeper.Config{LogStoreTimeout: time.Hour}
   492  		cfg1.Fill()
   493  		working, expired = parseLogStores(cfg1, c.infos, c.tick)
   494  		assert.Equal(t, []string{}, expired)
   495  	}
   496  }