go.ligato.io/vpp-agent/v3@v3.5.0/plugins/kvscheduler/notification_test.go (about)

     1  // Copyright (c) 2018 Cisco and/or its affiliates.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at:
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package kvscheduler
    16  
    17  import (
    18  	"errors"
    19  	"strings"
    20  	"testing"
    21  	"time"
    22  
    23  	. "github.com/onsi/gomega"
    24  	"google.golang.org/protobuf/proto"
    25  
    26  	. "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
    27  	"go.ligato.io/vpp-agent/v3/plugins/kvscheduler/internal/test"
    28  	"go.ligato.io/vpp-agent/v3/plugins/kvscheduler/internal/utils"
    29  	. "go.ligato.io/vpp-agent/v3/proto/ligato/kvscheduler"
    30  )
    31  
    32  const (
    33  	notificationTimeout = 2 * time.Second
    34  )
    35  
    36  func TestNotifications(t *testing.T) {
    37  	RegisterTestingT(t)
    38  
    39  	// prepare KV Scheduler
    40  	scheduler := NewPlugin(UseDeps(func(deps *Deps) {
    41  		deps.HTTPHandlers = nil
    42  	}))
    43  	err := scheduler.Init()
    44  	Expect(err).To(BeNil())
    45  	scheduler.config.EnableTxnSimulation = true
    46  
    47  	// prepare mocks
    48  	mockSB := test.NewMockSouthbound()
    49  	// -> descriptor1 (notifications):
    50  	descriptor1 := test.NewMockDescriptor(&KVDescriptor{
    51  		Name:        descriptor1Name,
    52  		NBKeyPrefix: prefixA,
    53  		KeySelector: func(key string) bool {
    54  			if !strings.HasPrefix(key, prefixA) {
    55  				return false
    56  			}
    57  			if strings.Contains(strings.TrimPrefix(key, prefixA), "/") {
    58  				return false // exclude derived values
    59  			}
    60  			return true
    61  		},
    62  		ValueTypeName: string(proto.MessageName(test.NewArrayValue())),
    63  		DerivedValues: test.ArrayValueDerBuilder,
    64  		WithMetadata:  true,
    65  	}, mockSB, 0, test.WithoutRetrieve)
    66  	// -> descriptor2:
    67  	descriptor2 := test.NewMockDescriptor(&KVDescriptor{
    68  		Name:          descriptor2Name,
    69  		NBKeyPrefix:   prefixB,
    70  		KeySelector:   prefixSelector(prefixB),
    71  		ValueTypeName: string(proto.MessageName(test.NewArrayValue())),
    72  		DerivedValues: test.ArrayValueDerBuilder,
    73  		Dependencies: func(key string, value proto.Message) []Dependency {
    74  			if key == prefixB+baseValue2 {
    75  				depKey := prefixA + baseValue1
    76  				return []Dependency{
    77  					{Label: depKey, Key: depKey},
    78  				}
    79  			}
    80  			if key == prefixB+baseValue2+"/item2" {
    81  				depKey := prefixA + baseValue1 + "/item2"
    82  				return []Dependency{
    83  					{Label: depKey, Key: depKey},
    84  				}
    85  			}
    86  			return nil
    87  		},
    88  		WithMetadata:         true,
    89  		RetrieveDependencies: []string{descriptor1Name},
    90  	}, mockSB, 0)
    91  
    92  	// register both descriptors with the scheduler
    93  	scheduler.RegisterKVDescriptor(descriptor1)
    94  	scheduler.RegisterKVDescriptor(descriptor2)
    95  
    96  	// get metadata map created for each descriptor
    97  	metadataMap := scheduler.GetMetadataMap(descriptor1.Name)
    98  	nameToInteger1, withMetadataMap := metadataMap.(test.NameToInteger)
    99  	Expect(withMetadataMap).To(BeTrue())
   100  	metadataMap = scheduler.GetMetadataMap(descriptor2.Name)
   101  	nameToInteger2, withMetadataMap := metadataMap.(test.NameToInteger)
   102  	Expect(withMetadataMap).To(BeTrue())
   103  
   104  	// run resync transaction against empty SB
   105  	startTime := time.Now()
   106  	schedulerTxn := scheduler.StartNBTransaction()
   107  	schedulerTxn.SetValue(prefixB+baseValue2, test.NewArrayValue("item1", "item2"))
   108  	seqNum, err := schedulerTxn.Commit(WithResync(testCtx, FullResync, true))
   109  	stopTime := time.Now()
   110  	Expect(seqNum).To(BeEquivalentTo(0))
   111  	Expect(err).ShouldNot(HaveOccurred())
   112  
   113  	// check the state of SB
   114  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
   115  	Expect(mockSB.GetValues(nil)).To(BeEmpty())
   116  
   117  	// check metadata
   118  	Expect(metadataMap.ListAllNames()).To(BeEmpty())
   119  
   120  	// check operations executed in SB
   121  	opHistory := mockSB.PopHistoryOfOps()
   122  	Expect(opHistory).To(HaveLen(1))
   123  	operation := opHistory[0]
   124  	Expect(operation.OpType).To(Equal(test.MockRetrieve))
   125  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   126  	checkValues(operation.CorrelateRetrieve, []KVWithMetadata{
   127  		{
   128  			Key:      prefixB + baseValue2,
   129  			Value:    test.NewArrayValue("item1", "item2"),
   130  			Metadata: nil,
   131  			Origin:   FromNB,
   132  		},
   133  	})
   134  
   135  	// check transaction operations
   136  	txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Now())
   137  	Expect(txnHistory).To(HaveLen(1))
   138  	txn := txnHistory[0]
   139  	Expect(txn.PreRecord).To(BeFalse())
   140  	Expect(txn.Start.After(startTime)).To(BeTrue())
   141  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
   142  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
   143  	Expect(txn.SeqNum).To(BeEquivalentTo(0))
   144  	Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction))
   145  	Expect(txn.ResyncType).To(BeEquivalentTo(FullResync))
   146  	Expect(txn.Description).To(BeEmpty())
   147  	checkRecordedValues(txn.Values, []RecordedKVPair{
   148  		{Key: prefixB + baseValue2, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB},
   149  	})
   150  
   151  	txnOps := RecordedTxnOps{
   152  		{
   153  			Operation: TxnOperation_CREATE,
   154  			Key:       prefixB + baseValue2,
   155  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
   156  			PrevState: ValueState_NONEXISTENT,
   157  			NewState:  ValueState_PENDING,
   158  			NOOP:      true,
   159  		},
   160  	}
   161  	checkTxnOperations(txn.Planned, txnOps)
   162  	checkTxnOperations(txn.Executed, txnOps)
   163  
   164  	// check flag stats
   165  	graphR := scheduler.graph.Read()
   166  	errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil)
   167  	Expect(errorStats.TotalCount).To(BeEquivalentTo(0))
   168  	pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil)
   169  	Expect(pendingStats.TotalCount).To(BeEquivalentTo(1))
   170  	derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil)
   171  	Expect(derivedStats.TotalCount).To(BeEquivalentTo(0))
   172  	lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil)
   173  	Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(1))
   174  	descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil)
   175  	Expect(descriptorStats.TotalCount).To(BeEquivalentTo(1))
   176  	Expect(descriptorStats.PerValueCount).ToNot(HaveKey(descriptor1Name))
   177  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor2Name))
   178  	Expect(descriptorStats.PerValueCount[descriptor2Name]).To(BeEquivalentTo(1))
   179  	valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil)
   180  	Expect(valueStateStats.TotalCount).To(BeEquivalentTo(1))
   181  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_PENDING.String()))
   182  	Expect(valueStateStats.PerValueCount[ValueState_PENDING.String()]).To(BeEquivalentTo(1))
   183  	graphR.Release()
   184  
   185  	// check value dumps for prefix B
   186  	nbConfig := []KVWithMetadata{
   187  		{Key: prefixB + baseValue2, Value: test.NewArrayValue("item1", "item2"), Origin: FromNB},
   188  	}
   189  	views := []View{NBView, SBView, CachedView}
   190  	for _, view := range views {
   191  		var expValues []KVWithMetadata
   192  		if view == NBView {
   193  			expValues = nbConfig
   194  		} // else empty expected set of values
   195  		dumpedValues, err := scheduler.DumpValuesByKeyPrefix(prefixB, view)
   196  		Expect(err).To(BeNil())
   197  		checkValues(dumpedValues, expValues)
   198  		dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor2Name, view)
   199  		Expect(err).To(BeNil())
   200  		checkValues(dumpedValues, expValues)
   201  	}
   202  	mockSB.PopHistoryOfOps() // remove Retrieve-s from the history
   203  
   204  	// check value status
   205  	status := scheduler.GetValueStatus(prefixB + baseValue2)
   206  	Expect(status).ToNot(BeNil())
   207  	checkBaseValueStatus(status, &BaseValueStatus{
   208  		Value: &ValueStatus{
   209  			Key:           prefixB + baseValue2,
   210  			State:         ValueState_PENDING,
   211  			LastOperation: TxnOperation_CREATE,
   212  			Details:       []string{prefixA + baseValue1},
   213  		},
   214  	})
   215  
   216  	// subscribe to receive notifications about value state changes for prefixA
   217  	statusChan := make(chan *BaseValueStatus, 5)
   218  	scheduler.WatchValueStatus(statusChan, prefixSelector(prefixA))
   219  
   220  	// send notification
   221  	startTime = time.Now()
   222  	mockSB.SetValue(prefixA+baseValue1, test.NewArrayValue("item1"), &test.OnlyInteger{Integer: 10}, FromSB, false)
   223  	notifError := scheduler.PushSBNotification(KVWithMetadata{
   224  		Key:      prefixA + baseValue1,
   225  		Value:    test.NewArrayValue("item1"),
   226  		Metadata: &test.OnlyInteger{Integer: 10},
   227  	})
   228  	Expect(notifError).ShouldNot(HaveOccurred())
   229  
   230  	// wait until the notification is processed
   231  	Eventually(func() int {
   232  		return len(scheduler.GetTransactionHistory(startTime, time.Now()))
   233  	}, notificationTimeout).ShouldNot(BeZero())
   234  	stopTime = time.Now()
   235  
   236  	// check the state of SB
   237  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
   238  	// -> base value 1
   239  	value := mockSB.GetValue(prefixA + baseValue1)
   240  	Expect(value).ToNot(BeNil())
   241  	Expect(proto.Equal(value.Value, test.NewArrayValue("item1"))).To(BeTrue())
   242  	Expect(value.Metadata).ToNot(BeNil())
   243  	Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(10))
   244  	Expect(value.Origin).To(BeEquivalentTo(FromSB))
   245  	// -> base value 2
   246  	value = mockSB.GetValue(prefixB + baseValue2)
   247  	Expect(value).ToNot(BeNil())
   248  	Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue())
   249  	Expect(value.Metadata).ToNot(BeNil())
   250  	Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0))
   251  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   252  	// -> item1 derived from base value 2
   253  	value = mockSB.GetValue(prefixB + baseValue2 + "/item1")
   254  	Expect(value).ToNot(BeNil())
   255  	Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue())
   256  	Expect(value.Metadata).To(BeNil())
   257  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   258  	// -> item2 derived from base value 2 is pending
   259  	value = mockSB.GetValue(prefixB + baseValue2 + "/item2")
   260  	Expect(value).To(BeNil())
   261  
   262  	// check metadata
   263  	metadata, exists := nameToInteger1.LookupByName(baseValue1)
   264  	Expect(exists).To(BeTrue())
   265  	Expect(metadata.GetInteger()).To(BeEquivalentTo(10))
   266  	metadata, exists = nameToInteger2.LookupByName(baseValue2)
   267  	Expect(exists).To(BeTrue())
   268  	Expect(metadata.GetInteger()).To(BeEquivalentTo(0))
   269  
   270  	// check operations executed in SB
   271  	opHistory = mockSB.PopHistoryOfOps()
   272  	Expect(opHistory).To(HaveLen(2))
   273  	operation = opHistory[0]
   274  	Expect(operation.OpType).To(Equal(test.MockCreate))
   275  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   276  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2))
   277  	Expect(operation.Err).To(BeNil())
   278  	operation = opHistory[1]
   279  	Expect(operation.OpType).To(Equal(test.MockCreate))
   280  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   281  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item1"))
   282  	Expect(operation.Err).To(BeNil())
   283  
   284  	// check value dumps for prefix A
   285  	sbState := []KVWithMetadata{
   286  		{Key: prefixA + baseValue1, Value: test.NewArrayValue("item1"), Origin: FromSB, Metadata: &test.OnlyInteger{Integer: 10}},
   287  	}
   288  	for _, view := range views {
   289  		// prefix
   290  		var expValues []KVWithMetadata
   291  		expErrMsg := "descriptor does not support Retrieve operation" // Retrieve not supported
   292  		if view != NBView {
   293  			expValues = sbState
   294  		} // else empty set of dumped values
   295  		dumpedValues, err := scheduler.DumpValuesByKeyPrefix(prefixA, view)
   296  		if view == SBView {
   297  			Expect(err).ToNot(BeNil())
   298  			Expect(err.Error()).To(BeEquivalentTo(expErrMsg))
   299  		} else {
   300  			Expect(err).To(BeNil())
   301  			checkValues(dumpedValues, expValues)
   302  		}
   303  		dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor1Name, view)
   304  		if view == SBView {
   305  			Expect(err).ToNot(BeNil())
   306  			Expect(err.Error()).To(BeEquivalentTo(expErrMsg))
   307  		} else {
   308  			Expect(err).To(BeNil())
   309  			checkValues(dumpedValues, expValues)
   310  		}
   311  	}
   312  	mockSB.PopHistoryOfOps() // remove Retrieve-s from the history
   313  
   314  	// check value status
   315  	status = scheduler.GetValueStatus(prefixB + baseValue2)
   316  	Expect(status).ToNot(BeNil())
   317  	checkBaseValueStatus(status, &BaseValueStatus{
   318  		Value: &ValueStatus{
   319  			Key:           prefixB + baseValue2,
   320  			State:         ValueState_CONFIGURED,
   321  			LastOperation: TxnOperation_CREATE,
   322  		},
   323  		DerivedValues: []*ValueStatus{
   324  			{
   325  				Key:           prefixB + baseValue2 + "/item1",
   326  				State:         ValueState_CONFIGURED,
   327  				LastOperation: TxnOperation_CREATE,
   328  			},
   329  			{
   330  				Key:           prefixB + baseValue2 + "/item2",
   331  				State:         ValueState_PENDING,
   332  				LastOperation: TxnOperation_CREATE,
   333  				Details:       []string{prefixA + baseValue1 + "/item2"},
   334  			},
   335  		},
   336  	})
   337  	status = scheduler.GetValueStatus(prefixA + baseValue1)
   338  	Expect(status).ToNot(BeNil())
   339  	checkBaseValueStatus(status, &BaseValueStatus{
   340  		Value: &ValueStatus{
   341  			Key:           prefixA + baseValue1,
   342  			State:         ValueState_OBTAINED,
   343  			LastOperation: TxnOperation_UNDEFINED,
   344  		},
   345  		DerivedValues: []*ValueStatus{
   346  			{
   347  				Key:           prefixA + baseValue1 + "/item1",
   348  				State:         ValueState_OBTAINED,
   349  				LastOperation: TxnOperation_UNDEFINED,
   350  			},
   351  		},
   352  	})
   353  
   354  	// check transaction operations
   355  	txnHistory = scheduler.GetTransactionHistory(startTime, time.Now())
   356  	Expect(txnHistory).To(HaveLen(1))
   357  	txn = txnHistory[0]
   358  	Expect(txn.PreRecord).To(BeFalse())
   359  	Expect(txn.Start.After(startTime)).To(BeTrue())
   360  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
   361  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
   362  	Expect(txn.SeqNum).To(BeEquivalentTo(1))
   363  	Expect(txn.TxnType).To(BeEquivalentTo(SBNotification))
   364  	Expect(txn.ResyncType).To(BeEquivalentTo(NotResync))
   365  	Expect(txn.Description).To(BeEmpty())
   366  	checkRecordedValues(txn.Values, []RecordedKVPair{
   367  		{Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item1")), Origin: FromSB},
   368  	})
   369  
   370  	txnOps = RecordedTxnOps{
   371  		{
   372  			Operation: TxnOperation_CREATE,
   373  			Key:       prefixA + baseValue1,
   374  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1")),
   375  			PrevState: ValueState_NONEXISTENT,
   376  			NewState:  ValueState_OBTAINED,
   377  		},
   378  		{
   379  			Operation: TxnOperation_CREATE,
   380  			Key:       prefixB + baseValue2,
   381  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
   382  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
   383  			PrevState: ValueState_PENDING,
   384  			NewState:  ValueState_CONFIGURED,
   385  		},
   386  		{
   387  			Operation: TxnOperation_CREATE,
   388  			Key:       prefixB + baseValue2 + "/item1",
   389  			IsDerived: true,
   390  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item1")),
   391  			PrevState: ValueState_NONEXISTENT,
   392  			NewState:  ValueState_CONFIGURED,
   393  		},
   394  		{
   395  			Operation: TxnOperation_CREATE,
   396  			Key:       prefixB + baseValue2 + "/item2",
   397  			IsDerived: true,
   398  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
   399  			PrevState: ValueState_NONEXISTENT,
   400  			NewState:  ValueState_PENDING,
   401  			NOOP:      true,
   402  		},
   403  		{
   404  			Operation:  TxnOperation_CREATE,
   405  			Key:        prefixA + baseValue1 + "/item1",
   406  			IsDerived:  true,
   407  			IsProperty: true,
   408  			NewValue:   utils.RecordProtoMessage(test.NewStringValue("item1")),
   409  			PrevState:  ValueState_NONEXISTENT,
   410  			NewState:   ValueState_OBTAINED,
   411  		},
   412  	}
   413  	checkTxnOperations(txn.Planned, txnOps)
   414  	checkTxnOperations(txn.Executed, txnOps)
   415  
   416  	// check flag stats
   417  	graphR = scheduler.graph.Read()
   418  	errorStats = graphR.GetFlagStats(ErrorFlagIndex, nil)
   419  	Expect(errorStats.TotalCount).To(BeEquivalentTo(0))
   420  	pendingStats = graphR.GetFlagStats(UnavailValueFlagIndex, nil)
   421  	Expect(pendingStats.TotalCount).To(BeEquivalentTo(2))
   422  	derivedStats = graphR.GetFlagStats(DerivedFlagIndex, nil)
   423  	Expect(derivedStats.TotalCount).To(BeEquivalentTo(3))
   424  	lastUpdateStats = graphR.GetFlagStats(LastUpdateFlagIndex, nil)
   425  	Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(6))
   426  	descriptorStats = graphR.GetFlagStats(DescriptorFlagIndex, nil)
   427  	Expect(descriptorStats.TotalCount).To(BeEquivalentTo(5))
   428  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name))
   429  	Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(1))
   430  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor2Name))
   431  	Expect(descriptorStats.PerValueCount[descriptor2Name]).To(BeEquivalentTo(4))
   432  	valueStateStats = graphR.GetFlagStats(ValueStateFlagIndex, nil)
   433  	Expect(valueStateStats.TotalCount).To(BeEquivalentTo(6))
   434  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_PENDING.String()))
   435  	Expect(valueStateStats.PerValueCount[ValueState_PENDING.String()]).To(BeEquivalentTo(2))
   436  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String()))
   437  	Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(2))
   438  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_OBTAINED.String()))
   439  	Expect(valueStateStats.PerValueCount[ValueState_OBTAINED.String()]).To(BeEquivalentTo(2))
   440  	graphR.Release()
   441  
   442  	// send 2nd notification
   443  	startTime = time.Now()
   444  	mockSB.SetValue(prefixA+baseValue1, test.NewArrayValue("item1", "item2"), &test.OnlyInteger{Integer: 11}, FromSB, false)
   445  	notifError = scheduler.PushSBNotification(KVWithMetadata{
   446  		Key:      prefixA + baseValue1,
   447  		Value:    test.NewArrayValue("item1", "item2"),
   448  		Metadata: &test.OnlyInteger{Integer: 11}})
   449  	Expect(notifError).ShouldNot(HaveOccurred())
   450  
   451  	// wait until the notification is processed
   452  	Eventually(func() int {
   453  		return len(scheduler.GetTransactionHistory(startTime, time.Now()))
   454  	}, notificationTimeout).ShouldNot(BeZero())
   455  	stopTime = time.Now()
   456  
   457  	// check the state of SB
   458  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
   459  	// -> base value 1
   460  	value = mockSB.GetValue(prefixA + baseValue1)
   461  	Expect(value).ToNot(BeNil())
   462  	Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue())
   463  	Expect(value.Metadata).ToNot(BeNil())
   464  	Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(11))
   465  	Expect(value.Origin).To(BeEquivalentTo(FromSB))
   466  	// -> base value 2
   467  	value = mockSB.GetValue(prefixB + baseValue2)
   468  	Expect(value).ToNot(BeNil())
   469  	Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue())
   470  	Expect(value.Metadata).ToNot(BeNil())
   471  	Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0))
   472  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   473  	// -> item1 derived from base value 2
   474  	value = mockSB.GetValue(prefixB + baseValue2 + "/item1")
   475  	Expect(value).ToNot(BeNil())
   476  	Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue())
   477  	Expect(value.Metadata).To(BeNil())
   478  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   479  	// -> item2 derived from base value 2
   480  	value = mockSB.GetValue(prefixB + baseValue2 + "/item2")
   481  	Expect(value).ToNot(BeNil())
   482  	Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue())
   483  	Expect(value.Metadata).To(BeNil())
   484  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   485  
   486  	// check metadata
   487  	metadata, exists = nameToInteger1.LookupByName(baseValue1)
   488  	Expect(exists).To(BeTrue())
   489  	Expect(metadata.GetInteger()).To(BeEquivalentTo(11))
   490  	metadata, exists = nameToInteger2.LookupByName(baseValue2)
   491  	Expect(exists).To(BeTrue())
   492  	Expect(metadata.GetInteger()).To(BeEquivalentTo(0))
   493  
   494  	// check operations executed in SB
   495  	opHistory = mockSB.PopHistoryOfOps()
   496  	Expect(opHistory).To(HaveLen(1))
   497  	operation = opHistory[0]
   498  	Expect(operation.OpType).To(Equal(test.MockCreate))
   499  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   500  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item2"))
   501  	Expect(operation.Err).To(BeNil())
   502  
   503  	// check value status
   504  	status = scheduler.GetValueStatus(prefixB + baseValue2)
   505  	Expect(status).ToNot(BeNil())
   506  	checkBaseValueStatus(status, &BaseValueStatus{
   507  		Value: &ValueStatus{
   508  			Key:           prefixB + baseValue2,
   509  			State:         ValueState_CONFIGURED,
   510  			LastOperation: TxnOperation_CREATE,
   511  		},
   512  		DerivedValues: []*ValueStatus{
   513  			{
   514  				Key:           prefixB + baseValue2 + "/item1",
   515  				State:         ValueState_CONFIGURED,
   516  				LastOperation: TxnOperation_CREATE,
   517  			},
   518  			{
   519  				Key:           prefixB + baseValue2 + "/item2",
   520  				State:         ValueState_CONFIGURED,
   521  				LastOperation: TxnOperation_CREATE,
   522  			},
   523  		},
   524  	})
   525  
   526  	// check transaction operations
   527  	txnHistory = scheduler.GetTransactionHistory(startTime, time.Now())
   528  	Expect(txnHistory).To(HaveLen(1))
   529  	txn = txnHistory[0]
   530  	Expect(txn.PreRecord).To(BeFalse())
   531  	Expect(txn.Start.After(startTime)).To(BeTrue())
   532  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
   533  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
   534  	Expect(txn.SeqNum).To(BeEquivalentTo(2))
   535  	Expect(txn.TxnType).To(BeEquivalentTo(SBNotification))
   536  	Expect(txn.ResyncType).To(BeEquivalentTo(NotResync))
   537  	Expect(txn.Description).To(BeEmpty())
   538  	checkRecordedValues(txn.Values, []RecordedKVPair{
   539  		{Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromSB},
   540  	})
   541  
   542  	txnOps = RecordedTxnOps{
   543  		{
   544  			Operation: TxnOperation_UPDATE,
   545  			Key:       prefixA + baseValue1,
   546  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")),
   547  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
   548  			PrevState: ValueState_OBTAINED,
   549  			NewState:  ValueState_OBTAINED,
   550  		},
   551  		{
   552  			Operation:  TxnOperation_CREATE,
   553  			Key:        prefixA + baseValue1 + "/item2",
   554  			IsDerived:  true,
   555  			IsProperty: true,
   556  			NewValue:   utils.RecordProtoMessage(test.NewStringValue("item2")),
   557  			PrevState:  ValueState_NONEXISTENT,
   558  			NewState:   ValueState_OBTAINED,
   559  		},
   560  		{
   561  			Operation: TxnOperation_CREATE,
   562  			Key:       prefixB + baseValue2 + "/item2",
   563  			IsDerived: true,
   564  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("item2")),
   565  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
   566  			PrevState: ValueState_PENDING,
   567  			NewState:  ValueState_CONFIGURED,
   568  		},
   569  	}
   570  	checkTxnOperations(txn.Planned, txnOps)
   571  	checkTxnOperations(txn.Executed, txnOps)
   572  
   573  	// check flag stats
   574  	graphR = scheduler.graph.Read()
   575  	errorStats = graphR.GetFlagStats(ErrorFlagIndex, nil)
   576  	Expect(errorStats.TotalCount).To(BeEquivalentTo(0))
   577  	pendingStats = graphR.GetFlagStats(UnavailValueFlagIndex, nil)
   578  	Expect(pendingStats.TotalCount).To(BeEquivalentTo(2))
   579  	derivedStats = graphR.GetFlagStats(DerivedFlagIndex, nil)
   580  	Expect(derivedStats.TotalCount).To(BeEquivalentTo(6))
   581  	lastUpdateStats = graphR.GetFlagStats(LastUpdateFlagIndex, nil)
   582  	Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(10))
   583  	descriptorStats = graphR.GetFlagStats(DescriptorFlagIndex, nil)
   584  	Expect(descriptorStats.TotalCount).To(BeEquivalentTo(7))
   585  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name))
   586  	Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(2))
   587  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor2Name))
   588  	Expect(descriptorStats.PerValueCount[descriptor2Name]).To(BeEquivalentTo(5))
   589  	valueStateStats = graphR.GetFlagStats(ValueStateFlagIndex, nil)
   590  	Expect(valueStateStats.TotalCount).To(BeEquivalentTo(10))
   591  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_PENDING.String()))
   592  	Expect(valueStateStats.PerValueCount[ValueState_PENDING.String()]).To(BeEquivalentTo(2))
   593  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String()))
   594  	Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(3))
   595  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_OBTAINED.String()))
   596  	Expect(valueStateStats.PerValueCount[ValueState_OBTAINED.String()]).To(BeEquivalentTo(5))
   597  	graphR.Release()
   598  
   599  	// send 3rd notification
   600  	startTime = time.Now()
   601  	mockSB.SetValue(prefixA+baseValue1, nil, nil, FromSB, false)
   602  	notifError = scheduler.PushSBNotification(KVWithMetadata{
   603  		Key:      prefixA + baseValue1,
   604  		Value:    nil,
   605  		Metadata: nil,
   606  	})
   607  	Expect(notifError).ShouldNot(HaveOccurred())
   608  
   609  	// wait until the notification is processed
   610  	Eventually(func() int {
   611  		return len(scheduler.GetTransactionHistory(startTime, time.Now()))
   612  	}, notificationTimeout).ShouldNot(BeZero())
   613  	stopTime = time.Now()
   614  
   615  	// check the state of SB
   616  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
   617  	Expect(mockSB.GetValues(nil)).To(BeEmpty())
   618  
   619  	// check metadata
   620  	Expect(metadataMap.ListAllNames()).To(BeEmpty())
   621  
   622  	// check operations executed in SB
   623  	opHistory = mockSB.PopHistoryOfOps()
   624  	Expect(opHistory).To(HaveLen(3))
   625  	operation = opHistory[0]
   626  	Expect(operation.OpType).To(Equal(test.MockDelete))
   627  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   628  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item2"))
   629  	Expect(operation.Err).To(BeNil())
   630  	operation = opHistory[1]
   631  	Expect(operation.OpType).To(Equal(test.MockDelete))
   632  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   633  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item1"))
   634  	Expect(operation.Err).To(BeNil())
   635  	operation = opHistory[2]
   636  	Expect(operation.OpType).To(Equal(test.MockDelete))
   637  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   638  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2))
   639  	Expect(operation.Err).To(BeNil())
   640  
   641  	// check value status
   642  	status = scheduler.GetValueStatus(prefixB + baseValue2)
   643  	Expect(status).ToNot(BeNil())
   644  	checkBaseValueStatus(status, &BaseValueStatus{
   645  		Value: &ValueStatus{
   646  			Key:           prefixB + baseValue2,
   647  			State:         ValueState_PENDING,
   648  			LastOperation: TxnOperation_DELETE,
   649  			Details:       []string{prefixA + baseValue1},
   650  		},
   651  	})
   652  	status = scheduler.GetValueStatus(prefixA + baseValue1)
   653  	Expect(status).ToNot(BeNil())
   654  	checkBaseValueStatus(status, &BaseValueStatus{
   655  		Value: &ValueStatus{
   656  			Key:           prefixA + baseValue1,
   657  			State:         ValueState_NONEXISTENT,
   658  			LastOperation: TxnOperation_UNDEFINED,
   659  		},
   660  	})
   661  
   662  	// check transaction operations
   663  	txnHistory = scheduler.GetTransactionHistory(startTime, time.Now())
   664  	Expect(txnHistory).To(HaveLen(1))
   665  	txn = txnHistory[0]
   666  	Expect(txn.PreRecord).To(BeFalse())
   667  	Expect(txn.Start.After(startTime)).To(BeTrue())
   668  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
   669  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
   670  	Expect(txn.SeqNum).To(BeEquivalentTo(3))
   671  	Expect(txn.TxnType).To(BeEquivalentTo(SBNotification))
   672  	Expect(txn.ResyncType).To(BeEquivalentTo(NotResync))
   673  	Expect(txn.Description).To(BeEmpty())
   674  	checkRecordedValues(txn.Values, []RecordedKVPair{
   675  		{Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(nil), Origin: FromSB},
   676  	})
   677  
   678  	txnOps = RecordedTxnOps{
   679  		{
   680  			Operation:  TxnOperation_DELETE,
   681  			Key:        prefixA + baseValue1 + "/item1",
   682  			IsDerived:  true,
   683  			IsProperty: true,
   684  			PrevValue:  utils.RecordProtoMessage(test.NewStringValue("item1")),
   685  			PrevState:  ValueState_OBTAINED,
   686  			NewState:   ValueState_REMOVED,
   687  		},
   688  		{
   689  			Operation: TxnOperation_DELETE,
   690  			Key:       prefixB + baseValue2 + "/item2",
   691  			IsDerived: true,
   692  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("item2")),
   693  			PrevState: ValueState_CONFIGURED,
   694  			NewState:  ValueState_REMOVED,
   695  		},
   696  		{
   697  			Operation:  TxnOperation_DELETE,
   698  			Key:        prefixA + baseValue1 + "/item2",
   699  			IsDerived:  true,
   700  			IsProperty: true,
   701  			PrevValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
   702  			PrevState:  ValueState_OBTAINED,
   703  			NewState:   ValueState_REMOVED,
   704  		},
   705  		{
   706  			Operation: TxnOperation_DELETE,
   707  			Key:       prefixB + baseValue2 + "/item1",
   708  			IsDerived: true,
   709  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")),
   710  			PrevState: ValueState_CONFIGURED,
   711  			NewState:  ValueState_REMOVED,
   712  		},
   713  		{
   714  			Operation: TxnOperation_DELETE,
   715  			Key:       prefixB + baseValue2,
   716  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
   717  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), // TODO: do we want nil instead?
   718  			PrevState: ValueState_CONFIGURED,
   719  			NewState:  ValueState_PENDING,
   720  		},
   721  		{
   722  			Operation: TxnOperation_DELETE,
   723  			Key:       prefixA + baseValue1,
   724  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
   725  			PrevState: ValueState_OBTAINED,
   726  			NewState:  ValueState_REMOVED,
   727  		},
   728  	}
   729  	checkTxnOperations(txn.Planned, txnOps)
   730  	checkTxnOperations(txn.Executed, txnOps)
   731  
   732  	// close scheduler
   733  	err = scheduler.Close()
   734  	Expect(err).To(BeNil())
   735  }
   736  
   737  func TestNotificationsWithRetry(t *testing.T) {
   738  	RegisterTestingT(t)
   739  
   740  	// prepare KV Scheduler
   741  	scheduler := NewPlugin(UseDeps(func(deps *Deps) {
   742  		deps.HTTPHandlers = nil
   743  	}))
   744  	err := scheduler.Init()
   745  	Expect(err).To(BeNil())
   746  	scheduler.config.EnableTxnSimulation = true
   747  
   748  	// prepare mocks
   749  	mockSB := test.NewMockSouthbound()
   750  	// -> descriptor1 (notifications):
   751  	descriptor1 := test.NewMockDescriptor(&KVDescriptor{
   752  		Name:          descriptor1Name,
   753  		NBKeyPrefix:   prefixA,
   754  		KeySelector:   prefixSelector(prefixA),
   755  		ValueTypeName: string(proto.MessageName(test.NewArrayValue())),
   756  		DerivedValues: test.ArrayValueDerBuilder,
   757  		WithMetadata:  true,
   758  	}, mockSB, 0, test.WithoutRetrieve)
   759  	// -> descriptor2:
   760  	descriptor2 := test.NewMockDescriptor(&KVDescriptor{
   761  		Name:          descriptor2Name,
   762  		NBKeyPrefix:   prefixB,
   763  		KeySelector:   prefixSelector(prefixB),
   764  		ValueTypeName: string(proto.MessageName(test.NewArrayValue())),
   765  		Dependencies: func(key string, value proto.Message) []Dependency {
   766  			if key == prefixB+baseValue2 {
   767  				depKey := prefixA + baseValue1
   768  				return []Dependency{
   769  					{Label: depKey, Key: depKey},
   770  				}
   771  			}
   772  			if key == prefixB+baseValue2+"/item2" {
   773  				depKey := prefixA + baseValue1 + "/item2"
   774  				return []Dependency{
   775  					{Label: depKey, Key: depKey},
   776  				}
   777  			}
   778  			return nil
   779  		},
   780  		DerivedValues: test.ArrayValueDerBuilder,
   781  		WithMetadata:  true,
   782  	}, mockSB, 0)
   783  	// -> descriptor3:
   784  	descriptor3 := test.NewMockDescriptor(&KVDescriptor{
   785  		Name:            descriptor3Name,
   786  		NBKeyPrefix:     prefixC,
   787  		KeySelector:     prefixSelector(prefixC),
   788  		ValueTypeName:   string(proto.MessageName(test.NewStringValue(""))),
   789  		ValueComparator: test.StringValueComparator,
   790  		Dependencies: func(key string, value proto.Message) []Dependency {
   791  			if key == prefixC+baseValue3 {
   792  				return []Dependency{
   793  					{
   794  						Label: prefixA,
   795  						AnyOf: AnyOfDependency{
   796  							KeyPrefixes: []string{prefixA},
   797  						},
   798  					},
   799  				}
   800  			}
   801  			return nil
   802  		},
   803  		WithMetadata:         true,
   804  		RetrieveDependencies: []string{descriptor2Name},
   805  	}, mockSB, 0)
   806  
   807  	// -> planned errors
   808  	mockSB.PlanError(prefixB+baseValue2+"/item2", errors.New("failed to add derived value"),
   809  		func() {
   810  			mockSB.SetValue(prefixB+baseValue2, test.NewArrayValue("item1"),
   811  				&test.OnlyInteger{Integer: 0}, FromNB, false)
   812  		})
   813  	mockSB.PlanError(prefixC+baseValue3, errors.New("failed to add value"),
   814  		func() {
   815  			mockSB.SetValue(prefixC+baseValue3, nil, nil, FromNB, false)
   816  		})
   817  
   818  	// register all 3 descriptors with the scheduler
   819  	scheduler.RegisterKVDescriptor(descriptor1)
   820  	scheduler.RegisterKVDescriptor(descriptor2)
   821  	scheduler.RegisterKVDescriptor(descriptor3)
   822  
   823  	// get metadata map created for each descriptor
   824  	metadataMap := scheduler.GetMetadataMap(descriptor1.Name)
   825  	nameToInteger1, withMetadataMap := metadataMap.(test.NameToInteger)
   826  	Expect(withMetadataMap).To(BeTrue())
   827  	metadataMap = scheduler.GetMetadataMap(descriptor2.Name)
   828  	nameToInteger2, withMetadataMap := metadataMap.(test.NameToInteger)
   829  	Expect(withMetadataMap).To(BeTrue())
   830  	metadataMap = scheduler.GetMetadataMap(descriptor3.Name)
   831  	nameToInteger3, withMetadataMap := metadataMap.(test.NameToInteger)
   832  	Expect(withMetadataMap).To(BeTrue())
   833  
   834  	// run 1st data-change transaction with retry against empty SB
   835  	schedulerTxn1 := scheduler.StartNBTransaction()
   836  	schedulerTxn1.SetValue(prefixB+baseValue2, test.NewArrayValue("item1", "item2"))
   837  	seqNum, err := schedulerTxn1.Commit(WithRetryDefault(testCtx))
   838  	Expect(seqNum).To(BeEquivalentTo(0))
   839  	Expect(err).ShouldNot(HaveOccurred())
   840  
   841  	// run 2nd data-change transaction with retry
   842  	schedulerTxn2 := scheduler.StartNBTransaction()
   843  	schedulerTxn2.SetValue(prefixC+baseValue3, test.NewStringValue("base-value3-data"))
   844  	seqNum, err = schedulerTxn2.Commit(WithRetry(testCtx, 3*time.Second, 3, true))
   845  	Expect(seqNum).To(BeEquivalentTo(1))
   846  	Expect(err).ShouldNot(HaveOccurred())
   847  
   848  	// check the state of SB - empty since dependencies are not met
   849  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
   850  	Expect(mockSB.GetValues(nil)).To(BeEmpty())
   851  	Expect(mockSB.PopHistoryOfOps()).To(HaveLen(0))
   852  
   853  	// check metadata
   854  	Expect(metadataMap.ListAllNames()).To(BeEmpty())
   855  
   856  	// subscribe to receive notifications about values which are going to fail
   857  	prefBStatusChan := make(chan *BaseValueStatus, 5)
   858  	scheduler.WatchValueStatus(prefBStatusChan, prefixSelector(prefixB))
   859  	prefCStatusChan := make(chan *BaseValueStatus, 5)
   860  	scheduler.WatchValueStatus(prefCStatusChan, prefixSelector(prefixC))
   861  
   862  	// send notification
   863  	startTime := time.Now()
   864  	notifError := scheduler.PushSBNotification(KVWithMetadata{
   865  		Key:      prefixA + baseValue1,
   866  		Value:    test.NewArrayValue("item1", "item2"),
   867  		Metadata: &test.OnlyInteger{Integer: 10},
   868  	})
   869  	Expect(notifError).ShouldNot(HaveOccurred())
   870  
   871  	// wait until the notification is processed
   872  	Eventually(func() int {
   873  		return len(scheduler.GetTransactionHistory(startTime, time.Now()))
   874  	}, 2*time.Second).ShouldNot(BeZero())
   875  	stopTime := time.Now()
   876  
   877  	// check value state updates received through the channels
   878  	var valueStatus *BaseValueStatus
   879  	Eventually(prefBStatusChan, time.Second).Should(Receive(&valueStatus))
   880  	checkBaseValueStatus(valueStatus, &BaseValueStatus{
   881  		Value: &ValueStatus{
   882  			Key:           prefixB + baseValue2,
   883  			State:         ValueState_CONFIGURED,
   884  			LastOperation: TxnOperation_CREATE,
   885  		},
   886  		DerivedValues: []*ValueStatus{
   887  			{
   888  				Key:           prefixB + baseValue2 + "/item1",
   889  				State:         ValueState_CONFIGURED,
   890  				LastOperation: TxnOperation_CREATE,
   891  			},
   892  			{
   893  				Key:           prefixB + baseValue2 + "/item2",
   894  				State:         ValueState_RETRYING,
   895  				LastOperation: TxnOperation_CREATE,
   896  				Error:         "failed to add derived value",
   897  			},
   898  		},
   899  	})
   900  	Eventually(prefCStatusChan, time.Second).Should(Receive(&valueStatus))
   901  	checkBaseValueStatus(valueStatus, &BaseValueStatus{
   902  		Value: &ValueStatus{
   903  			Key:           prefixC + baseValue3,
   904  			State:         ValueState_RETRYING,
   905  			LastOperation: TxnOperation_CREATE,
   906  			Error:         "failed to add value",
   907  		},
   908  	})
   909  
   910  	// check the state of SB
   911  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
   912  	// -> base value 2
   913  	value := mockSB.GetValue(prefixB + baseValue2)
   914  	Expect(value).ToNot(BeNil())
   915  	Expect(proto.Equal(value.Value, test.NewArrayValue("item1"))).To(BeTrue())
   916  	Expect(value.Metadata).ToNot(BeNil())
   917  	Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0))
   918  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   919  	// -> item1 derived from base value 2
   920  	value = mockSB.GetValue(prefixB + baseValue2 + "/item1")
   921  	Expect(value).ToNot(BeNil())
   922  	Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue())
   923  	Expect(value.Metadata).To(BeNil())
   924  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
   925  	// -> item2 derived from base value 2 failed to get created
   926  	value = mockSB.GetValue(prefixB + baseValue2 + "/item2")
   927  	Expect(value).To(BeNil())
   928  	// -> base value 3 failed to get created
   929  	value = mockSB.GetValue(prefixC + baseValue3)
   930  	Expect(value).To(BeNil())
   931  	Expect(mockSB.GetValues(nil)).To(HaveLen(2))
   932  
   933  	// check metadata
   934  	metadata, exists := nameToInteger1.LookupByName(baseValue1)
   935  	Expect(exists).To(BeTrue())
   936  	Expect(metadata.GetInteger()).To(BeEquivalentTo(10))
   937  	metadata, exists = nameToInteger2.LookupByName(baseValue2)
   938  	Expect(exists).To(BeTrue())
   939  	Expect(metadata.GetInteger()).To(BeEquivalentTo(0))
   940  	metadata, exists = nameToInteger3.LookupByName(baseValue3)
   941  	Expect(exists).To(BeFalse())
   942  	Expect(metadata).To(BeNil())
   943  
   944  	// check operations executed in SB
   945  	opHistory := mockSB.PopHistoryOfOps()
   946  	Expect(opHistory).To(HaveLen(6))
   947  	operation := opHistory[0]
   948  	Expect(operation.OpType).To(Equal(test.MockCreate))
   949  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   950  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2))
   951  	Expect(operation.Err).To(BeNil())
   952  	operation = opHistory[1]
   953  	Expect(operation.OpType).To(Equal(test.MockCreate))
   954  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   955  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item1"))
   956  	Expect(operation.Err).To(BeNil())
   957  	operation = opHistory[2]
   958  	Expect(operation.OpType).To(Equal(test.MockCreate))
   959  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name))
   960  	Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3))
   961  	Expect(operation.Err).ToNot(BeNil())
   962  	Expect(operation.Err.Error()).To(BeEquivalentTo("failed to add value"))
   963  	operation = opHistory[3]
   964  	Expect(operation.OpType).To(Equal(test.MockCreate))
   965  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   966  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item2"))
   967  	Expect(operation.Err).ToNot(BeNil())
   968  	Expect(operation.Err.Error()).To(BeEquivalentTo("failed to add derived value"))
   969  	operation = opHistory[4] // refresh failed value
   970  	Expect(operation.OpType).To(Equal(test.MockRetrieve))
   971  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
   972  	checkValues(operation.CorrelateRetrieve, []KVWithMetadata{
   973  		{
   974  			Key:      prefixB + baseValue2,
   975  			Value:    test.NewArrayValue("item1", "item2"),
   976  			Metadata: &test.OnlyInteger{Integer: 0},
   977  			Origin:   FromNB,
   978  		},
   979  	})
   980  	operation = opHistory[5] // refresh failed value
   981  	Expect(operation.OpType).To(Equal(test.MockRetrieve))
   982  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name))
   983  	checkValues(operation.CorrelateRetrieve, []KVWithMetadata{})
   984  
   985  	// check last transaction
   986  	txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Now())
   987  	Expect(txnHistory).To(HaveLen(3))
   988  	txn := txnHistory[2]
   989  	Expect(txn.PreRecord).To(BeFalse())
   990  	Expect(txn.Start.After(startTime)).To(BeTrue())
   991  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
   992  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
   993  	Expect(txn.SeqNum).To(BeEquivalentTo(2))
   994  	Expect(txn.TxnType).To(BeEquivalentTo(SBNotification))
   995  	Expect(txn.ResyncType).To(BeEquivalentTo(NotResync))
   996  	Expect(txn.Description).To(BeEmpty())
   997  	checkRecordedValues(txn.Values, []RecordedKVPair{
   998  		{Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromSB},
   999  	})
  1000  
  1001  	// -> planned operations
  1002  	txnOps := RecordedTxnOps{
  1003  		{
  1004  			Operation: TxnOperation_CREATE,
  1005  			Key:       prefixA + baseValue1,
  1006  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1007  			PrevState: ValueState_NONEXISTENT,
  1008  			NewState:  ValueState_OBTAINED,
  1009  		},
  1010  		{
  1011  			Operation: TxnOperation_CREATE,
  1012  			Key:       prefixB + baseValue2,
  1013  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1014  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1015  			PrevState: ValueState_PENDING,
  1016  			NewState:  ValueState_CONFIGURED,
  1017  		},
  1018  		{
  1019  			Operation: TxnOperation_CREATE,
  1020  			Key:       prefixB + baseValue2 + "/item1",
  1021  			IsDerived: true,
  1022  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item1")),
  1023  			PrevState: ValueState_NONEXISTENT,
  1024  			NewState:  ValueState_CONFIGURED,
  1025  		},
  1026  		{
  1027  			Operation: TxnOperation_CREATE,
  1028  			Key:       prefixC + baseValue3,
  1029  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("base-value3-data")),
  1030  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("base-value3-data")),
  1031  			PrevState: ValueState_PENDING,
  1032  			NewState:  ValueState_CONFIGURED,
  1033  		},
  1034  		{
  1035  			Operation: TxnOperation_CREATE,
  1036  			Key:       prefixA + baseValue1 + "/item1",
  1037  			IsDerived: true,
  1038  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item1")),
  1039  			PrevState: ValueState_NONEXISTENT,
  1040  			NewState:  ValueState_OBTAINED,
  1041  		},
  1042  		{
  1043  			Operation: TxnOperation_CREATE,
  1044  			Key:       prefixA + baseValue1 + "/item2",
  1045  			IsDerived: true,
  1046  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
  1047  			PrevState: ValueState_NONEXISTENT,
  1048  			NewState:  ValueState_OBTAINED,
  1049  		},
  1050  		{
  1051  			Operation: TxnOperation_CREATE,
  1052  			Key:       prefixB + baseValue2 + "/item2",
  1053  			IsDerived: true,
  1054  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
  1055  			PrevState: ValueState_NONEXISTENT,
  1056  			NewState:  ValueState_CONFIGURED,
  1057  		},
  1058  	}
  1059  	checkTxnOperations(txn.Planned, txnOps)
  1060  
  1061  	// -> executed operations
  1062  	txnOps = RecordedTxnOps{
  1063  		{
  1064  			Operation: TxnOperation_CREATE,
  1065  			Key:       prefixA + baseValue1,
  1066  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1067  			PrevState: ValueState_NONEXISTENT,
  1068  			NewState:  ValueState_OBTAINED,
  1069  		},
  1070  		{
  1071  			Operation: TxnOperation_CREATE,
  1072  			Key:       prefixB + baseValue2,
  1073  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1074  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1075  			PrevState: ValueState_PENDING,
  1076  			NewState:  ValueState_CONFIGURED,
  1077  		},
  1078  		{
  1079  			Operation: TxnOperation_CREATE,
  1080  			Key:       prefixB + baseValue2 + "/item1",
  1081  			IsDerived: true,
  1082  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item1")),
  1083  			PrevState: ValueState_NONEXISTENT,
  1084  			NewState:  ValueState_CONFIGURED,
  1085  		},
  1086  		{
  1087  			Operation: TxnOperation_CREATE,
  1088  			Key:       prefixC + baseValue3,
  1089  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("base-value3-data")),
  1090  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("base-value3-data")),
  1091  			PrevState: ValueState_PENDING,
  1092  			NewState:  ValueState_RETRYING,
  1093  			NewErr:    errors.New("failed to add value"),
  1094  		},
  1095  		{
  1096  			Operation: TxnOperation_CREATE,
  1097  			Key:       prefixA + baseValue1 + "/item1",
  1098  			IsDerived: true,
  1099  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item1")),
  1100  			PrevState: ValueState_NONEXISTENT,
  1101  			NewState:  ValueState_OBTAINED,
  1102  		},
  1103  		{
  1104  			Operation: TxnOperation_CREATE,
  1105  			Key:       prefixA + baseValue1 + "/item2",
  1106  			IsDerived: true,
  1107  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
  1108  			PrevState: ValueState_NONEXISTENT,
  1109  			NewState:  ValueState_OBTAINED,
  1110  		},
  1111  		{
  1112  			Operation: TxnOperation_CREATE,
  1113  			Key:       prefixB + baseValue2 + "/item2",
  1114  			IsDerived: true,
  1115  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
  1116  			PrevState: ValueState_NONEXISTENT,
  1117  			NewState:  ValueState_RETRYING,
  1118  			NewErr:    errors.New("failed to add derived value"),
  1119  		},
  1120  	}
  1121  	checkTxnOperations(txn.Executed, txnOps)
  1122  
  1123  	// check flag stats
  1124  	graphR := scheduler.graph.Read()
  1125  	errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil)
  1126  	Expect(errorStats.TotalCount).To(BeEquivalentTo(2))
  1127  	pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil)
  1128  	Expect(pendingStats.TotalCount).To(BeEquivalentTo(4))
  1129  	derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil)
  1130  	Expect(derivedStats.TotalCount).To(BeEquivalentTo(4))
  1131  	lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil)
  1132  	Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(9))
  1133  	descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil)
  1134  	Expect(descriptorStats.TotalCount).To(BeEquivalentTo(9))
  1135  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name))
  1136  	Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(3))
  1137  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor2Name))
  1138  	Expect(descriptorStats.PerValueCount[descriptor2Name]).To(BeEquivalentTo(4))
  1139  	Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor3Name))
  1140  	Expect(descriptorStats.PerValueCount[descriptor3Name]).To(BeEquivalentTo(2))
  1141  	valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil)
  1142  	Expect(valueStateStats.TotalCount).To(BeEquivalentTo(9))
  1143  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_OBTAINED.String()))
  1144  	Expect(valueStateStats.PerValueCount[ValueState_OBTAINED.String()]).To(BeEquivalentTo(3))
  1145  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String()))
  1146  	Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(2))
  1147  	Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_RETRYING.String()))
  1148  	Expect(valueStateStats.PerValueCount[ValueState_RETRYING.String()]).To(BeEquivalentTo(2))
  1149  	graphR.Release()
  1150  
  1151  	// item2 derived from baseValue2 should get fixed first
  1152  	startTime = time.Now()
  1153  	Eventually(prefBStatusChan, 3*time.Second).Should(Receive(&valueStatus))
  1154  	// TODO: do we want UPDATEs here? (or just CREATE since nothing has changed)
  1155  	checkBaseValueStatus(valueStatus, &BaseValueStatus{
  1156  		Value: &ValueStatus{
  1157  			Key:           prefixB + baseValue2,
  1158  			State:         ValueState_CONFIGURED,
  1159  			LastOperation: TxnOperation_UPDATE,
  1160  		},
  1161  		DerivedValues: []*ValueStatus{
  1162  			{
  1163  				Key:           prefixB + baseValue2 + "/item1",
  1164  				State:         ValueState_CONFIGURED,
  1165  				LastOperation: TxnOperation_UPDATE,
  1166  			},
  1167  			{
  1168  				Key:           prefixB + baseValue2 + "/item2",
  1169  				State:         ValueState_CONFIGURED,
  1170  				LastOperation: TxnOperation_CREATE,
  1171  			},
  1172  		},
  1173  	})
  1174  	stopTime = time.Now()
  1175  
  1176  	// check the state of SB
  1177  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
  1178  	// -> item2 derived from base value 2 is now created
  1179  	value = mockSB.GetValue(prefixB + baseValue2 + "/item2")
  1180  	Expect(value).ToNot(BeNil())
  1181  	Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue())
  1182  	Expect(value.Metadata).To(BeNil())
  1183  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
  1184  	Expect(mockSB.GetValues(nil)).To(HaveLen(3))
  1185  
  1186  	// check operations executed in SB
  1187  	opHistory = mockSB.PopHistoryOfOps()
  1188  	Expect(opHistory).To(HaveLen(2))
  1189  	operation = opHistory[0]
  1190  	Expect(operation.OpType).To(Equal(test.MockUpdate))
  1191  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
  1192  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2))
  1193  	Expect(operation.Err).To(BeNil())
  1194  	operation = opHistory[1]
  1195  	Expect(operation.OpType).To(Equal(test.MockCreate))
  1196  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name))
  1197  	Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2 + "/item2"))
  1198  	Expect(operation.Err).To(BeNil())
  1199  
  1200  	// check last transaction
  1201  	txnHistory = scheduler.GetTransactionHistory(startTime, time.Now())
  1202  	Expect(txnHistory).To(HaveLen(1))
  1203  	txn = txnHistory[0]
  1204  	Expect(txn.PreRecord).To(BeFalse())
  1205  	Expect(txn.Start.After(startTime)).To(BeTrue())
  1206  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
  1207  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
  1208  	Expect(txn.SeqNum).To(BeEquivalentTo(3))
  1209  	Expect(txn.TxnType).To(BeEquivalentTo(RetryFailedOps))
  1210  	Expect(txn.ResyncType).To(BeEquivalentTo(NotResync))
  1211  	Expect(txn.Description).To(BeEmpty())
  1212  	checkRecordedValues(txn.Values, []RecordedKVPair{
  1213  		{Key: prefixB + baseValue2, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB},
  1214  	})
  1215  	txnOps = RecordedTxnOps{
  1216  		{
  1217  			Operation: TxnOperation_UPDATE,
  1218  			Key:       prefixB + baseValue2,
  1219  			PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")),
  1220  			NewValue:  utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")),
  1221  			PrevState: ValueState_CONFIGURED,
  1222  			NewState:  ValueState_CONFIGURED,
  1223  			IsRetry:   true,
  1224  		},
  1225  		{
  1226  			Operation: TxnOperation_CREATE,
  1227  			Key:       prefixB + baseValue2 + "/item2",
  1228  			IsDerived: true,
  1229  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("item2")),
  1230  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("item2")),
  1231  			PrevState: ValueState_RETRYING,
  1232  			NewState:  ValueState_CONFIGURED,
  1233  			PrevErr:   errors.New("failed to add derived value"),
  1234  			IsRetry:   true,
  1235  		},
  1236  	}
  1237  	checkTxnOperations(txn.Planned, txnOps)
  1238  	checkTxnOperations(txn.Executed, txnOps)
  1239  
  1240  	// base-value3 should get fixed eventually as well
  1241  	startTime = time.Now()
  1242  	Eventually(prefCStatusChan, 5*time.Second).Should(Receive(&valueStatus))
  1243  	checkBaseValueStatus(valueStatus, &BaseValueStatus{
  1244  		Value: &ValueStatus{
  1245  			Key:           prefixC + baseValue3,
  1246  			State:         ValueState_CONFIGURED,
  1247  			LastOperation: TxnOperation_CREATE,
  1248  		},
  1249  	})
  1250  	stopTime = time.Now()
  1251  
  1252  	// check the state of SB
  1253  	Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty())
  1254  	// -> base value 3 is now created
  1255  	value = mockSB.GetValue(prefixC + baseValue3)
  1256  	Expect(value).ToNot(BeNil())
  1257  	Expect(proto.Equal(value.Value, test.NewStringValue("base-value3-data"))).To(BeTrue())
  1258  	Expect(value.Metadata).ToNot(BeNil())
  1259  	Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0))
  1260  	Expect(value.Origin).To(BeEquivalentTo(FromNB))
  1261  	Expect(mockSB.GetValues(nil)).To(HaveLen(4))
  1262  
  1263  	// check operations executed in SB
  1264  	opHistory = mockSB.PopHistoryOfOps()
  1265  	Expect(opHistory).To(HaveLen(1))
  1266  	operation = opHistory[0]
  1267  	Expect(operation.OpType).To(Equal(test.MockCreate))
  1268  	Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name))
  1269  	Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3))
  1270  	Expect(operation.Err).To(BeNil())
  1271  
  1272  	// check last transaction
  1273  	txnHistory = scheduler.GetTransactionHistory(startTime, time.Time{})
  1274  	Expect(txnHistory).To(HaveLen(1))
  1275  	txn = txnHistory[0]
  1276  	Expect(txn.PreRecord).To(BeFalse())
  1277  	Expect(txn.Start.After(startTime)).To(BeTrue())
  1278  	Expect(txn.Start.Before(txn.Stop)).To(BeTrue())
  1279  	Expect(txn.Stop.Before(stopTime)).To(BeTrue())
  1280  	Expect(txn.SeqNum).To(BeEquivalentTo(4))
  1281  	Expect(txn.TxnType).To(BeEquivalentTo(RetryFailedOps))
  1282  	Expect(txn.ResyncType).To(BeEquivalentTo(NotResync))
  1283  	Expect(txn.Description).To(BeEmpty())
  1284  	checkRecordedValues(txn.Values, []RecordedKVPair{
  1285  		{Key: prefixC + baseValue3, Value: utils.RecordProtoMessage(test.NewStringValue("base-value3-data")), Origin: FromNB},
  1286  	})
  1287  	txnOps = RecordedTxnOps{
  1288  		{
  1289  			Operation: TxnOperation_CREATE,
  1290  			Key:       prefixC + baseValue3,
  1291  			PrevValue: utils.RecordProtoMessage(test.NewStringValue("base-value3-data")),
  1292  			NewValue:  utils.RecordProtoMessage(test.NewStringValue("base-value3-data")),
  1293  			PrevState: ValueState_RETRYING,
  1294  			NewState:  ValueState_CONFIGURED,
  1295  			PrevErr:   errors.New("failed to add value"),
  1296  			IsRetry:   true,
  1297  		},
  1298  	}
  1299  	checkTxnOperations(txn.Planned, txnOps)
  1300  	checkTxnOperations(txn.Executed, txnOps)
  1301  
  1302  	// check metadata
  1303  	metadata, exists = nameToInteger1.LookupByName(baseValue1)
  1304  	Expect(exists).To(BeTrue())
  1305  	Expect(metadata.GetInteger()).To(BeEquivalentTo(10))
  1306  	metadata, exists = nameToInteger2.LookupByName(baseValue2)
  1307  	Expect(exists).To(BeTrue())
  1308  	Expect(metadata.GetInteger()).To(BeEquivalentTo(0))
  1309  	metadata, exists = nameToInteger3.LookupByName(baseValue3)
  1310  	Expect(exists).To(BeTrue())
  1311  	Expect(metadata.GetInteger()).To(BeEquivalentTo(0))
  1312  
  1313  	// close scheduler
  1314  	err = scheduler.Close()
  1315  	Expect(err).To(BeNil())
  1316  }