go.ligato.io/vpp-agent/v3@v3.5.0/plugins/kvscheduler/resync_test.go (about) 1 // Copyright (c) 2018 Cisco and/or its affiliates. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at: 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package kvscheduler 16 17 import ( 18 "errors" 19 "testing" 20 "time" 21 22 . "github.com/onsi/gomega" 23 24 "google.golang.org/protobuf/proto" 25 26 . "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api" 27 "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/internal/test" 28 "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/internal/utils" 29 . "go.ligato.io/vpp-agent/v3/proto/ligato/kvscheduler" 30 ) 31 32 func TestEmptyResync(t *testing.T) { 33 RegisterTestingT(t) 34 35 // prepare KV Scheduler 36 scheduler := NewPlugin(UseDeps(func(deps *Deps) { 37 deps.HTTPHandlers = nil 38 })) 39 err := scheduler.Init() 40 Expect(err).To(BeNil()) 41 42 // prepare mocks 43 mockSB := test.NewMockSouthbound() 44 descriptor1 := test.NewMockDescriptor(&KVDescriptor{ 45 Name: descriptor1Name, 46 NBKeyPrefix: prefixA, 47 KeySelector: prefixSelector(prefixA), 48 WithMetadata: true, 49 }, mockSB, 0) 50 51 // register descriptor with the scheduler 52 scheduler.RegisterKVDescriptor(descriptor1) 53 nbPrefixes := scheduler.GetRegisteredNBKeyPrefixes() 54 Expect(nbPrefixes).To(HaveLen(1)) 55 Expect(nbPrefixes).To(ContainElement(prefixA)) 56 57 // get metadata map created for the descriptor 58 metadataMap := scheduler.GetMetadataMap(descriptor1.Name) 59 _, withMetadataMap := metadataMap.(test.NameToInteger) 60 Expect(withMetadataMap).To(BeTrue()) 61 62 // transaction history should be initially empty 63 Expect(scheduler.GetTransactionHistory(time.Time{}, time.Time{})).To(BeEmpty()) 64 65 // run transaction with empty resync 66 startTime := time.Now() 67 ctx := WithResync(testCtx, FullResync, true) 68 description := "testing empty resync" 69 ctx = WithDescription(ctx, description) 70 seqNum, err := scheduler.StartNBTransaction().Commit(ctx) 71 stopTime := time.Now() 72 Expect(seqNum).To(BeEquivalentTo(0)) 73 Expect(err).ShouldNot(HaveOccurred()) 74 75 // check the state of SB 76 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 77 Expect(mockSB.GetValues(nil)).To(BeEmpty()) 78 79 // check metadata 80 Expect(metadataMap.ListAllNames()).To(BeEmpty()) 81 82 // check executed operations 83 opHistory := mockSB.PopHistoryOfOps() 84 Expect(opHistory).To(HaveLen(1)) 85 Expect(opHistory[0].OpType).To(Equal(test.MockRetrieve)) 86 Expect(opHistory[0].CorrelateRetrieve).To(BeEmpty()) 87 Expect(opHistory[0].Descriptor).To(BeEquivalentTo(descriptor1Name)) 88 89 // single transaction consisted of zero operations 90 txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Time{}) 91 Expect(txnHistory).To(HaveLen(1)) 92 txn := txnHistory[0] 93 Expect(txn.PreRecord).To(BeFalse()) 94 Expect(txn.Start.After(startTime)).To(BeTrue()) 95 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 96 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 97 Expect(txn.SeqNum).To(BeEquivalentTo(0)) 98 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 99 Expect(txn.RetryAttempt).To(BeEquivalentTo(0)) 100 Expect(txn.RetryForTxn).To(BeEquivalentTo(0)) 101 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 102 Expect(txn.Description).To(Equal(description)) 103 Expect(txn.Values).To(BeEmpty()) 104 Expect(txn.Planned).To(BeEmpty()) 105 Expect(txn.Executed).To(BeEmpty()) 106 107 // check flag stats 108 graphR := scheduler.graph.Read() 109 errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil) 110 Expect(errorStats.TotalCount).To(BeEquivalentTo(0)) 111 pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil) 112 Expect(pendingStats.TotalCount).To(BeEquivalentTo(0)) 113 derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil) 114 Expect(derivedStats.TotalCount).To(BeEquivalentTo(0)) 115 lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil) 116 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(0)) 117 descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil) 118 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(0)) 119 valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil) 120 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(0)) 121 122 graphR.Release() 123 124 // close scheduler 125 err = scheduler.Close() 126 Expect(err).To(BeNil()) 127 } 128 129 func TestResyncWithEmptySB(t *testing.T) { 130 RegisterTestingT(t) 131 132 // prepare KV Scheduler 133 scheduler := NewPlugin(UseDeps(func(deps *Deps) { 134 deps.HTTPHandlers = nil 135 })) 136 err := scheduler.Init() 137 Expect(err).To(BeNil()) 138 139 // prepare mocks 140 mockSB := test.NewMockSouthbound() 141 descriptor1 := test.NewMockDescriptor(&KVDescriptor{ 142 Name: descriptor1Name, 143 NBKeyPrefix: prefixA, 144 KeySelector: prefixSelector(prefixA), 145 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 146 DerivedValues: test.ArrayValueDerBuilder, 147 Dependencies: func(key string, value proto.Message) []Dependency { 148 if key == prefixA+baseValue2 { 149 depKey := prefixA + baseValue1 + "/item1" // base value depends on a derived value 150 return []Dependency{ 151 {Label: depKey, Key: depKey}, 152 } 153 } 154 if key == prefixA+baseValue1+"/item2" { 155 depKey := prefixA + baseValue2 + "/item1" // derived value depends on another derived value 156 return []Dependency{ 157 {Label: depKey, Key: depKey}, 158 } 159 } 160 return nil 161 }, 162 WithMetadata: true, 163 }, mockSB, 0) 164 165 // register descriptor with the scheduler 166 scheduler.RegisterKVDescriptor(descriptor1) 167 168 // get metadata map created for the descriptor 169 metadataMap := scheduler.GetMetadataMap(descriptor1.Name) 170 nameToInteger, withMetadataMap := metadataMap.(test.NameToInteger) 171 Expect(withMetadataMap).To(BeTrue()) 172 173 // run resync transaction with empty SB 174 startTime := time.Now() 175 schedulerTxn := scheduler.StartNBTransaction() 176 schedulerTxn.SetValue(prefixA+baseValue2, test.NewArrayValue("item1")) 177 schedulerTxn.SetValue(prefixA+baseValue1, test.NewArrayValue("item1", "item2")) 178 ctx := WithResync(testCtx, FullResync, true) 179 description := "testing resync against empty SB" 180 ctx = WithDescription(ctx, description) 181 seqNum, err := schedulerTxn.Commit(ctx) 182 stopTime := time.Now() 183 Expect(seqNum).To(BeEquivalentTo(0)) 184 Expect(err).ShouldNot(HaveOccurred()) 185 186 // check the state of SB 187 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 188 // -> base value 1 189 value := mockSB.GetValue(prefixA + baseValue1) 190 Expect(value).ToNot(BeNil()) 191 Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue()) 192 Expect(value.Metadata).ToNot(BeNil()) 193 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 194 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 195 // -> item1 derived from base value 1 196 value = mockSB.GetValue(prefixA + baseValue1 + "/item1") 197 Expect(value).ToNot(BeNil()) 198 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 199 Expect(value.Metadata).To(BeNil()) 200 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 201 // -> item2 derived from base value 1 202 value = mockSB.GetValue(prefixA + baseValue1 + "/item2") 203 Expect(value).ToNot(BeNil()) 204 Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue()) 205 Expect(value.Metadata).To(BeNil()) 206 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 207 // -> base value 2 208 value = mockSB.GetValue(prefixA + baseValue2) 209 Expect(value).ToNot(BeNil()) 210 Expect(proto.Equal(value.Value, test.NewArrayValue("item1"))).To(BeTrue()) 211 Expect(value.Metadata).ToNot(BeNil()) 212 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(1)) 213 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 214 // -> item1 derived from base value 2 215 value = mockSB.GetValue(prefixA + baseValue2 + "/item1") 216 Expect(value).ToNot(BeNil()) 217 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 218 Expect(value.Metadata).To(BeNil()) 219 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 220 Expect(mockSB.GetValues(nil)).To(HaveLen(5)) 221 222 // check metadata 223 metadata, exists := nameToInteger.LookupByName(baseValue1) 224 Expect(exists).To(BeTrue()) 225 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 226 metadata, exists = nameToInteger.LookupByName(baseValue2) 227 Expect(exists).To(BeTrue()) 228 Expect(metadata.GetInteger()).To(BeEquivalentTo(1)) 229 230 // check executed operations 231 opHistory := mockSB.PopHistoryOfOps() 232 Expect(opHistory).To(HaveLen(6)) 233 operation := opHistory[0] 234 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 235 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 236 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 237 { 238 Key: prefixA + baseValue1, 239 Value: test.NewArrayValue("item1", "item2"), 240 Metadata: nil, 241 Origin: FromNB, 242 }, 243 { 244 Key: prefixA + baseValue2, 245 Value: test.NewArrayValue("item1"), 246 Metadata: nil, 247 Origin: FromNB, 248 }, 249 }) 250 operation = opHistory[1] 251 Expect(operation.OpType).To(Equal(test.MockCreate)) 252 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 253 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1)) 254 Expect(operation.Err).To(BeNil()) 255 operation = opHistory[2] 256 Expect(operation.OpType).To(Equal(test.MockCreate)) 257 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 258 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item1")) 259 Expect(operation.Err).To(BeNil()) 260 operation = opHistory[3] 261 Expect(operation.OpType).To(Equal(test.MockCreate)) 262 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 263 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2)) 264 Expect(operation.Err).To(BeNil()) 265 operation = opHistory[4] 266 Expect(operation.OpType).To(Equal(test.MockCreate)) 267 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 268 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2 + "/item1")) 269 Expect(operation.Err).To(BeNil()) 270 operation = opHistory[5] 271 Expect(operation.OpType).To(Equal(test.MockCreate)) 272 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 273 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 274 Expect(operation.Err).To(BeNil()) 275 276 // check value dumps 277 expValues := []KVWithMetadata{ 278 {Key: prefixA + baseValue1, Value: test.NewArrayValue("item1", "item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 0}}, 279 {Key: prefixA + baseValue2, Value: test.NewArrayValue("item1"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 1}}, 280 } 281 views := []View{NBView, SBView, CachedView} 282 for _, view := range views { 283 dumpedValues, err := scheduler.DumpValuesByKeyPrefix(prefixA, view) 284 Expect(err).To(BeNil()) 285 checkValues(dumpedValues, expValues) 286 } 287 for _, view := range views { 288 dumpedValues, err := scheduler.DumpValuesByDescriptor(descriptor1Name, view) 289 Expect(err).To(BeNil()) 290 checkValues(dumpedValues, expValues) 291 } 292 mockSB.PopHistoryOfOps() // remove Retrieve-s from the history 293 294 // check value states 295 status := scheduler.GetValueStatus(prefixA + baseValue1) 296 Expect(status).ToNot(BeNil()) 297 checkBaseValueStatus(status, &BaseValueStatus{ 298 Value: &ValueStatus{ 299 Key: prefixA + baseValue1, 300 State: ValueState_CONFIGURED, 301 LastOperation: TxnOperation_CREATE, 302 }, 303 DerivedValues: []*ValueStatus{ 304 { 305 Key: prefixA + baseValue1 + "/item1", 306 State: ValueState_CONFIGURED, 307 LastOperation: TxnOperation_CREATE, 308 }, 309 { 310 Key: prefixA + baseValue1 + "/item2", 311 State: ValueState_CONFIGURED, 312 LastOperation: TxnOperation_CREATE, 313 }, 314 }, 315 }) 316 status = scheduler.GetValueStatus(prefixA + baseValue2) 317 Expect(status).ToNot(BeNil()) 318 checkBaseValueStatus(status, &BaseValueStatus{ 319 Value: &ValueStatus{ 320 Key: prefixA + baseValue2, 321 State: ValueState_CONFIGURED, 322 LastOperation: TxnOperation_CREATE, 323 }, 324 DerivedValues: []*ValueStatus{ 325 { 326 Key: prefixA + baseValue2 + "/item1", 327 State: ValueState_CONFIGURED, 328 LastOperation: TxnOperation_CREATE, 329 }, 330 }, 331 }) 332 333 // single transaction consisted of 6 operations 334 txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Now()) 335 Expect(txnHistory).To(HaveLen(1)) 336 txn := txnHistory[0] 337 Expect(txn.PreRecord).To(BeFalse()) 338 Expect(txn.Start.After(startTime)).To(BeTrue()) 339 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 340 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 341 Expect(txn.SeqNum).To(BeEquivalentTo(0)) 342 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 343 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 344 Expect(txn.Description).To(Equal(description)) 345 checkRecordedValues(txn.Values, []RecordedKVPair{ 346 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 347 {Key: prefixA + baseValue2, Value: utils.RecordProtoMessage(test.NewArrayValue("item1")), Origin: FromNB}, 348 }) 349 350 txnOps := RecordedTxnOps{ 351 { 352 Operation: TxnOperation_CREATE, 353 Key: prefixA + baseValue1, 354 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 355 PrevState: ValueState_NONEXISTENT, 356 NewState: ValueState_CONFIGURED, 357 }, 358 { 359 Operation: TxnOperation_CREATE, 360 Key: prefixA + baseValue1 + "/item1", 361 IsDerived: true, 362 NewValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 363 PrevState: ValueState_NONEXISTENT, 364 NewState: ValueState_CONFIGURED, 365 }, 366 { 367 Operation: TxnOperation_CREATE, 368 Key: prefixA + baseValue2, 369 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 370 PrevState: ValueState_NONEXISTENT, 371 NewState: ValueState_CONFIGURED, 372 }, 373 { 374 Operation: TxnOperation_CREATE, 375 Key: prefixA + baseValue2 + "/item1", 376 IsDerived: true, 377 NewValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 378 PrevState: ValueState_NONEXISTENT, 379 NewState: ValueState_CONFIGURED, 380 }, 381 { 382 Operation: TxnOperation_CREATE, 383 Key: prefixA + baseValue1 + "/item2", 384 IsDerived: true, 385 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 386 PrevState: ValueState_NONEXISTENT, 387 NewState: ValueState_CONFIGURED, 388 }, 389 } 390 checkTxnOperations(txn.Planned, txnOps) 391 checkTxnOperations(txn.Executed, txnOps) 392 393 // now remove everything using resync with empty data 394 startTime = time.Now() 395 seqNum, err = scheduler.StartNBTransaction().Commit(WithResync(testCtx, FullResync, true)) 396 stopTime = time.Now() 397 Expect(seqNum).To(BeEquivalentTo(1)) 398 Expect(err).ShouldNot(HaveOccurred()) 399 400 // check the state of SB 401 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 402 Expect(mockSB.GetValues(nil)).To(BeEmpty()) 403 404 // check metadata 405 Expect(metadataMap.ListAllNames()).To(BeEmpty()) 406 407 // check executed operations 408 opHistory = mockSB.PopHistoryOfOps() 409 Expect(opHistory).To(HaveLen(6)) 410 operation = opHistory[0] 411 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 412 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 413 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 414 { 415 Key: prefixA + baseValue1, 416 Value: test.NewArrayValue("item1", "item2"), 417 Metadata: &test.OnlyInteger{Integer: 0}, 418 Origin: FromNB, 419 }, 420 { 421 Key: prefixA + baseValue2, 422 Value: test.NewArrayValue("item1"), 423 Metadata: &test.OnlyInteger{Integer: 1}, 424 Origin: FromNB, 425 }, 426 }) 427 operation = opHistory[1] 428 Expect(operation.OpType).To(Equal(test.MockDelete)) 429 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 430 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 431 Expect(operation.Err).To(BeNil()) 432 operation = opHistory[2] 433 Expect(operation.OpType).To(Equal(test.MockDelete)) 434 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 435 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2 + "/item1")) 436 Expect(operation.Err).To(BeNil()) 437 operation = opHistory[3] 438 Expect(operation.OpType).To(Equal(test.MockDelete)) 439 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 440 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2)) 441 Expect(operation.Err).To(BeNil()) 442 operation = opHistory[4] 443 Expect(operation.OpType).To(Equal(test.MockDelete)) 444 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 445 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item1")) 446 Expect(operation.Err).To(BeNil()) 447 operation = opHistory[5] 448 Expect(operation.OpType).To(Equal(test.MockDelete)) 449 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 450 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1)) 451 Expect(operation.Err).To(BeNil()) 452 453 // this second transaction consisted of 6 operations 454 txnHistory = scheduler.GetTransactionHistory(time.Time{}, time.Now()) 455 Expect(txnHistory).To(HaveLen(2)) 456 txn = txnHistory[1] 457 Expect(txn.PreRecord).To(BeFalse()) 458 Expect(txn.Start.After(startTime)).To(BeTrue()) 459 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 460 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 461 Expect(txn.SeqNum).To(BeEquivalentTo(1)) 462 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 463 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 464 Expect(txn.Description).To(BeEmpty()) 465 checkRecordedValues(txn.Values, []RecordedKVPair{ 466 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(nil), Origin: FromNB}, 467 {Key: prefixA + baseValue2, Value: utils.RecordProtoMessage(nil), Origin: FromNB}, 468 }) 469 470 txnOps = RecordedTxnOps{ 471 { 472 Operation: TxnOperation_DELETE, 473 Key: prefixA + baseValue1 + "/item2", 474 IsDerived: true, 475 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 476 PrevState: ValueState_CONFIGURED, 477 NewState: ValueState_REMOVED, 478 }, 479 { 480 Operation: TxnOperation_DELETE, 481 Key: prefixA + baseValue2 + "/item1", 482 IsDerived: true, 483 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 484 PrevState: ValueState_CONFIGURED, 485 NewState: ValueState_REMOVED, 486 }, 487 { 488 Operation: TxnOperation_DELETE, 489 Key: prefixA + baseValue2, 490 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 491 PrevState: ValueState_CONFIGURED, 492 NewState: ValueState_REMOVED, 493 }, 494 { 495 Operation: TxnOperation_DELETE, 496 Key: prefixA + baseValue1 + "/item1", 497 IsDerived: true, 498 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 499 PrevState: ValueState_CONFIGURED, 500 NewState: ValueState_REMOVED, 501 }, 502 { 503 Operation: TxnOperation_DELETE, 504 Key: prefixA + baseValue1, 505 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 506 PrevState: ValueState_CONFIGURED, 507 NewState: ValueState_REMOVED, 508 }, 509 } 510 checkTxnOperations(txn.Planned, txnOps) 511 checkTxnOperations(txn.Executed, txnOps) 512 513 // check flag stats 514 // Note: removed derived values are not kept in the graph 515 graphR := scheduler.graph.Read() 516 errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil) 517 Expect(errorStats.TotalCount).To(BeEquivalentTo(0)) 518 pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil) 519 Expect(pendingStats.TotalCount).To(BeEquivalentTo(2)) 520 derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil) 521 Expect(derivedStats.TotalCount).To(BeEquivalentTo(3)) 522 lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil) 523 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(7)) 524 descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil) 525 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(7)) 526 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name)) 527 Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(7)) 528 valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil) 529 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(7)) 530 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String())) 531 Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(5)) 532 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_REMOVED.String())) 533 Expect(valueStateStats.PerValueCount[ValueState_REMOVED.String()]).To(BeEquivalentTo(2)) 534 graphR.Release() 535 536 // close scheduler 537 err = scheduler.Close() 538 Expect(err).To(BeNil()) 539 } 540 541 func TestResyncWithNonEmptySB(t *testing.T) { 542 RegisterTestingT(t) 543 544 // prepare KV Scheduler 545 scheduler := NewPlugin(UseDeps(func(deps *Deps) { 546 deps.HTTPHandlers = nil 547 })) 548 err := scheduler.Init() 549 Expect(err).To(BeNil()) 550 551 // prepare mocks 552 mockSB := test.NewMockSouthbound() 553 // -> initial content: 554 mockSB.SetValue(prefixA+baseValue1, test.NewArrayValue("item1"), 555 &test.OnlyInteger{Integer: 0}, FromNB, false) 556 mockSB.SetValue(prefixA+baseValue1+"/item1", test.NewStringValue("item1"), 557 nil, FromNB, true) 558 mockSB.SetValue(prefixA+baseValue2, test.NewArrayValue("item1"), 559 &test.OnlyInteger{Integer: 1}, FromNB, false) 560 mockSB.SetValue(prefixA+baseValue2+"/item1", test.NewStringValue("item1"), 561 nil, FromNB, true) 562 mockSB.SetValue(prefixA+baseValue3, test.NewArrayValue("item1"), 563 &test.OnlyInteger{Integer: 2}, FromNB, false) 564 mockSB.SetValue(prefixA+baseValue3+"/item1", test.NewStringValue("item1"), 565 nil, FromNB, true) 566 // -> descriptor1: 567 descriptor1 := test.NewMockDescriptor(&KVDescriptor{ 568 Name: descriptor1Name, 569 NBKeyPrefix: prefixA, 570 KeySelector: prefixSelector(prefixA), 571 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 572 DerivedValues: test.ArrayValueDerBuilder, 573 Dependencies: func(key string, value proto.Message) []Dependency { 574 if key == prefixA+baseValue2+"/item1" { 575 depKey := prefixA + baseValue1 576 return []Dependency{ 577 {Label: depKey, Key: depKey}, 578 } 579 } 580 if key == prefixA+baseValue2+"/item2" { 581 depKey := prefixA + baseValue1 + "/item1" 582 return []Dependency{ 583 {Label: depKey, Key: depKey}, 584 } 585 } 586 return nil 587 }, 588 UpdateWithRecreate: func(key string, oldValue, newValue proto.Message, metadata Metadata) bool { 589 return key == prefixA+baseValue3 590 }, 591 WithMetadata: true, 592 }, mockSB, 3) 593 594 // register descriptor with the scheduler 595 scheduler.RegisterKVDescriptor(descriptor1) 596 597 // get metadata map created for the descriptor 598 metadataMap := scheduler.GetMetadataMap(descriptor1.Name) 599 nameToInteger, withMetadataMap := metadataMap.(test.NameToInteger) 600 Expect(withMetadataMap).To(BeTrue()) 601 602 // run resync transaction with SB that already has some values added 603 startTime := time.Now() 604 schedulerTxn := scheduler.StartNBTransaction() 605 schedulerTxn.SetValue(prefixA+baseValue1, test.NewArrayValue("item2")) 606 schedulerTxn.SetValue(prefixA+baseValue2, test.NewArrayValue("item1", "item2")) 607 schedulerTxn.SetValue(prefixA+baseValue3, test.NewArrayValue("item1", "item2")) 608 seqNum, err := schedulerTxn.Commit(WithResync(testCtx, FullResync, true)) 609 stopTime := time.Now() 610 Expect(seqNum).To(BeEquivalentTo(0)) 611 Expect(err).ShouldNot(HaveOccurred()) 612 613 // check the state of SB 614 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 615 // -> base value 1 616 value := mockSB.GetValue(prefixA + baseValue1) 617 Expect(value).ToNot(BeNil()) 618 Expect(proto.Equal(value.Value, test.NewArrayValue("item2"))).To(BeTrue()) 619 Expect(value.Metadata).ToNot(BeNil()) 620 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 621 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 622 // -> item1 derived from base value 1 was removed 623 value = mockSB.GetValue(prefixA + baseValue1 + "/item1") 624 Expect(value).To(BeNil()) 625 // -> item2 derived from base value 1 626 value = mockSB.GetValue(prefixA + baseValue1 + "/item2") 627 Expect(value).ToNot(BeNil()) 628 Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue()) 629 Expect(value.Metadata).To(BeNil()) 630 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 631 // -> base value 2 632 value = mockSB.GetValue(prefixA + baseValue2) 633 Expect(value).ToNot(BeNil()) 634 Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue()) 635 Expect(value.Metadata).ToNot(BeNil()) 636 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(1)) 637 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 638 // -> item1 derived from base value 2 639 value = mockSB.GetValue(prefixA + baseValue2 + "/item1") 640 Expect(value).ToNot(BeNil()) 641 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 642 Expect(value.Metadata).To(BeNil()) 643 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 644 // -> item2 derived from base value 2 is pending 645 value = mockSB.GetValue(prefixA + baseValue2 + "/item2") 646 Expect(value).To(BeNil()) 647 // -> base value 3 648 value = mockSB.GetValue(prefixA + baseValue3) 649 Expect(value).ToNot(BeNil()) 650 Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue()) 651 Expect(value.Metadata).ToNot(BeNil()) 652 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(3)) 653 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 654 // -> item1 derived from base value 3 655 value = mockSB.GetValue(prefixA + baseValue3 + "/item1") 656 Expect(value).ToNot(BeNil()) 657 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 658 Expect(value.Metadata).To(BeNil()) 659 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 660 // -> item2 derived from base value 3 661 value = mockSB.GetValue(prefixA + baseValue3 + "/item2") 662 Expect(value).ToNot(BeNil()) 663 Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue()) 664 Expect(value.Metadata).To(BeNil()) 665 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 666 Expect(mockSB.GetValues(nil)).To(HaveLen(7)) 667 668 // check metadata 669 metadata, exists := nameToInteger.LookupByName(baseValue1) 670 Expect(exists).To(BeTrue()) 671 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 672 metadata, exists = nameToInteger.LookupByName(baseValue2) 673 Expect(exists).To(BeTrue()) 674 Expect(metadata.GetInteger()).To(BeEquivalentTo(1)) 675 metadata, exists = nameToInteger.LookupByName(baseValue3) 676 Expect(exists).To(BeTrue()) 677 Expect(metadata.GetInteger()).To(BeEquivalentTo(3)) 678 679 // check operations executed in SB 680 opHistory := mockSB.PopHistoryOfOps() 681 Expect(opHistory).To(HaveLen(10)) 682 operation := opHistory[0] 683 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 684 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 685 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 686 { 687 Key: prefixA + baseValue1, 688 Value: test.NewArrayValue("item2"), 689 Metadata: nil, 690 Origin: FromNB, 691 }, 692 { 693 Key: prefixA + baseValue2, 694 Value: test.NewArrayValue("item1", "item2"), 695 Metadata: nil, 696 Origin: FromNB, 697 }, 698 { 699 Key: prefixA + baseValue3, 700 Value: test.NewArrayValue("item1", "item2"), 701 Metadata: nil, 702 Origin: FromNB, 703 }, 704 }) 705 operation = opHistory[1] 706 Expect(operation.OpType).To(Equal(test.MockDelete)) 707 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 708 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue3 + "/item1")) 709 Expect(operation.Err).To(BeNil()) 710 operation = opHistory[2] 711 Expect(operation.OpType).To(Equal(test.MockDelete)) 712 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 713 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue3)) 714 Expect(operation.Err).To(BeNil()) 715 operation = opHistory[3] 716 Expect(operation.OpType).To(Equal(test.MockCreate)) 717 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 718 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue3)) 719 Expect(operation.Err).To(BeNil()) 720 operation = opHistory[4] 721 Expect(operation.OpType).To(Equal(test.MockCreate)) 722 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 723 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue3 + "/item1")) 724 Expect(operation.Err).To(BeNil()) 725 operation = opHistory[5] 726 Expect(operation.OpType).To(Equal(test.MockCreate)) 727 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 728 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue3 + "/item2")) 729 Expect(operation.Err).To(BeNil()) 730 operation = opHistory[6] 731 Expect(operation.OpType).To(Equal(test.MockDelete)) 732 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 733 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item1")) 734 Expect(operation.Err).To(BeNil()) 735 operation = opHistory[7] 736 Expect(operation.OpType).To(Equal(test.MockUpdate)) 737 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 738 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1)) 739 Expect(operation.Err).To(BeNil()) 740 operation = opHistory[8] 741 Expect(operation.OpType).To(Equal(test.MockCreate)) 742 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 743 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 744 Expect(operation.Err).To(BeNil()) 745 operation = opHistory[9] 746 Expect(operation.OpType).To(Equal(test.MockUpdate)) 747 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 748 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2)) 749 Expect(operation.Err).To(BeNil()) 750 751 // check value dumps 752 // TODO: actual configuration (when kept in-graph separately) will have to not include 753 // baseValue2/item2, but that means we will have to refresh the graph 754 // always after something (base or derived value) is found to be pending 755 expValues := []KVWithMetadata{ 756 {Key: prefixA + baseValue1, Value: test.NewArrayValue("item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 0}}, 757 {Key: prefixA + baseValue2, Value: test.NewArrayValue("item1", "item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 1}}, 758 {Key: prefixA + baseValue3, Value: test.NewArrayValue("item1", "item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 3}}, 759 } 760 views := []View{NBView, SBView, CachedView} 761 for _, view := range views { 762 dumpedValues, err := scheduler.DumpValuesByKeyPrefix(prefixA, view) 763 Expect(err).To(BeNil()) 764 checkValues(dumpedValues, expValues) 765 dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor1Name, view) 766 Expect(err).To(BeNil()) 767 checkValues(dumpedValues, expValues) 768 } 769 mockSB.PopHistoryOfOps() // remove Retrieve-s from the history 770 771 // check value states 772 status := scheduler.GetValueStatus(prefixA + baseValue1) 773 Expect(status).ToNot(BeNil()) 774 checkBaseValueStatus(status, &BaseValueStatus{ 775 Value: &ValueStatus{ 776 Key: prefixA + baseValue1, 777 State: ValueState_CONFIGURED, 778 LastOperation: TxnOperation_UPDATE, 779 }, 780 DerivedValues: []*ValueStatus{ 781 { 782 Key: prefixA + baseValue1 + "/item2", 783 State: ValueState_CONFIGURED, 784 LastOperation: TxnOperation_CREATE, 785 }, 786 }, 787 }) 788 status = scheduler.GetValueStatus(prefixA + baseValue2) 789 Expect(status).ToNot(BeNil()) 790 checkBaseValueStatus(status, &BaseValueStatus{ 791 Value: &ValueStatus{ 792 Key: prefixA + baseValue2, 793 State: ValueState_CONFIGURED, 794 LastOperation: TxnOperation_UPDATE, 795 }, 796 DerivedValues: []*ValueStatus{ 797 { 798 Key: prefixA + baseValue2 + "/item1", 799 State: ValueState_CONFIGURED, 800 LastOperation: TxnOperation_UPDATE, 801 }, 802 { 803 Key: prefixA + baseValue2 + "/item2", 804 State: ValueState_PENDING, 805 LastOperation: TxnOperation_CREATE, 806 Details: []string{prefixA + baseValue1 + "/item1"}, 807 }, 808 }, 809 }) 810 status = scheduler.GetValueStatus(prefixA + baseValue3) 811 Expect(status).ToNot(BeNil()) 812 checkBaseValueStatus(status, &BaseValueStatus{ 813 Value: &ValueStatus{ 814 Key: prefixA + baseValue3, 815 State: ValueState_CONFIGURED, 816 LastOperation: TxnOperation_UPDATE, 817 }, 818 DerivedValues: []*ValueStatus{ 819 { 820 Key: prefixA + baseValue3 + "/item1", 821 State: ValueState_CONFIGURED, 822 LastOperation: TxnOperation_CREATE, 823 }, 824 { 825 Key: prefixA + baseValue3 + "/item2", 826 State: ValueState_CONFIGURED, 827 LastOperation: TxnOperation_CREATE, 828 }, 829 }, 830 }) 831 832 // check transaction operations 833 txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Time{}) 834 Expect(txnHistory).To(HaveLen(1)) 835 txn := txnHistory[0] 836 Expect(txn.PreRecord).To(BeFalse()) 837 Expect(txn.Start.After(startTime)).To(BeTrue()) 838 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 839 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 840 Expect(txn.SeqNum).To(BeEquivalentTo(0)) 841 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 842 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 843 Expect(txn.Description).To(BeEmpty()) 844 checkRecordedValues(txn.Values, []RecordedKVPair{ 845 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item2")), Origin: FromNB}, 846 {Key: prefixA + baseValue2, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 847 {Key: prefixA + baseValue3, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 848 }) 849 850 txnOps := RecordedTxnOps{ 851 { 852 Operation: TxnOperation_DELETE, 853 Key: prefixA + baseValue3 + "/item1", 854 IsDerived: true, 855 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 856 PrevState: ValueState_DISCOVERED, 857 NewState: ValueState_REMOVED, 858 IsRecreate: true, 859 }, 860 { 861 Operation: TxnOperation_DELETE, 862 Key: prefixA + baseValue3, 863 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 864 PrevState: ValueState_DISCOVERED, 865 NewState: ValueState_REMOVED, 866 IsRecreate: true, 867 }, 868 { 869 Operation: TxnOperation_CREATE, 870 Key: prefixA + baseValue3, 871 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 872 PrevState: ValueState_REMOVED, 873 NewState: ValueState_CONFIGURED, 874 IsRecreate: true, 875 }, 876 { 877 Operation: TxnOperation_CREATE, 878 Key: prefixA + baseValue3 + "/item1", 879 IsDerived: true, 880 NewValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 881 PrevState: ValueState_NONEXISTENT, // TODO: derived value removed from the graph, ok? 882 NewState: ValueState_CONFIGURED, 883 IsRecreate: true, 884 }, 885 { 886 Operation: TxnOperation_CREATE, 887 Key: prefixA + baseValue3 + "/item2", 888 IsDerived: true, 889 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 890 PrevState: ValueState_NONEXISTENT, 891 NewState: ValueState_CONFIGURED, 892 }, 893 { 894 Operation: TxnOperation_DELETE, 895 Key: prefixA + baseValue1 + "/item1", 896 IsDerived: true, 897 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 898 PrevState: ValueState_DISCOVERED, 899 NewState: ValueState_REMOVED, 900 }, 901 { 902 Operation: TxnOperation_UPDATE, 903 Key: prefixA + baseValue1, 904 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 905 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item2")), 906 PrevState: ValueState_DISCOVERED, 907 NewState: ValueState_CONFIGURED, 908 }, 909 { 910 Operation: TxnOperation_CREATE, 911 Key: prefixA + baseValue1 + "/item2", 912 IsDerived: true, 913 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 914 PrevState: ValueState_NONEXISTENT, 915 NewState: ValueState_CONFIGURED, 916 }, 917 { 918 Operation: TxnOperation_UPDATE, 919 Key: prefixA + baseValue2, 920 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 921 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 922 PrevState: ValueState_DISCOVERED, 923 NewState: ValueState_CONFIGURED, 924 }, 925 { 926 Operation: TxnOperation_CREATE, 927 Key: prefixA + baseValue2 + "/item2", 928 IsDerived: true, 929 NOOP: true, 930 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 931 PrevState: ValueState_NONEXISTENT, 932 NewState: ValueState_PENDING, 933 }, 934 } 935 checkTxnOperations(txn.Planned, txnOps) 936 checkTxnOperations(txn.Executed, txnOps) 937 938 // check flag stats 939 graphR := scheduler.graph.Read() 940 errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil) 941 Expect(errorStats.TotalCount).To(BeEquivalentTo(0)) 942 pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil) 943 Expect(pendingStats.TotalCount).To(BeEquivalentTo(1)) 944 derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil) 945 Expect(derivedStats.TotalCount).To(BeEquivalentTo(5)) 946 lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil) 947 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(8)) 948 descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil) 949 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(8)) 950 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name)) 951 Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(8)) 952 valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil) 953 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(8)) 954 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String())) 955 Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(7)) 956 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_PENDING.String())) 957 Expect(valueStateStats.PerValueCount[ValueState_PENDING.String()]).To(BeEquivalentTo(1)) 958 graphR.Release() 959 960 // close scheduler 961 err = scheduler.Close() 962 Expect(err).To(BeNil()) 963 } 964 965 func TestResyncNotRemovingSBValues(t *testing.T) { 966 RegisterTestingT(t) 967 968 // prepare KV Scheduler 969 scheduler := NewPlugin(UseDeps(func(deps *Deps) { 970 deps.HTTPHandlers = nil 971 })) 972 err := scheduler.Init() 973 Expect(err).To(BeNil()) 974 975 // prepare mocks 976 mockSB := test.NewMockSouthbound() 977 // -> initial content: 978 mockSB.SetValue(prefixA+baseValue1, test.NewStringValue(baseValue1), 979 nil, FromSB, false) 980 // -> descriptor1: 981 descriptor1 := test.NewMockDescriptor(&KVDescriptor{ 982 Name: descriptor1Name, 983 KeySelector: prefixSelector(prefixA), 984 NBKeyPrefix: prefixA, 985 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 986 DerivedValues: test.ArrayValueDerBuilder, 987 Dependencies: func(key string, value proto.Message) []Dependency { 988 if key == prefixA+baseValue2 { 989 depKey := prefixA + baseValue1 990 return []Dependency{ 991 {Label: depKey, Key: depKey}, 992 } 993 } 994 return nil 995 }, 996 WithMetadata: true, 997 }, mockSB, 0) 998 999 // register descriptor with the scheduler 1000 scheduler.RegisterKVDescriptor(descriptor1) 1001 1002 // get metadata map created for the descriptor 1003 metadataMap := scheduler.GetMetadataMap(descriptor1.Name) 1004 nameToInteger, withMetadataMap := metadataMap.(test.NameToInteger) 1005 Expect(withMetadataMap).To(BeTrue()) 1006 1007 // run resync transaction that should keep values not managed by NB untouched 1008 startTime := time.Now() 1009 schedulerTxn := scheduler.StartNBTransaction() 1010 schedulerTxn.SetValue(prefixA+baseValue2, test.NewArrayValue("item1")) 1011 seqNum, err := schedulerTxn.Commit(WithResync(testCtx, FullResync, true)) 1012 stopTime := time.Now() 1013 Expect(seqNum).To(BeEquivalentTo(0)) 1014 Expect(err).ShouldNot(HaveOccurred()) 1015 1016 // check the state of SB 1017 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 1018 // -> base value 1 1019 value := mockSB.GetValue(prefixA + baseValue1) 1020 Expect(value).ToNot(BeNil()) 1021 Expect(proto.Equal(value.Value, test.NewStringValue(baseValue1))).To(BeTrue()) 1022 Expect(value.Metadata).To(BeNil()) 1023 Expect(value.Origin).To(BeEquivalentTo(FromSB)) 1024 // -> base value 2 1025 value = mockSB.GetValue(prefixA + baseValue2) 1026 Expect(value).ToNot(BeNil()) 1027 Expect(proto.Equal(value.Value, test.NewArrayValue("item1"))).To(BeTrue()) 1028 Expect(value.Metadata).ToNot(BeNil()) 1029 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 1030 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1031 // -> item1 derived from base value 2 1032 value = mockSB.GetValue(prefixA + baseValue2 + "/item1") 1033 Expect(value).ToNot(BeNil()) 1034 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 1035 Expect(value.Metadata).To(BeNil()) 1036 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1037 Expect(mockSB.GetValues(nil)).To(HaveLen(3)) 1038 1039 // check metadata 1040 metadata, exists := nameToInteger.LookupByName(baseValue1) 1041 Expect(exists).To(BeFalse()) 1042 Expect(metadata).To(BeNil()) 1043 metadata, exists = nameToInteger.LookupByName(baseValue2) 1044 Expect(exists).To(BeTrue()) 1045 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 1046 1047 // check operations executed in SB 1048 opHistory := mockSB.PopHistoryOfOps() 1049 Expect(opHistory).To(HaveLen(3)) 1050 operation := opHistory[0] 1051 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 1052 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1053 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 1054 { 1055 Key: prefixA + baseValue2, 1056 Value: test.NewArrayValue("item1"), 1057 Metadata: nil, 1058 Origin: FromNB, 1059 }, 1060 }) 1061 operation = opHistory[1] 1062 Expect(operation.OpType).To(Equal(test.MockCreate)) 1063 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1064 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2)) 1065 Expect(operation.Err).To(BeNil()) 1066 operation = opHistory[2] 1067 Expect(operation.OpType).To(Equal(test.MockCreate)) 1068 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1069 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue2 + "/item1")) 1070 Expect(operation.Err).To(BeNil()) 1071 1072 // check value dumps 1073 nbConfig := []KVWithMetadata{ 1074 {Key: prefixA + baseValue2, Value: test.NewArrayValue("item1"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 0}}, 1075 } 1076 sbState := []KVWithMetadata{ 1077 {Key: prefixA + baseValue1, Value: test.NewStringValue(baseValue1), Origin: FromSB, Metadata: nil}, 1078 {Key: prefixA + baseValue2, Value: test.NewArrayValue("item1"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 0}}, 1079 } 1080 views := []View{NBView, SBView, CachedView} 1081 for _, view := range views { 1082 var expValues []KVWithMetadata 1083 if view == NBView { 1084 expValues = nbConfig 1085 } else { 1086 expValues = sbState 1087 } 1088 dumpedValues, err := scheduler.DumpValuesByKeyPrefix(prefixA, view) 1089 Expect(err).To(BeNil()) 1090 checkValues(dumpedValues, expValues) 1091 dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor1Name, view) 1092 Expect(err).To(BeNil()) 1093 checkValues(dumpedValues, expValues) 1094 } 1095 mockSB.PopHistoryOfOps() // remove Retrieve-s from the history 1096 1097 // check value states 1098 status := scheduler.GetValueStatus(prefixA + baseValue1) 1099 Expect(status).ToNot(BeNil()) 1100 checkBaseValueStatus(status, &BaseValueStatus{ 1101 Value: &ValueStatus{ 1102 Key: prefixA + baseValue1, 1103 State: ValueState_OBTAINED, 1104 LastOperation: TxnOperation_UNDEFINED, 1105 }, 1106 }) 1107 status = scheduler.GetValueStatus(prefixA + baseValue2) 1108 Expect(status).ToNot(BeNil()) 1109 checkBaseValueStatus(status, &BaseValueStatus{ 1110 Value: &ValueStatus{ 1111 Key: prefixA + baseValue2, 1112 State: ValueState_CONFIGURED, 1113 LastOperation: TxnOperation_CREATE, 1114 }, 1115 DerivedValues: []*ValueStatus{ 1116 { 1117 Key: prefixA + baseValue2 + "/item1", 1118 State: ValueState_CONFIGURED, 1119 LastOperation: TxnOperation_CREATE, 1120 }, 1121 }, 1122 }) 1123 1124 // check transaction operations 1125 txnHistory := scheduler.GetTransactionHistory(startTime, time.Now()) 1126 Expect(txnHistory).To(HaveLen(1)) 1127 txn := txnHistory[0] 1128 Expect(txn.PreRecord).To(BeFalse()) 1129 Expect(txn.Start.After(startTime)).To(BeTrue()) 1130 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 1131 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 1132 Expect(txn.SeqNum).To(BeEquivalentTo(0)) 1133 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 1134 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 1135 Expect(txn.Description).To(BeEmpty()) 1136 checkRecordedValues(txn.Values, []RecordedKVPair{ 1137 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewStringValue(baseValue1)), Origin: FromSB}, 1138 {Key: prefixA + baseValue2, Value: utils.RecordProtoMessage(test.NewArrayValue("item1")), Origin: FromNB}, 1139 }) 1140 1141 txnOps := RecordedTxnOps{ 1142 { 1143 Operation: TxnOperation_CREATE, 1144 Key: prefixA + baseValue2, 1145 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 1146 PrevState: ValueState_NONEXISTENT, 1147 NewState: ValueState_CONFIGURED, 1148 }, 1149 { 1150 Operation: TxnOperation_CREATE, 1151 Key: prefixA + baseValue2 + "/item1", 1152 IsDerived: true, 1153 NewValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 1154 PrevState: ValueState_NONEXISTENT, 1155 NewState: ValueState_CONFIGURED, 1156 }, 1157 } 1158 checkTxnOperations(txn.Planned, txnOps) 1159 checkTxnOperations(txn.Executed, txnOps) 1160 1161 // check flag stats 1162 graphR := scheduler.graph.Read() 1163 errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil) 1164 Expect(errorStats.TotalCount).To(BeEquivalentTo(0)) 1165 pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil) 1166 Expect(pendingStats.TotalCount).To(BeEquivalentTo(0)) 1167 derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil) 1168 Expect(derivedStats.TotalCount).To(BeEquivalentTo(1)) 1169 lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil) 1170 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(3)) 1171 descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil) 1172 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(3)) 1173 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name)) 1174 Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(3)) 1175 valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil) 1176 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(3)) 1177 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String())) 1178 Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(2)) 1179 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_OBTAINED.String())) 1180 Expect(valueStateStats.PerValueCount[ValueState_OBTAINED.String()]).To(BeEquivalentTo(1)) 1181 graphR.Release() 1182 1183 // close scheduler 1184 err = scheduler.Close() 1185 Expect(err).To(BeNil()) 1186 } 1187 1188 func TestResyncWithMultipleDescriptors(t *testing.T) { 1189 RegisterTestingT(t) 1190 1191 // prepare KV Scheduler 1192 scheduler := NewPlugin(UseDeps(func(deps *Deps) { 1193 deps.HTTPHandlers = nil 1194 })) 1195 err := scheduler.Init() 1196 Expect(err).To(BeNil()) 1197 1198 // prepare mocks 1199 mockSB := test.NewMockSouthbound() 1200 // -> initial content: 1201 mockSB.SetValue(prefixA+baseValue1, test.NewArrayValue("item1"), 1202 &test.OnlyInteger{Integer: 0}, FromNB, false) 1203 mockSB.SetValue(prefixA+baseValue1+"/item1", test.NewStringValue("item1"), 1204 nil, FromNB, true) 1205 mockSB.SetValue(prefixB+baseValue2, test.NewArrayValue("item1"), 1206 &test.OnlyInteger{Integer: 0}, FromNB, false) 1207 mockSB.SetValue(prefixB+baseValue2+"/item1", test.NewStringValue("item1"), 1208 nil, FromNB, true) 1209 mockSB.SetValue(prefixC+baseValue3, test.NewArrayValue("item1"), 1210 &test.OnlyInteger{Integer: 0}, FromNB, false) 1211 mockSB.SetValue(prefixC+baseValue3+"/item1", test.NewStringValue("item1"), 1212 nil, FromNB, true) 1213 // -> descriptor1: 1214 descriptor1 := test.NewMockDescriptor(&KVDescriptor{ 1215 Name: descriptor1Name, 1216 NBKeyPrefix: prefixA, 1217 KeySelector: prefixSelector(prefixA), 1218 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 1219 DerivedValues: test.ArrayValueDerBuilder, 1220 WithMetadata: true, 1221 }, mockSB, 1) 1222 // -> descriptor2: 1223 descriptor2 := test.NewMockDescriptor(&KVDescriptor{ 1224 Name: descriptor2Name, 1225 NBKeyPrefix: prefixB, 1226 KeySelector: prefixSelector(prefixB), 1227 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 1228 DerivedValues: test.ArrayValueDerBuilder, 1229 Dependencies: func(key string, value proto.Message) []Dependency { 1230 if key == prefixB+baseValue2+"/item1" { 1231 depKey := prefixA + baseValue1 1232 return []Dependency{ 1233 {Label: depKey, Key: depKey}, 1234 } 1235 } 1236 if key == prefixB+baseValue2+"/item2" { 1237 depKey := prefixA + baseValue1 + "/item1" 1238 return []Dependency{ 1239 {Label: depKey, Key: depKey}, 1240 } 1241 } 1242 return nil 1243 }, 1244 WithMetadata: true, 1245 RetrieveDependencies: []string{descriptor1Name}, 1246 }, mockSB, 1) 1247 // -> descriptor3: 1248 descriptor3 := test.NewMockDescriptor(&KVDescriptor{ 1249 Name: descriptor3Name, 1250 NBKeyPrefix: prefixC, 1251 KeySelector: prefixSelector(prefixC), 1252 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 1253 DerivedValues: test.ArrayValueDerBuilder, 1254 UpdateWithRecreate: func(key string, oldValue, newValue proto.Message, metadata Metadata) bool { 1255 return key == prefixC+baseValue3 1256 }, 1257 WithMetadata: true, 1258 RetrieveDependencies: []string{descriptor2Name}, 1259 }, mockSB, 1) 1260 1261 // register all 3 descriptors with the scheduler 1262 scheduler.RegisterKVDescriptor(descriptor1) 1263 scheduler.RegisterKVDescriptor(descriptor2) 1264 scheduler.RegisterKVDescriptor(descriptor3) 1265 nbPrefixes := scheduler.GetRegisteredNBKeyPrefixes() 1266 Expect(nbPrefixes).To(HaveLen(3)) 1267 Expect(nbPrefixes).To(ContainElement(prefixA)) 1268 Expect(nbPrefixes).To(ContainElement(prefixB)) 1269 Expect(nbPrefixes).To(ContainElement(prefixC)) 1270 1271 // get metadata map created for each descriptor 1272 metadataMap := scheduler.GetMetadataMap(descriptor1.Name) 1273 nameToInteger1, withMetadataMap := metadataMap.(test.NameToInteger) 1274 Expect(withMetadataMap).To(BeTrue()) 1275 metadataMap = scheduler.GetMetadataMap(descriptor2.Name) 1276 nameToInteger2, withMetadataMap := metadataMap.(test.NameToInteger) 1277 Expect(withMetadataMap).To(BeTrue()) 1278 metadataMap = scheduler.GetMetadataMap(descriptor3.Name) 1279 nameToInteger3, withMetadataMap := metadataMap.(test.NameToInteger) 1280 Expect(withMetadataMap).To(BeTrue()) 1281 1282 // run resync transaction with SB that already has some values added 1283 startTime := time.Now() 1284 schedulerTxn := scheduler.StartNBTransaction() 1285 schedulerTxn.SetValue(prefixB+baseValue2, test.NewArrayValue("item1", "item2")) 1286 schedulerTxn.SetValue(prefixA+baseValue1, test.NewArrayValue("item2")) 1287 schedulerTxn.SetValue(prefixC+baseValue3, test.NewArrayValue("item1", "item2")) 1288 seqNum, err := schedulerTxn.Commit(WithResync(testCtx, FullResync, true)) 1289 stopTime := time.Now() 1290 Expect(seqNum).To(BeEquivalentTo(0)) 1291 Expect(err).ShouldNot(HaveOccurred()) 1292 1293 // check the state of SB 1294 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 1295 // -> base value 1 1296 value := mockSB.GetValue(prefixA + baseValue1) 1297 Expect(value).ToNot(BeNil()) 1298 Expect(proto.Equal(value.Value, test.NewArrayValue("item2"))).To(BeTrue()) 1299 Expect(value.Metadata).ToNot(BeNil()) 1300 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 1301 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1302 // -> item1 derived from base value 1 was removed 1303 value = mockSB.GetValue(prefixA + baseValue1 + "/item1") 1304 Expect(value).To(BeNil()) 1305 // -> item2 derived from base value 1 1306 value = mockSB.GetValue(prefixA + baseValue1 + "/item2") 1307 Expect(value).ToNot(BeNil()) 1308 Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue()) 1309 Expect(value.Metadata).To(BeNil()) 1310 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1311 // -> base value 2 1312 value = mockSB.GetValue(prefixB + baseValue2) 1313 Expect(value).ToNot(BeNil()) 1314 Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue()) 1315 Expect(value.Metadata).ToNot(BeNil()) 1316 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 1317 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1318 // -> item1 derived from base value 2 1319 value = mockSB.GetValue(prefixB + baseValue2 + "/item1") 1320 Expect(value).ToNot(BeNil()) 1321 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 1322 Expect(value.Metadata).To(BeNil()) 1323 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1324 // -> item2 derived from base value 2 is pending 1325 value = mockSB.GetValue(prefixB + baseValue2 + "/item2") 1326 Expect(value).To(BeNil()) 1327 // -> base value 3 1328 value = mockSB.GetValue(prefixC + baseValue3) 1329 Expect(value).ToNot(BeNil()) 1330 Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue()) 1331 Expect(value.Metadata).ToNot(BeNil()) 1332 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(1)) 1333 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1334 // -> item1 derived from base value 3 1335 value = mockSB.GetValue(prefixC + baseValue3 + "/item1") 1336 Expect(value).ToNot(BeNil()) 1337 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 1338 Expect(value.Metadata).To(BeNil()) 1339 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1340 // -> item2 derived from base value 3 1341 value = mockSB.GetValue(prefixC + baseValue3 + "/item2") 1342 Expect(value).ToNot(BeNil()) 1343 Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue()) 1344 Expect(value.Metadata).To(BeNil()) 1345 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1346 Expect(mockSB.GetValues(nil)).To(HaveLen(7)) 1347 1348 // check metadata 1349 metadata, exists := nameToInteger1.LookupByName(baseValue1) 1350 Expect(exists).To(BeTrue()) 1351 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 1352 metadata, exists = nameToInteger2.LookupByName(baseValue2) 1353 Expect(exists).To(BeTrue()) 1354 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 1355 metadata, exists = nameToInteger3.LookupByName(baseValue3) 1356 Expect(exists).To(BeTrue()) 1357 Expect(metadata.GetInteger()).To(BeEquivalentTo(1)) 1358 1359 // check operations executed in SB 1360 opHistory := mockSB.PopHistoryOfOps() 1361 Expect(opHistory).To(HaveLen(12)) 1362 operation := opHistory[0] 1363 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 1364 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1365 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 1366 { 1367 Key: prefixA + baseValue1, 1368 Value: test.NewArrayValue("item2"), 1369 Metadata: nil, 1370 Origin: FromNB, 1371 }, 1372 }) 1373 operation = opHistory[1] 1374 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 1375 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name)) 1376 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 1377 { 1378 Key: prefixB + baseValue2, 1379 Value: test.NewArrayValue("item1", "item2"), 1380 Metadata: nil, 1381 Origin: FromNB, 1382 }, 1383 }) 1384 operation = opHistory[2] 1385 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 1386 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name)) 1387 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 1388 { 1389 Key: prefixC + baseValue3, 1390 Value: test.NewArrayValue("item1", "item2"), 1391 Metadata: nil, 1392 Origin: FromNB, 1393 }, 1394 }) 1395 operation = opHistory[3] 1396 Expect(operation.OpType).To(Equal(test.MockDelete)) 1397 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name)) 1398 Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3 + "/item1")) 1399 Expect(operation.Err).To(BeNil()) 1400 operation = opHistory[4] 1401 Expect(operation.OpType).To(Equal(test.MockDelete)) 1402 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name)) 1403 Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3)) 1404 Expect(operation.Err).To(BeNil()) 1405 operation = opHistory[5] 1406 Expect(operation.OpType).To(Equal(test.MockCreate)) 1407 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name)) 1408 Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3)) 1409 Expect(operation.Err).To(BeNil()) 1410 operation = opHistory[6] 1411 Expect(operation.OpType).To(Equal(test.MockCreate)) 1412 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name)) 1413 Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3 + "/item1")) 1414 Expect(operation.Err).To(BeNil()) 1415 operation = opHistory[7] 1416 Expect(operation.OpType).To(Equal(test.MockCreate)) 1417 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor3Name)) 1418 Expect(operation.Key).To(BeEquivalentTo(prefixC + baseValue3 + "/item2")) 1419 Expect(operation.Err).To(BeNil()) 1420 operation = opHistory[8] 1421 Expect(operation.OpType).To(Equal(test.MockDelete)) 1422 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1423 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item1")) 1424 Expect(operation.Err).To(BeNil()) 1425 operation = opHistory[9] 1426 Expect(operation.OpType).To(Equal(test.MockUpdate)) 1427 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1428 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1)) 1429 Expect(operation.Err).To(BeNil()) 1430 operation = opHistory[10] 1431 Expect(operation.OpType).To(Equal(test.MockCreate)) 1432 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1433 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 1434 Expect(operation.Err).To(BeNil()) 1435 operation = opHistory[11] 1436 Expect(operation.OpType).To(Equal(test.MockUpdate)) 1437 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor2Name)) 1438 Expect(operation.Key).To(BeEquivalentTo(prefixB + baseValue2)) 1439 Expect(operation.Err).To(BeNil()) 1440 1441 // check transaction operations 1442 txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Time{}) 1443 Expect(txnHistory).To(HaveLen(1)) 1444 txn := txnHistory[0] 1445 Expect(txn.PreRecord).To(BeFalse()) 1446 Expect(txn.Start.After(startTime)).To(BeTrue()) 1447 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 1448 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 1449 Expect(txn.SeqNum).To(BeEquivalentTo(0)) 1450 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 1451 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 1452 Expect(txn.Description).To(BeEmpty()) 1453 checkRecordedValues(txn.Values, []RecordedKVPair{ 1454 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item2")), Origin: FromNB}, 1455 {Key: prefixB + baseValue2, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 1456 {Key: prefixC + baseValue3, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 1457 }) 1458 1459 // check value dumps 1460 views := []View{NBView, SBView, CachedView} 1461 for _, view := range views { 1462 // descriptor1 1463 expValues := []KVWithMetadata{ 1464 {Key: prefixA + baseValue1, Value: test.NewArrayValue("item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 0}}, 1465 } 1466 dumpedValues, err := scheduler.DumpValuesByKeyPrefix(prefixA, view) 1467 Expect(err).To(BeNil()) 1468 checkValues(dumpedValues, expValues) 1469 dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor1Name, view) 1470 Expect(err).To(BeNil()) 1471 checkValues(dumpedValues, expValues) 1472 // descriptor2 1473 expValues = []KVWithMetadata{ 1474 {Key: prefixB + baseValue2, Value: test.NewArrayValue("item1", "item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 0}}, 1475 } 1476 dumpedValues, err = scheduler.DumpValuesByKeyPrefix(prefixB, view) 1477 Expect(err).To(BeNil()) 1478 checkValues(dumpedValues, expValues) 1479 dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor2Name, view) 1480 Expect(err).To(BeNil()) 1481 checkValues(dumpedValues, expValues) 1482 // descriptor3 1483 expValues = []KVWithMetadata{ 1484 {Key: prefixC + baseValue3, Value: test.NewArrayValue("item1", "item2"), Origin: FromNB, Metadata: &test.OnlyInteger{Integer: 1}}, 1485 } 1486 dumpedValues, err = scheduler.DumpValuesByKeyPrefix(prefixC, view) 1487 Expect(err).To(BeNil()) 1488 checkValues(dumpedValues, expValues) 1489 dumpedValues, err = scheduler.DumpValuesByDescriptor(descriptor3Name, view) 1490 Expect(err).To(BeNil()) 1491 checkValues(dumpedValues, expValues) 1492 } 1493 mockSB.PopHistoryOfOps() // remove Retrieve-s from the history 1494 1495 // check value states 1496 status := scheduler.GetValueStatus(prefixA + baseValue1) 1497 Expect(status).ToNot(BeNil()) 1498 checkBaseValueStatus(status, &BaseValueStatus{ 1499 Value: &ValueStatus{ 1500 Key: prefixA + baseValue1, 1501 State: ValueState_CONFIGURED, 1502 LastOperation: TxnOperation_UPDATE, 1503 }, 1504 DerivedValues: []*ValueStatus{ 1505 { 1506 Key: prefixA + baseValue1 + "/item2", 1507 State: ValueState_CONFIGURED, 1508 LastOperation: TxnOperation_CREATE, 1509 }, 1510 }, 1511 }) 1512 status = scheduler.GetValueStatus(prefixB + baseValue2) 1513 Expect(status).ToNot(BeNil()) 1514 checkBaseValueStatus(status, &BaseValueStatus{ 1515 Value: &ValueStatus{ 1516 Key: prefixB + baseValue2, 1517 State: ValueState_CONFIGURED, 1518 LastOperation: TxnOperation_UPDATE, 1519 }, 1520 DerivedValues: []*ValueStatus{ 1521 { 1522 Key: prefixB + baseValue2 + "/item1", 1523 State: ValueState_CONFIGURED, 1524 LastOperation: TxnOperation_UPDATE, 1525 }, 1526 { 1527 Key: prefixB + baseValue2 + "/item2", 1528 State: ValueState_PENDING, 1529 LastOperation: TxnOperation_CREATE, 1530 Details: []string{prefixA + baseValue1 + "/item1"}, 1531 }, 1532 }, 1533 }) 1534 status = scheduler.GetValueStatus(prefixC + baseValue3) 1535 Expect(status).ToNot(BeNil()) 1536 checkBaseValueStatus(status, &BaseValueStatus{ 1537 Value: &ValueStatus{ 1538 Key: prefixC + baseValue3, 1539 State: ValueState_CONFIGURED, 1540 LastOperation: TxnOperation_UPDATE, 1541 }, 1542 DerivedValues: []*ValueStatus{ 1543 { 1544 Key: prefixC + baseValue3 + "/item1", 1545 State: ValueState_CONFIGURED, 1546 LastOperation: TxnOperation_CREATE, 1547 }, 1548 { 1549 Key: prefixC + baseValue3 + "/item2", 1550 State: ValueState_CONFIGURED, 1551 LastOperation: TxnOperation_CREATE, 1552 }, 1553 }, 1554 }) 1555 1556 txnOps := RecordedTxnOps{ 1557 { 1558 Operation: TxnOperation_DELETE, 1559 Key: prefixC + baseValue3 + "/item1", 1560 IsDerived: true, 1561 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 1562 PrevState: ValueState_DISCOVERED, 1563 NewState: ValueState_REMOVED, 1564 IsRecreate: true, 1565 }, 1566 { 1567 Operation: TxnOperation_DELETE, 1568 Key: prefixC + baseValue3, 1569 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 1570 PrevState: ValueState_DISCOVERED, 1571 NewState: ValueState_REMOVED, 1572 IsRecreate: true, 1573 }, 1574 { 1575 Operation: TxnOperation_CREATE, 1576 Key: prefixC + baseValue3, 1577 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 1578 PrevState: ValueState_REMOVED, 1579 NewState: ValueState_CONFIGURED, 1580 IsRecreate: true, 1581 }, 1582 { 1583 Operation: TxnOperation_CREATE, 1584 Key: prefixC + baseValue3 + "/item1", 1585 IsDerived: true, 1586 NewValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 1587 PrevState: ValueState_NONEXISTENT, // TODO: derived value removed from the graph, ok? 1588 NewState: ValueState_CONFIGURED, 1589 IsRecreate: true, 1590 }, 1591 { 1592 Operation: TxnOperation_CREATE, 1593 Key: prefixC + baseValue3 + "/item2", 1594 IsDerived: true, 1595 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 1596 PrevState: ValueState_NONEXISTENT, 1597 NewState: ValueState_CONFIGURED, 1598 }, 1599 { 1600 Operation: TxnOperation_DELETE, 1601 Key: prefixA + baseValue1 + "/item1", 1602 IsDerived: true, 1603 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 1604 PrevState: ValueState_DISCOVERED, 1605 NewState: ValueState_REMOVED, 1606 }, 1607 { 1608 Operation: TxnOperation_UPDATE, 1609 Key: prefixA + baseValue1, 1610 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 1611 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item2")), 1612 PrevState: ValueState_DISCOVERED, 1613 NewState: ValueState_CONFIGURED, 1614 }, 1615 { 1616 Operation: TxnOperation_CREATE, 1617 Key: prefixA + baseValue1 + "/item2", 1618 IsDerived: true, 1619 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 1620 PrevState: ValueState_NONEXISTENT, 1621 NewState: ValueState_CONFIGURED, 1622 }, 1623 { 1624 Operation: TxnOperation_UPDATE, 1625 Key: prefixB + baseValue2, 1626 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 1627 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 1628 PrevState: ValueState_DISCOVERED, 1629 NewState: ValueState_CONFIGURED, 1630 }, 1631 { 1632 Operation: TxnOperation_CREATE, 1633 Key: prefixB + baseValue2 + "/item2", 1634 IsDerived: true, 1635 NOOP: true, 1636 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 1637 PrevState: ValueState_NONEXISTENT, 1638 NewState: ValueState_PENDING, 1639 }, 1640 } 1641 checkTxnOperations(txn.Planned, txnOps) 1642 checkTxnOperations(txn.Executed, txnOps) 1643 1644 // check flag stats 1645 graphR := scheduler.graph.Read() 1646 errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil) 1647 Expect(errorStats.TotalCount).To(BeEquivalentTo(0)) 1648 pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil) 1649 Expect(pendingStats.TotalCount).To(BeEquivalentTo(1)) 1650 derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil) 1651 Expect(derivedStats.TotalCount).To(BeEquivalentTo(5)) 1652 lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil) 1653 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(8)) 1654 descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil) 1655 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(8)) 1656 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name)) 1657 Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(2)) 1658 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor2Name)) 1659 Expect(descriptorStats.PerValueCount[descriptor2Name]).To(BeEquivalentTo(3)) 1660 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor3Name)) 1661 Expect(descriptorStats.PerValueCount[descriptor3Name]).To(BeEquivalentTo(3)) 1662 valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil) 1663 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(8)) 1664 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String())) 1665 Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(7)) 1666 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_PENDING.String())) 1667 Expect(valueStateStats.PerValueCount[ValueState_PENDING.String()]).To(BeEquivalentTo(1)) 1668 graphR.Release() 1669 1670 // close scheduler 1671 err = scheduler.Close() 1672 Expect(err).To(BeNil()) 1673 } 1674 1675 func TestResyncWithRetry(t *testing.T) { 1676 RegisterTestingT(t) 1677 1678 // prepare KV Scheduler 1679 scheduler := NewPlugin(UseDeps(func(deps *Deps) { 1680 deps.HTTPHandlers = nil 1681 })) 1682 err := scheduler.Init() 1683 Expect(err).To(BeNil()) 1684 1685 // prepare mocks 1686 mockSB := test.NewMockSouthbound() 1687 // -> initial content: 1688 mockSB.SetValue(prefixA+baseValue1, test.NewArrayValue(), 1689 &test.OnlyInteger{Integer: 0}, FromNB, false) 1690 // -> descriptor1: 1691 descriptor1 := test.NewMockDescriptor(&KVDescriptor{ 1692 Name: descriptor1Name, 1693 NBKeyPrefix: prefixA, 1694 KeySelector: prefixSelector(prefixA), 1695 ValueTypeName: string(proto.MessageName(test.NewArrayValue())), 1696 DerivedValues: test.ArrayValueDerBuilder, 1697 WithMetadata: true, 1698 }, mockSB, 1) 1699 // -> planned error 1700 mockSB.PlanError(prefixA+baseValue1+"/item2", errors.New("failed to add value"), 1701 func() { 1702 mockSB.SetValue(prefixA+baseValue1, test.NewArrayValue("item1"), 1703 &test.OnlyInteger{Integer: 0}, FromNB, false) 1704 }) 1705 1706 // register descriptor with the scheduler 1707 scheduler.RegisterKVDescriptor(descriptor1) 1708 1709 // subscribe to receive notifications about value state changes 1710 errorChan := make(chan *BaseValueStatus, 5) 1711 scheduler.WatchValueStatus(errorChan, prefixSelector(prefixA)) 1712 1713 // get metadata map created for the descriptor 1714 metadataMap := scheduler.GetMetadataMap(descriptor1.Name) 1715 nameToInteger, withMetadataMap := metadataMap.(test.NameToInteger) 1716 Expect(withMetadataMap).To(BeTrue()) 1717 1718 // run resync transaction that will fail for one value 1719 startTime := time.Now() 1720 resyncTxn := scheduler.StartNBTransaction() 1721 resyncTxn.SetValue(prefixA+baseValue1, test.NewArrayValue("item1", "item2")) 1722 description := "testing resync with retry" 1723 ctx := testCtx 1724 ctx = WithRetry(ctx, 3*time.Second, 3, false) 1725 ctx = WithResync(ctx, FullResync, true) 1726 ctx = WithDescription(ctx, description) 1727 seqNum, err := resyncTxn.Commit(ctx) 1728 stopTime := time.Now() 1729 Expect(seqNum).To(BeEquivalentTo(0)) 1730 Expect(err).ToNot(BeNil()) 1731 txnErr := err.(*TransactionError) 1732 Expect(txnErr.GetTxnInitError()).ShouldNot(HaveOccurred()) 1733 kvErrors := txnErr.GetKVErrors() 1734 Expect(kvErrors).To(HaveLen(1)) 1735 Expect(kvErrors[0].TxnOperation).To(BeEquivalentTo(TxnOperation_CREATE)) 1736 Expect(kvErrors[0].Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 1737 Expect(kvErrors[0].Error.Error()).To(BeEquivalentTo("failed to add value")) 1738 1739 // check the state of SB 1740 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 1741 // -> base value 1 1742 value := mockSB.GetValue(prefixA + baseValue1) 1743 Expect(value).ToNot(BeNil()) 1744 Expect(proto.Equal(value.Value, test.NewArrayValue("item1"))).To(BeTrue()) 1745 Expect(value.Metadata).ToNot(BeNil()) 1746 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 1747 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1748 // -> item1 derived from base value 1 1749 value = mockSB.GetValue(prefixA + baseValue1 + "/item1") 1750 Expect(value).ToNot(BeNil()) 1751 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 1752 Expect(value.Metadata).To(BeNil()) 1753 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1754 // -> item2 derived from base value 1 failed to get added 1755 value = mockSB.GetValue(prefixA + baseValue1 + "/item2") 1756 Expect(value).To(BeNil()) 1757 Expect(mockSB.GetValues(nil)).To(HaveLen(2)) 1758 1759 // check metadata 1760 metadata, exists := nameToInteger.LookupByName(baseValue1) 1761 Expect(exists).To(BeTrue()) 1762 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 1763 1764 // check operations executed in SB 1765 opHistory := mockSB.PopHistoryOfOps() 1766 Expect(opHistory).To(HaveLen(5)) 1767 operation := opHistory[0] 1768 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 1769 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1770 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 1771 { 1772 Key: prefixA + baseValue1, 1773 Value: test.NewArrayValue("item1", "item2"), 1774 Metadata: nil, 1775 Origin: FromNB, 1776 }, 1777 }) 1778 operation = opHistory[1] 1779 Expect(operation.OpType).To(Equal(test.MockUpdate)) 1780 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1781 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1)) 1782 Expect(operation.Err).To(BeNil()) 1783 operation = opHistory[2] 1784 Expect(operation.OpType).To(Equal(test.MockCreate)) 1785 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1786 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item1")) 1787 Expect(operation.Err).To(BeNil()) 1788 operation = opHistory[3] 1789 Expect(operation.OpType).To(Equal(test.MockCreate)) 1790 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1791 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 1792 Expect(operation.Err).ToNot(BeNil()) 1793 Expect(operation.Err.Error()).To(BeEquivalentTo("failed to add value")) 1794 operation = opHistory[4] // refresh failed value 1795 Expect(operation.OpType).To(Equal(test.MockRetrieve)) 1796 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1797 checkValues(operation.CorrelateRetrieve, []KVWithMetadata{ 1798 { 1799 Key: prefixA + baseValue1, 1800 Value: test.NewArrayValue("item1", "item2"), 1801 Metadata: &test.OnlyInteger{Integer: 0}, 1802 Origin: FromNB, 1803 }, 1804 }) 1805 1806 // check transaction operations 1807 txnHistory := scheduler.GetTransactionHistory(time.Time{}, time.Time{}) 1808 Expect(txnHistory).To(HaveLen(1)) 1809 txn := txnHistory[0] 1810 Expect(txn.PreRecord).To(BeFalse()) 1811 Expect(txn.Start.After(startTime)).To(BeTrue()) 1812 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 1813 Expect(txn.Stop.Before(stopTime)).To(BeTrue()) 1814 Expect(txn.SeqNum).To(BeEquivalentTo(0)) 1815 Expect(txn.TxnType).To(BeEquivalentTo(NBTransaction)) 1816 Expect(txn.ResyncType).To(BeEquivalentTo(FullResync)) 1817 Expect(txn.Description).To(Equal(description)) 1818 checkRecordedValues(txn.Values, []RecordedKVPair{ 1819 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 1820 }) 1821 1822 txnOps := RecordedTxnOps{ 1823 { 1824 Operation: TxnOperation_UPDATE, 1825 Key: prefixA + baseValue1, 1826 PrevValue: utils.RecordProtoMessage(test.NewArrayValue()), 1827 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 1828 PrevState: ValueState_DISCOVERED, 1829 NewState: ValueState_CONFIGURED, 1830 }, 1831 { 1832 Operation: TxnOperation_CREATE, 1833 Key: prefixA + baseValue1 + "/item1", 1834 IsDerived: true, 1835 NewValue: utils.RecordProtoMessage(test.NewStringValue("item1")), 1836 PrevState: ValueState_NONEXISTENT, 1837 NewState: ValueState_CONFIGURED, 1838 }, 1839 { 1840 Operation: TxnOperation_CREATE, 1841 Key: prefixA + baseValue1 + "/item2", 1842 IsDerived: true, 1843 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 1844 PrevState: ValueState_NONEXISTENT, 1845 NewState: ValueState_CONFIGURED, 1846 }, 1847 } 1848 checkTxnOperations(txn.Planned, txnOps) 1849 txnOps[2].NewState = ValueState_RETRYING 1850 txnOps[2].NewErr = errors.New("failed to add value") 1851 checkTxnOperations(txn.Executed, txnOps) 1852 1853 // check flag stats 1854 graphR := scheduler.graph.Read() 1855 errorStats := graphR.GetFlagStats(ErrorFlagIndex, nil) 1856 Expect(errorStats.TotalCount).To(BeEquivalentTo(1)) 1857 pendingStats := graphR.GetFlagStats(UnavailValueFlagIndex, nil) 1858 Expect(pendingStats.TotalCount).To(BeEquivalentTo(1)) 1859 derivedStats := graphR.GetFlagStats(DerivedFlagIndex, nil) 1860 Expect(derivedStats.TotalCount).To(BeEquivalentTo(2)) 1861 lastUpdateStats := graphR.GetFlagStats(LastUpdateFlagIndex, nil) 1862 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(3)) 1863 descriptorStats := graphR.GetFlagStats(DescriptorFlagIndex, nil) 1864 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(3)) 1865 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name)) 1866 Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(3)) 1867 valueStateStats := graphR.GetFlagStats(ValueStateFlagIndex, nil) 1868 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(3)) 1869 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String())) 1870 Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(2)) 1871 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_RETRYING.String())) 1872 Expect(valueStateStats.PerValueCount[ValueState_RETRYING.String()]).To(BeEquivalentTo(1)) 1873 graphR.Release() 1874 1875 // check value state updates received through the channel 1876 var valueStatus *BaseValueStatus 1877 Eventually(errorChan, time.Second).Should(Receive(&valueStatus)) 1878 checkBaseValueStatus(valueStatus, &BaseValueStatus{ 1879 Value: &ValueStatus{ 1880 Key: prefixA + baseValue1, 1881 State: ValueState_CONFIGURED, 1882 LastOperation: TxnOperation_UPDATE, 1883 }, 1884 DerivedValues: []*ValueStatus{ 1885 { 1886 Key: prefixA + baseValue1 + "/item1", 1887 State: ValueState_CONFIGURED, 1888 LastOperation: TxnOperation_CREATE, 1889 }, 1890 { 1891 Key: prefixA + baseValue1 + "/item2", 1892 State: ValueState_RETRYING, 1893 LastOperation: TxnOperation_CREATE, 1894 Error: "failed to add value", 1895 }, 1896 }, 1897 }) 1898 1899 // eventually the value should get "fixed" 1900 Eventually(errorChan, 5*time.Second).Should(Receive(&valueStatus)) 1901 checkBaseValueStatus(valueStatus, &BaseValueStatus{ 1902 Value: &ValueStatus{ 1903 Key: prefixA + baseValue1, 1904 State: ValueState_CONFIGURED, 1905 LastOperation: TxnOperation_UPDATE, 1906 }, 1907 DerivedValues: []*ValueStatus{ 1908 { 1909 Key: prefixA + baseValue1 + "/item1", 1910 State: ValueState_CONFIGURED, 1911 LastOperation: TxnOperation_UPDATE, 1912 }, 1913 { 1914 Key: prefixA + baseValue1 + "/item2", 1915 State: ValueState_CONFIGURED, 1916 LastOperation: TxnOperation_CREATE, 1917 }, 1918 }, 1919 }) 1920 1921 // check the state of SB after retry 1922 Expect(mockSB.GetKeysWithInvalidData()).To(BeEmpty()) 1923 // -> base value 1 1924 value = mockSB.GetValue(prefixA + baseValue1) 1925 Expect(value).ToNot(BeNil()) 1926 Expect(proto.Equal(value.Value, test.NewArrayValue("item1", "item2"))).To(BeTrue()) 1927 Expect(value.Metadata).ToNot(BeNil()) 1928 Expect(value.Metadata.(test.MetaWithInteger).GetInteger()).To(BeEquivalentTo(0)) 1929 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1930 // -> item1 derived from base value 1 1931 value = mockSB.GetValue(prefixA + baseValue1 + "/item1") 1932 Expect(value).ToNot(BeNil()) 1933 Expect(proto.Equal(value.Value, test.NewStringValue("item1"))).To(BeTrue()) 1934 Expect(value.Metadata).To(BeNil()) 1935 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1936 Expect(mockSB.GetValues(nil)).To(HaveLen(3)) 1937 // -> item2 derived from base value 1 was re-added 1938 value = mockSB.GetValue(prefixA + baseValue1 + "/item2") 1939 Expect(value).ToNot(BeNil()) 1940 Expect(proto.Equal(value.Value, test.NewStringValue("item2"))).To(BeTrue()) 1941 Expect(value.Metadata).To(BeNil()) 1942 Expect(value.Origin).To(BeEquivalentTo(FromNB)) 1943 Expect(mockSB.GetValues(nil)).To(HaveLen(3)) 1944 1945 // check metadata 1946 metadata, exists = nameToInteger.LookupByName(baseValue1) 1947 Expect(exists).To(BeTrue()) 1948 Expect(metadata.GetInteger()).To(BeEquivalentTo(0)) 1949 1950 // check operations executed in SB during retry 1951 opHistory = mockSB.PopHistoryOfOps() 1952 Expect(opHistory).To(HaveLen(2)) 1953 operation = opHistory[0] 1954 Expect(operation.OpType).To(Equal(test.MockUpdate)) 1955 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1956 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1)) 1957 Expect(operation.Err).To(BeNil()) 1958 operation = opHistory[1] 1959 Expect(operation.OpType).To(Equal(test.MockCreate)) 1960 Expect(operation.Descriptor).To(BeEquivalentTo(descriptor1Name)) 1961 Expect(operation.Key).To(BeEquivalentTo(prefixA + baseValue1 + "/item2")) 1962 Expect(operation.Err).To(BeNil()) 1963 1964 // check retry transaction operations 1965 txnHistory = scheduler.GetTransactionHistory(time.Time{}, time.Now()) 1966 Expect(txnHistory).To(HaveLen(2)) 1967 txn = txnHistory[1] 1968 Expect(txn.PreRecord).To(BeFalse()) 1969 Expect(txn.Start.After(stopTime)).To(BeTrue()) 1970 Expect(txn.Start.Before(txn.Stop)).To(BeTrue()) 1971 Expect(txn.Stop.Before(time.Now())).To(BeTrue()) 1972 Expect(txn.SeqNum).To(BeEquivalentTo(1)) 1973 Expect(txn.TxnType).To(BeEquivalentTo(RetryFailedOps)) 1974 Expect(txn.ResyncType).To(BeEquivalentTo(NotResync)) 1975 Expect(txn.Description).To(BeEmpty()) 1976 checkRecordedValues(txn.Values, []RecordedKVPair{ 1977 {Key: prefixA + baseValue1, Value: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), Origin: FromNB}, 1978 }) 1979 1980 txnOps = RecordedTxnOps{ 1981 { 1982 Operation: TxnOperation_UPDATE, 1983 Key: prefixA + baseValue1, 1984 PrevValue: utils.RecordProtoMessage(test.NewArrayValue("item1")), 1985 NewValue: utils.RecordProtoMessage(test.NewArrayValue("item1", "item2")), 1986 PrevState: ValueState_CONFIGURED, 1987 NewState: ValueState_CONFIGURED, 1988 IsRetry: true, 1989 }, 1990 { 1991 Operation: TxnOperation_CREATE, 1992 Key: prefixA + baseValue1 + "/item2", 1993 IsDerived: true, 1994 PrevValue: utils.RecordProtoMessage(test.NewStringValue("item2")), // TODO: shouldn't be nil? 1995 NewValue: utils.RecordProtoMessage(test.NewStringValue("item2")), 1996 PrevState: ValueState_RETRYING, 1997 NewState: ValueState_CONFIGURED, 1998 PrevErr: errors.New("failed to add value"), 1999 IsRetry: true, 2000 }, 2001 } 2002 checkTxnOperations(txn.Planned, txnOps) 2003 checkTxnOperations(txn.Executed, txnOps) 2004 2005 // check flag stats 2006 graphR = scheduler.graph.Read() 2007 errorStats = graphR.GetFlagStats(ErrorFlagIndex, nil) 2008 Expect(errorStats.TotalCount).To(BeEquivalentTo(1)) 2009 pendingStats = graphR.GetFlagStats(UnavailValueFlagIndex, nil) 2010 Expect(pendingStats.TotalCount).To(BeEquivalentTo(1)) 2011 derivedStats = graphR.GetFlagStats(DerivedFlagIndex, nil) 2012 Expect(derivedStats.TotalCount).To(BeEquivalentTo(4)) 2013 lastUpdateStats = graphR.GetFlagStats(LastUpdateFlagIndex, nil) 2014 Expect(lastUpdateStats.TotalCount).To(BeEquivalentTo(6)) 2015 descriptorStats = graphR.GetFlagStats(DescriptorFlagIndex, nil) 2016 Expect(descriptorStats.TotalCount).To(BeEquivalentTo(6)) 2017 Expect(descriptorStats.PerValueCount).To(HaveKey(descriptor1Name)) 2018 Expect(descriptorStats.PerValueCount[descriptor1Name]).To(BeEquivalentTo(6)) 2019 valueStateStats = graphR.GetFlagStats(ValueStateFlagIndex, nil) 2020 Expect(valueStateStats.TotalCount).To(BeEquivalentTo(6)) 2021 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_CONFIGURED.String())) 2022 Expect(valueStateStats.PerValueCount[ValueState_CONFIGURED.String()]).To(BeEquivalentTo(5)) 2023 Expect(valueStateStats.PerValueCount).To(HaveKey(ValueState_RETRYING.String())) 2024 Expect(valueStateStats.PerValueCount[ValueState_RETRYING.String()]).To(BeEquivalentTo(1)) 2025 graphR.Release() 2026 2027 // close scheduler 2028 err = scheduler.Close() 2029 Expect(err).To(BeNil()) 2030 } 2031 2032 /* when graph dump is needed: 2033 graphR := scheduler.graph.Read() 2034 graphDump := graphR.Dump() 2035 fmt.Print(graphDump) 2036 graphR.Release() 2037 */