github.com/cilium/cilium@v1.16.2/pkg/envoy/xds/ack_test.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package xds 5 6 import ( 7 "context" 8 "testing" 9 "time" 10 11 "github.com/stretchr/testify/assert" 12 "github.com/stretchr/testify/require" 13 14 "github.com/cilium/cilium/pkg/completion" 15 ) 16 17 const ( 18 node0 = "10.0.0.0" 19 node1 = "10.0.0.1" 20 node2 = "10.0.0.2" 21 22 MaxCompletionDuration = 250 * time.Millisecond 23 ) 24 25 type compCheck struct { 26 err error 27 ch chan error 28 } 29 30 func newCompCheck() *compCheck { 31 return &compCheck{ 32 ch: make(chan error, 1), 33 } 34 } 35 36 func (c *compCheck) Err() error { 37 return c.err 38 } 39 40 // Return a new completion callback that will write the completion error to a channel 41 func newCompCallback() (func(error), *compCheck) { 42 comp := newCompCheck() 43 callback := func(err error) { 44 log.WithError(err).Debug("callback called") 45 comp.ch <- err 46 close(comp.ch) 47 } 48 return callback, comp 49 } 50 51 func completedComparison(comp *compCheck) assert.Comparison { 52 return func() bool { 53 return completedInTime(comp) 54 } 55 } 56 57 func isNotCompletedComparison(comp *compCheck) assert.Comparison { 58 return func() bool { 59 return !completedInTime(comp) 60 } 61 } 62 63 func completedInTime(comp *compCheck) bool { 64 if comp == nil { 65 return false 66 } 67 68 if comp.err != nil { 69 return false 70 } 71 72 select { 73 case comp.err = <-comp.ch: 74 return comp.err == nil 75 case <-time.After(MaxCompletionDuration): 76 return false 77 } 78 } 79 80 func TestUpsertSingleNode(t *testing.T) { 81 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 82 defer cancel() 83 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 84 wg := completion.NewWaitGroup(ctx) 85 86 // Empty cache is the version 1 87 cache := NewCache() 88 acker := NewAckingResourceMutatorWrapper(cache) 89 require.Len(t, acker.ackedVersions, 0) 90 91 // Create version 2 with resource 0. 92 callback, comp := newCompCallback() 93 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, wg, callback) 94 require.Condition(t, isNotCompletedComparison(comp)) 95 require.Len(t, acker.ackedVersions, 0) 96 97 // Ack the right version, for the right resource, from another node. 98 acker.HandleResourceVersionAck(2, 2, node1, []string{resources[0].Name}, typeURL, "") 99 require.Condition(t, isNotCompletedComparison(comp)) 100 require.Len(t, acker.ackedVersions, 1) 101 require.Equal(t, uint64(2), acker.ackedVersions[node1]) 102 103 // Ack the right version, for another resource, from the right node. 104 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[1].Name}, typeURL, "") 105 require.Condition(t, isNotCompletedComparison(comp)) 106 require.Len(t, acker.ackedVersions, 2) 107 require.Equal(t, uint64(2), acker.ackedVersions[node0]) 108 109 // Ack an older version, for the right resource, from the right node. 110 acker.HandleResourceVersionAck(1, 1, node0, []string{resources[0].Name}, typeURL, "") 111 require.Condition(t, isNotCompletedComparison(comp)) 112 require.Len(t, acker.ackedVersions, 2) 113 require.Equal(t, uint64(2), acker.ackedVersions[node0]) 114 115 // Ack the right version, for the right resource, from the right node. 116 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[0].Name}, typeURL, "") 117 require.Condition(t, completedComparison(comp)) 118 require.Len(t, acker.ackedVersions, 2) 119 require.Equal(t, uint64(2), acker.ackedVersions[node0]) 120 } 121 122 // UseCurrent adds a completion to the WaitGroup if the current 123 // version of the cached resource has not been acked yet, allowing the 124 // caller to wait for the ACK. 125 func (m *AckingResourceMutatorWrapper) UseCurrent(typeURL string, nodeIDs []string, wg *completion.WaitGroup) { 126 m.locker.Lock() 127 defer m.locker.Unlock() 128 129 if wg != nil { 130 m.useCurrent(typeURL, nodeIDs, wg, nil) 131 } 132 } 133 134 func TestUseCurrent(t *testing.T) { 135 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 136 defer cancel() 137 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 138 wg := completion.NewWaitGroup(ctx) 139 140 // Empty cache is the version 1 141 cache := NewCache() 142 acker := NewAckingResourceMutatorWrapper(cache) 143 require.Len(t, acker.ackedVersions, 0) 144 145 // Create version 2 with resource 0. 146 callback, comp := newCompCallback() 147 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, wg, callback) 148 require.Condition(t, isNotCompletedComparison(comp)) 149 require.Len(t, acker.ackedVersions, 0) 150 require.Len(t, acker.pendingCompletions, 1) 151 152 // Ack the right version, for the right resource, from another node. 153 acker.HandleResourceVersionAck(2, 2, node1, []string{resources[0].Name}, typeURL, "") 154 require.Condition(t, isNotCompletedComparison(comp)) 155 require.Len(t, acker.ackedVersions, 1) 156 require.Equal(t, uint64(2), acker.ackedVersions[node1]) 157 require.Len(t, acker.pendingCompletions, 1) 158 159 // Use current version, not yet acked 160 acker.UseCurrent(typeURL, []string{node0}, wg) 161 require.Len(t, acker.pendingCompletions, 2) 162 163 // Ack the right version, for another resource, from the right node. 164 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[1].Name}, typeURL, "") 165 require.Condition(t, isNotCompletedComparison(comp)) 166 require.Len(t, acker.ackedVersions, 2) 167 require.Equal(t, uint64(2), acker.ackedVersions[node0]) 168 // UseCurrent ignores resource names, so an ack of the same or later version from the right node will complete it 169 require.Len(t, acker.pendingCompletions, 1) 170 171 // Ack an older version, for the right resource, from the right node. 172 acker.HandleResourceVersionAck(1, 1, node0, []string{resources[0].Name}, typeURL, "") 173 require.Condition(t, isNotCompletedComparison(comp)) 174 require.Len(t, acker.ackedVersions, 2) 175 require.Equal(t, uint64(2), acker.ackedVersions[node0]) 176 require.Len(t, acker.pendingCompletions, 1) 177 178 // Ack the right version, for the right resource, from the right node. 179 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[0].Name}, typeURL, "") 180 require.Condition(t, completedComparison(comp)) 181 require.Len(t, acker.ackedVersions, 2) 182 require.Equal(t, uint64(2), acker.ackedVersions[node0]) 183 require.Len(t, acker.pendingCompletions, 0) 184 } 185 186 func TestUpsertMultipleNodes(t *testing.T) { 187 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 188 defer cancel() 189 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 190 wg := completion.NewWaitGroup(ctx) 191 192 // Empty cache is the version 1 193 cache := NewCache() 194 acker := NewAckingResourceMutatorWrapper(cache) 195 require.Len(t, acker.ackedVersions, 0) 196 197 // Create version 2 with resource 0. 198 callback, comp := newCompCallback() 199 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0, node1}, wg, callback) 200 require.Condition(t, isNotCompletedComparison(comp)) 201 require.Equal(t, false, acker.currentVersionAcked([]string{node0})) 202 require.Equal(t, false, acker.currentVersionAcked([]string{node1})) 203 require.Equal(t, false, acker.currentVersionAcked([]string{node2})) 204 205 // Ack the right version, for the right resource, from another node. 206 acker.HandleResourceVersionAck(2, 2, node2, []string{resources[0].Name}, typeURL, "") 207 require.Condition(t, isNotCompletedComparison(comp)) 208 require.Equal(t, false, acker.currentVersionAcked([]string{node0})) 209 require.Equal(t, false, acker.currentVersionAcked([]string{node1})) 210 require.Equal(t, true, acker.currentVersionAcked([]string{node2})) 211 212 // Ack the right version, for the right resource, from one of the nodes (node0). 213 // One of the nodes (node1) still needs to ACK. 214 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[0].Name}, typeURL, "") 215 require.Condition(t, isNotCompletedComparison(comp)) 216 require.Equal(t, true, acker.currentVersionAcked([]string{node0})) 217 require.Equal(t, false, acker.currentVersionAcked([]string{node1})) 218 require.Equal(t, true, acker.currentVersionAcked([]string{node2})) 219 require.Equal(t, false, acker.currentVersionAcked([]string{node0, node1})) 220 require.Equal(t, true, acker.currentVersionAcked([]string{node0, node2})) 221 222 // Ack the right version, for the right resource, from the last remaining node (node1). 223 acker.HandleResourceVersionAck(2, 2, node1, []string{resources[0].Name}, typeURL, "") 224 require.Condition(t, completedComparison(comp)) 225 require.Equal(t, true, acker.currentVersionAcked([]string{node0})) 226 require.Equal(t, true, acker.currentVersionAcked([]string{node1})) 227 require.Equal(t, true, acker.currentVersionAcked([]string{node2})) 228 require.Equal(t, true, acker.currentVersionAcked([]string{node0, node1, node2})) 229 } 230 231 func TestUpsertMoreRecentVersion(t *testing.T) { 232 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 233 defer cancel() 234 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 235 wg := completion.NewWaitGroup(ctx) 236 237 // Empty cache is the version 1 238 cache := NewCache() 239 acker := NewAckingResourceMutatorWrapper(cache) 240 241 // Create version 2 with resource 0. 242 callback, comp := newCompCallback() 243 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, wg, callback) 244 require.Condition(t, isNotCompletedComparison(comp)) 245 246 // Ack an older version, for the right resource, from the right node. 247 acker.HandleResourceVersionAck(1, 1, node0, []string{resources[0].Name}, typeURL, "") 248 require.Condition(t, isNotCompletedComparison(comp)) 249 250 // Ack a more recent version, for the right resource, from the right node. 251 acker.HandleResourceVersionAck(123, 123, node0, []string{resources[0].Name}, typeURL, "") 252 require.Condition(t, completedComparison(comp)) 253 } 254 255 func TestUpsertMoreRecentVersionNack(t *testing.T) { 256 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 257 defer cancel() 258 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 259 wg := completion.NewWaitGroup(ctx) 260 261 // Empty cache is the version 1 262 cache := NewCache() 263 acker := NewAckingResourceMutatorWrapper(cache) 264 265 // Create version 2 with resource 0. 266 callback, comp := newCompCallback() 267 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, wg, callback) 268 require.Condition(t, isNotCompletedComparison(comp)) 269 270 // Ack an older version, for the right resource, from the right node. 271 acker.HandleResourceVersionAck(1, 1, node0, []string{resources[0].Name}, typeURL, "") 272 require.Condition(t, isNotCompletedComparison(comp)) 273 274 // NAck a more recent version, for the right resource, from the right node. 275 acker.HandleResourceVersionAck(1, 2, node0, []string{resources[0].Name}, typeURL, "Detail") 276 // IsCompleted is true only for completions without error 277 require.Condition(t, isNotCompletedComparison(comp)) 278 require.NotEqual(t, nil, comp.Err()) 279 require.EqualValues(t, &ProxyError{Err: ErrNackReceived, Detail: "Detail"}, comp.Err()) 280 } 281 282 func TestDeleteSingleNode(t *testing.T) { 283 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 284 defer cancel() 285 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 286 wg := completion.NewWaitGroup(ctx) 287 288 // Empty cache is the version 1 289 cache := NewCache() 290 acker := NewAckingResourceMutatorWrapper(cache) 291 292 // Create version 2 with resource 0. 293 callback, comp := newCompCallback() 294 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, wg, callback) 295 require.Condition(t, isNotCompletedComparison(comp)) 296 297 // Ack the right version, for the right resource, from the right node. 298 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[0].Name}, typeURL, "") 299 require.Condition(t, completedComparison(comp)) 300 301 // Create version 3 with no resources. 302 callback, comp = newCompCallback() 303 acker.Delete(typeURL, resources[0].Name, []string{node0}, wg, callback) 304 require.Condition(t, isNotCompletedComparison(comp)) 305 306 // Ack the right version, for another resource, from another node. 307 acker.HandleResourceVersionAck(3, 3, node1, []string{resources[2].Name}, typeURL, "") 308 require.Condition(t, isNotCompletedComparison(comp)) 309 310 // Ack the right version, for another resource, from the right node. 311 acker.HandleResourceVersionAck(3, 3, node0, []string{resources[2].Name}, typeURL, "") 312 // The resource name is ignored. For delete, we only consider the version. 313 require.Condition(t, completedComparison(comp)) 314 } 315 316 func TestDeleteMultipleNodes(t *testing.T) { 317 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 318 defer cancel() 319 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 320 wg := completion.NewWaitGroup(ctx) 321 322 // Empty cache is the version 1 323 cache := NewCache() 324 acker := NewAckingResourceMutatorWrapper(cache) 325 326 // Create version 2 with resource 0. 327 callback, comp := newCompCallback() 328 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, wg, callback) 329 require.Condition(t, isNotCompletedComparison(comp)) 330 331 // Ack the right version, for the right resource, from the right node. 332 acker.HandleResourceVersionAck(2, 2, node0, []string{resources[0].Name}, typeURL, "") 333 require.Condition(t, completedComparison(comp)) 334 335 // Create version 3 with no resources. 336 callback, comp = newCompCallback() 337 acker.Delete(typeURL, resources[0].Name, []string{node0, node1}, wg, callback) 338 require.Condition(t, isNotCompletedComparison(comp)) 339 340 // Ack the right version, for another resource, from one of the nodes. 341 acker.HandleResourceVersionAck(3, 3, node1, []string{resources[2].Name}, typeURL, "") 342 require.Condition(t, isNotCompletedComparison(comp)) 343 344 // Ack the right version, for another resource, from the remaining node. 345 acker.HandleResourceVersionAck(3, 3, node0, []string{resources[2].Name}, typeURL, "") 346 // The resource name is ignored. For delete, we only consider the version. 347 require.Condition(t, completedComparison(comp)) 348 } 349 350 func TestRevertInsert(t *testing.T) { 351 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 352 defer cancel() 353 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 354 wg := completion.NewWaitGroup(ctx) 355 356 cache := NewCache() 357 acker := NewAckingResourceMutatorWrapper(cache) 358 359 // Create version 1 with resource 0. 360 // Insert. 361 revert := acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, nil, nil) 362 363 // Insert another resource. 364 _ = acker.Upsert(typeURL, resources[2].Name, resources[2], []string{node0}, nil, nil) 365 366 res, err := cache.Lookup(typeURL, resources[0].Name) 367 require.NoError(t, err) 368 require.Equal(t, resources[0], res) 369 370 res, err = cache.Lookup(typeURL, resources[2].Name) 371 require.NoError(t, err) 372 require.Equal(t, resources[2], res) 373 374 comp := wg.AddCompletion() 375 defer comp.Complete(nil) 376 revert(comp) 377 378 res, err = cache.Lookup(typeURL, resources[0].Name) 379 require.NoError(t, err) 380 require.Nil(t, res) 381 382 res, err = cache.Lookup(typeURL, resources[2].Name) 383 require.NoError(t, err) 384 require.Equal(t, resources[2], res) 385 } 386 387 func TestRevertUpdate(t *testing.T) { 388 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 389 defer cancel() 390 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 391 wg := completion.NewWaitGroup(ctx) 392 393 cache := NewCache() 394 acker := NewAckingResourceMutatorWrapper(cache) 395 396 // Create version 1 with resource 0. 397 // Insert. 398 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, nil, nil) 399 400 // Insert another resource. 401 _ = acker.Upsert(typeURL, resources[2].Name, resources[2], []string{node0}, nil, nil) 402 403 res, err := cache.Lookup(typeURL, resources[0].Name) 404 require.NoError(t, err) 405 require.Equal(t, resources[0], res) 406 407 res, err = cache.Lookup(typeURL, resources[2].Name) 408 require.NoError(t, err) 409 require.Equal(t, resources[2], res) 410 411 // Update. 412 revert := acker.Upsert(typeURL, resources[0].Name, resources[1], []string{node0}, nil, nil) 413 414 res, err = cache.Lookup(typeURL, resources[0].Name) 415 require.NoError(t, err) 416 require.Equal(t, resources[1], res) 417 418 comp := wg.AddCompletion() 419 defer comp.Complete(nil) 420 revert(comp) 421 422 res, err = cache.Lookup(typeURL, resources[0].Name) 423 require.NoError(t, err) 424 require.Equal(t, resources[0], res) 425 426 res, err = cache.Lookup(typeURL, resources[2].Name) 427 require.NoError(t, err) 428 require.Equal(t, resources[2], res) 429 } 430 431 func TestRevertDelete(t *testing.T) { 432 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 433 defer cancel() 434 typeURL := "type.googleapis.com/envoy.config.v3.DummyConfiguration" 435 wg := completion.NewWaitGroup(ctx) 436 437 cache := NewCache() 438 acker := NewAckingResourceMutatorWrapper(cache) 439 440 // Create version 1 with resource 0. 441 // Insert. 442 acker.Upsert(typeURL, resources[0].Name, resources[0], []string{node0}, nil, nil) 443 444 // Insert another resource. 445 _ = acker.Upsert(typeURL, resources[2].Name, resources[2], []string{node0}, nil, nil) 446 447 res, err := cache.Lookup(typeURL, resources[0].Name) 448 require.NoError(t, err) 449 require.Equal(t, resources[0], res) 450 451 res, err = cache.Lookup(typeURL, resources[2].Name) 452 require.NoError(t, err) 453 require.Equal(t, resources[2], res) 454 455 // Delete. 456 revert := acker.Delete(typeURL, resources[0].Name, []string{node0}, nil, nil) 457 458 res, err = cache.Lookup(typeURL, resources[0].Name) 459 require.NoError(t, err) 460 require.Nil(t, res) 461 462 res, err = cache.Lookup(typeURL, resources[2].Name) 463 require.NoError(t, err) 464 require.Equal(t, resources[2], res) 465 466 comp := wg.AddCompletion() 467 defer comp.Complete(nil) 468 revert(comp) 469 470 res, err = cache.Lookup(typeURL, resources[0].Name) 471 require.NoError(t, err) 472 require.Equal(t, resources[0], res) 473 474 res, err = cache.Lookup(typeURL, resources[2].Name) 475 require.NoError(t, err) 476 require.Equal(t, resources[2], res) 477 }