github.com/hernad/nomad@v1.6.112/nomad/plan_apply_node_tracker_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package nomad 5 6 import ( 7 "fmt" 8 "testing" 9 "time" 10 11 "github.com/hashicorp/go-hclog" 12 "github.com/hernad/nomad/ci" 13 "github.com/stretchr/testify/require" 14 ) 15 16 func TestCachedtBadNodeTracker(t *testing.T) { 17 ci.Parallel(t) 18 19 config := DefaultCachedBadNodeTrackerConfig() 20 config.CacheSize = 3 21 22 tracker, err := NewCachedBadNodeTracker(hclog.NewNullLogger(), config) 23 require.NoError(t, err) 24 25 for i := 0; i < 10; i++ { 26 tracker.Add(fmt.Sprintf("node-%d", i+1)) 27 } 28 29 require.Equal(t, config.CacheSize, tracker.cache.Len()) 30 31 // Only track the most recent values. 32 expected := []string{"node-8", "node-9", "node-10"} 33 require.ElementsMatch(t, expected, tracker.cache.Keys()) 34 } 35 36 func TestCachedBadNodeTracker_isBad(t *testing.T) { 37 ci.Parallel(t) 38 39 config := DefaultCachedBadNodeTrackerConfig() 40 config.CacheSize = 3 41 config.Threshold = 4 42 43 tracker, err := NewCachedBadNodeTracker(hclog.NewNullLogger(), config) 44 require.NoError(t, err) 45 46 // Populate cache. 47 tracker.Add("node-1") 48 49 tracker.Add("node-2") 50 tracker.Add("node-2") 51 52 tracker.Add("node-3") 53 tracker.Add("node-3") 54 tracker.Add("node-3") 55 tracker.Add("node-3") 56 tracker.Add("node-3") 57 tracker.Add("node-3") 58 59 testCases := []struct { 60 name string 61 nodeID string 62 bad bool 63 }{ 64 { 65 name: "node-1 is not bad", 66 nodeID: "node-1", 67 bad: false, 68 }, 69 { 70 name: "node-3 is bad", 71 nodeID: "node-3", 72 bad: true, 73 }, 74 } 75 76 now := time.Now() 77 for _, tc := range testCases { 78 t.Run(tc.name, func(t *testing.T) { 79 // Read value from cached. 80 stats, ok := tracker.cache.Get(tc.nodeID) 81 require.True(t, ok) 82 83 // Check if it's bad. 84 got := tracker.isBad(now, stats) 85 require.Equal(t, tc.bad, got) 86 }) 87 } 88 89 future := time.Now().Add(2 * config.Window) 90 nodes := []string{"node-1", "node-2", "node-3"} 91 for _, n := range nodes { 92 t.Run(fmt.Sprintf("%s cache expires", n), func(t *testing.T) { 93 stats, ok := tracker.cache.Get(n) 94 require.True(t, ok) 95 96 bad := tracker.isBad(future, stats) 97 require.False(t, bad) 98 }) 99 } 100 } 101 102 func TesCachedtBadNodeTracker_rateLimit(t *testing.T) { 103 ci.Parallel(t) 104 105 config := DefaultCachedBadNodeTrackerConfig() 106 config.Threshold = 3 107 config.RateLimit = float64(1) // Get a new token every second. 108 config.BurstSize = 3 109 110 tracker, err := NewCachedBadNodeTracker(hclog.NewNullLogger(), config) 111 require.NoError(t, err) 112 113 tracker.Add("node-1") 114 tracker.Add("node-1") 115 tracker.Add("node-1") 116 tracker.Add("node-1") 117 tracker.Add("node-1") 118 119 stats, ok := tracker.cache.Get("node-1") 120 require.True(t, ok) 121 122 // Burst allows for max 3 operations. 123 now := time.Now() 124 require.True(t, tracker.isBad(now, stats)) 125 require.True(t, tracker.isBad(now, stats)) 126 require.True(t, tracker.isBad(now, stats)) 127 require.False(t, tracker.isBad(now, stats)) 128 129 // Wait for a new token. 130 time.Sleep(time.Second) 131 now = time.Now() 132 require.True(t, tracker.isBad(now, stats)) 133 require.False(t, tracker.isBad(now, stats)) 134 } 135 136 func TestBadNodeStats_score(t *testing.T) { 137 ci.Parallel(t) 138 139 window := time.Minute 140 stats := newBadNodeStats("node-1", window) 141 142 now := time.Now() 143 require.Equal(t, 0, stats.score(now)) 144 145 stats.record(now) 146 stats.record(now) 147 stats.record(now) 148 require.Equal(t, 3, stats.score(now)) 149 require.Len(t, stats.history, 3) 150 151 halfWindow := now.Add(window / 2) 152 stats.record(halfWindow) 153 require.Equal(t, 4, stats.score(halfWindow)) 154 require.Len(t, stats.history, 4) 155 156 fullWindow := now.Add(window).Add(time.Second) 157 require.Equal(t, 1, stats.score(fullWindow)) 158 require.Len(t, stats.history, 1) 159 160 afterWindow := now.Add(2 * window) 161 require.Equal(t, 0, stats.score(afterWindow)) 162 require.Len(t, stats.history, 0) 163 }