github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/scheduler/reconcile_util.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "sort" 6 "strings" 7 8 "github.com/hashicorp/nomad/nomad/structs" 9 ) 10 11 // placementResult is an allocation that must be placed. It potentionally has a 12 // previous allocation attached to it that should be stopped only if the 13 // paired placement is complete. This gives an atomic place/stop behavior to 14 // prevent an impossible resource ask as part of a rolling update to wipe the 15 // job out. 16 type placementResult interface { 17 // TaskGroup returns the task group the placement is for 18 TaskGroup() *structs.TaskGroup 19 20 // Name returns the name of the desired allocation 21 Name() string 22 23 // Canary returns whether the placement should be a canary 24 Canary() bool 25 26 // PreviousAllocation returns the previous allocation 27 PreviousAllocation() *structs.Allocation 28 29 // StopPreviousAlloc returns whether the previous allocation should be 30 // stopped and if so the status description. 31 StopPreviousAlloc() (bool, string) 32 } 33 34 // allocStopResult contains the information required to stop a single allocation 35 type allocStopResult struct { 36 alloc *structs.Allocation 37 clientStatus string 38 statusDescription string 39 } 40 41 // allocPlaceResult contains the information required to place a single 42 // allocation 43 type allocPlaceResult struct { 44 name string 45 canary bool 46 taskGroup *structs.TaskGroup 47 previousAlloc *structs.Allocation 48 } 49 50 func (a allocPlaceResult) TaskGroup() *structs.TaskGroup { return a.taskGroup } 51 func (a allocPlaceResult) Name() string { return a.name } 52 func (a allocPlaceResult) Canary() bool { return a.canary } 53 func (a allocPlaceResult) PreviousAllocation() *structs.Allocation { return a.previousAlloc } 54 func (a allocPlaceResult) StopPreviousAlloc() (bool, string) { return false, "" } 55 56 // allocDestructiveResult contains the information required to do a destructive 57 // update. Destructive changes should be applied atomically, as in the old alloc 58 // is only stopped if the new one can be placed. 59 type allocDestructiveResult struct { 60 placeName string 61 placeTaskGroup *structs.TaskGroup 62 stopAlloc *structs.Allocation 63 stopStatusDescription string 64 } 65 66 func (a allocDestructiveResult) TaskGroup() *structs.TaskGroup { return a.placeTaskGroup } 67 func (a allocDestructiveResult) Name() string { return a.placeName } 68 func (a allocDestructiveResult) Canary() bool { return false } 69 func (a allocDestructiveResult) PreviousAllocation() *structs.Allocation { return a.stopAlloc } 70 func (a allocDestructiveResult) StopPreviousAlloc() (bool, string) { 71 return true, a.stopStatusDescription 72 } 73 74 // allocMatrix is a mapping of task groups to their allocation set. 75 type allocMatrix map[string]allocSet 76 77 // newAllocMatrix takes a job and the existing allocations for the job and 78 // creates an allocMatrix 79 func newAllocMatrix(job *structs.Job, allocs []*structs.Allocation) allocMatrix { 80 m := allocMatrix(make(map[string]allocSet)) 81 for _, a := range allocs { 82 s, ok := m[a.TaskGroup] 83 if !ok { 84 s = make(map[string]*structs.Allocation) 85 m[a.TaskGroup] = s 86 } 87 s[a.ID] = a 88 } 89 90 if job != nil { 91 for _, tg := range job.TaskGroups { 92 s, ok := m[tg.Name] 93 if !ok { 94 s = make(map[string]*structs.Allocation) 95 m[tg.Name] = s 96 } 97 } 98 } 99 return m 100 } 101 102 // allocSet is a set of allocations with a series of helper functions defined 103 // that help reconcile state. 104 type allocSet map[string]*structs.Allocation 105 106 // newAllocSet creates an allocation set given a set of allocations 107 func newAllocSet(allocs []*structs.Allocation) allocSet { 108 s := make(map[string]*structs.Allocation, len(allocs)) 109 for _, a := range allocs { 110 s[a.ID] = a 111 } 112 return s 113 } 114 115 // GoString provides a human readable view of the set 116 func (a allocSet) GoString() string { 117 if len(a) == 0 { 118 return "[]" 119 } 120 121 start := fmt.Sprintf("len(%d) [\n", len(a)) 122 var s []string 123 for k, v := range a { 124 s = append(s, fmt.Sprintf("%q: %v", k, v.Name)) 125 } 126 return start + strings.Join(s, "\n") + "]" 127 } 128 129 // nameSet returns the set of allocation names 130 func (a allocSet) nameSet() map[string]struct{} { 131 names := make(map[string]struct{}, len(a)) 132 for _, alloc := range a { 133 names[alloc.Name] = struct{}{} 134 } 135 return names 136 } 137 138 // nameOrder returns the set of allocation names in sorted order 139 func (a allocSet) nameOrder() []*structs.Allocation { 140 allocs := make([]*structs.Allocation, 0, len(a)) 141 for _, alloc := range a { 142 allocs = append(allocs, alloc) 143 } 144 sort.Slice(allocs, func(i, j int) bool { 145 return allocs[i].Index() < allocs[j].Index() 146 }) 147 return allocs 148 } 149 150 // difference returns a new allocSet that has all the existing item except those 151 // contained within the other allocation sets 152 func (a allocSet) difference(others ...allocSet) allocSet { 153 diff := make(map[string]*structs.Allocation) 154 OUTER: 155 for k, v := range a { 156 for _, other := range others { 157 if _, ok := other[k]; ok { 158 continue OUTER 159 } 160 } 161 diff[k] = v 162 } 163 return diff 164 } 165 166 // union returns a new allocSet that has the union of the two allocSets. 167 // Conflicts prefer the last passed allocSet containing the value 168 func (a allocSet) union(others ...allocSet) allocSet { 169 union := make(map[string]*structs.Allocation, len(a)) 170 order := []allocSet{a} 171 order = append(order, others...) 172 173 for _, set := range order { 174 for k, v := range set { 175 union[k] = v 176 } 177 } 178 179 return union 180 } 181 182 // fromKeys returns an alloc set matching the passed keys 183 func (a allocSet) fromKeys(keys ...[]string) allocSet { 184 from := make(map[string]*structs.Allocation) 185 for _, set := range keys { 186 for _, k := range set { 187 if alloc, ok := a[k]; ok { 188 from[k] = alloc 189 } 190 } 191 } 192 return from 193 } 194 195 // fitlerByTainted takes a set of tainted nodes and filters the allocation set 196 // into three groups: 197 // 1. Those that exist on untainted nodes 198 // 2. Those exist on nodes that are draining 199 // 3. Those that exist on lost nodes 200 func (a allocSet) filterByTainted(nodes map[string]*structs.Node) (untainted, migrate, lost allocSet) { 201 untainted = make(map[string]*structs.Allocation) 202 migrate = make(map[string]*structs.Allocation) 203 lost = make(map[string]*structs.Allocation) 204 for _, alloc := range a { 205 n, ok := nodes[alloc.NodeID] 206 if !ok { 207 untainted[alloc.ID] = alloc 208 continue 209 } 210 211 // If the job is batch and finished successfully, the fact that the 212 // node is tainted does not mean it should be migrated or marked as 213 // lost as the work was already successfully finished. However for 214 // service/system jobs, tasks should never complete. The check of 215 // batch type, defends against client bugs. 216 if alloc.Job.Type == structs.JobTypeBatch && alloc.RanSuccessfully() { 217 untainted[alloc.ID] = alloc 218 continue 219 } 220 221 if n == nil || n.TerminalStatus() { 222 lost[alloc.ID] = alloc 223 } else { 224 migrate[alloc.ID] = alloc 225 } 226 } 227 return 228 } 229 230 // filterByDeployment filters allocations into two sets, those that match the 231 // given deployment ID and those that don't 232 func (a allocSet) filterByDeployment(id string) (match, nonmatch allocSet) { 233 match = make(map[string]*structs.Allocation) 234 nonmatch = make(map[string]*structs.Allocation) 235 for _, alloc := range a { 236 if alloc.DeploymentID == id { 237 match[alloc.ID] = alloc 238 } else { 239 nonmatch[alloc.ID] = alloc 240 } 241 } 242 return 243 } 244 245 // allocNameIndex is used to select allocation names for placement or removal 246 // given an existing set of placed allocations. 247 type allocNameIndex struct { 248 job, taskGroup string 249 count int 250 b structs.Bitmap 251 } 252 253 // newAllocNameIndex returns an allocNameIndex for use in selecting names of 254 // allocations to create or stop. It takes the job and task group name, desired 255 // count and any existing allocations as input. 256 func newAllocNameIndex(job, taskGroup string, count int, in allocSet) *allocNameIndex { 257 return &allocNameIndex{ 258 count: count, 259 b: bitmapFrom(in, uint(count)), 260 job: job, 261 taskGroup: taskGroup, 262 } 263 } 264 265 // bitmapFrom creates a bitmap from the given allocation set and a minimum size 266 // maybe given. The size of the bitmap is as the larger of the passed minimum 267 // and t the maximum alloc index of the passed input (byte alligned). 268 func bitmapFrom(input allocSet, minSize uint) structs.Bitmap { 269 var max uint 270 for _, a := range input { 271 if num := a.Index(); num > max { 272 max = num 273 } 274 } 275 276 if l := uint(len(input)); minSize < l { 277 minSize = l 278 } 279 if max < minSize { 280 max = minSize 281 } 282 if max == 0 { 283 max = 8 284 } 285 286 // byteAlign the count 287 if remainder := max % 8; remainder != 0 { 288 max = max + 8 - remainder 289 } 290 291 bitmap, err := structs.NewBitmap(max) 292 if err != nil { 293 panic(err) 294 } 295 296 for _, a := range input { 297 bitmap.Set(a.Index()) 298 } 299 300 return bitmap 301 } 302 303 // RemoveHighest removes and returns the hightest n used names. The returned set 304 // can be less than n if there aren't n names set in the index 305 func (a *allocNameIndex) Highest(n uint) map[string]struct{} { 306 h := make(map[string]struct{}, n) 307 for i := a.b.Size(); i > uint(0) && uint(len(h)) < n; i-- { 308 // Use this to avoid wrapping around b/c of the unsigned int 309 idx := i - 1 310 if a.b.Check(idx) { 311 a.b.Unset(idx) 312 h[structs.AllocName(a.job, a.taskGroup, idx)] = struct{}{} 313 } 314 } 315 316 return h 317 } 318 319 // Set sets the indexes from the passed alloc set as used 320 func (a *allocNameIndex) Set(set allocSet) { 321 for _, alloc := range set { 322 a.b.Set(alloc.Index()) 323 } 324 } 325 326 // Unset unsets all indexes of the passed alloc set as being used 327 func (a *allocNameIndex) Unset(as allocSet) { 328 for _, alloc := range as { 329 a.b.Unset(alloc.Index()) 330 } 331 } 332 333 // UnsetIndex unsets the index as having its name used 334 func (a *allocNameIndex) UnsetIndex(idx uint) { 335 a.b.Unset(idx) 336 } 337 338 // NextCanaries returns the next n names for use as canaries and sets them as 339 // used. The existing canaries and destructive updates are also passed in. 340 func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) []string { 341 next := make([]string, 0, n) 342 343 // Create a name index 344 existingNames := existing.nameSet() 345 346 // First select indexes from the allocations that are undergoing destructive 347 // updates. This way we avoid duplicate names as they will get replaced. 348 dmap := bitmapFrom(destructive, uint(a.count)) 349 var remainder uint 350 for _, idx := range dmap.IndexesInRange(true, uint(0), uint(a.count)-1) { 351 name := structs.AllocName(a.job, a.taskGroup, uint(idx)) 352 if _, used := existingNames[name]; !used { 353 next = append(next, name) 354 a.b.Set(uint(idx)) 355 356 // If we have enough, return 357 remainder := n - uint(len(next)) 358 if remainder == 0 { 359 return next 360 } 361 } 362 } 363 364 // Get the set of unset names that can be used 365 for _, idx := range a.b.IndexesInRange(false, uint(0), uint(a.count)-1) { 366 name := structs.AllocName(a.job, a.taskGroup, uint(idx)) 367 if _, used := existingNames[name]; !used { 368 next = append(next, name) 369 a.b.Set(uint(idx)) 370 371 // If we have enough, return 372 remainder = n - uint(len(next)) 373 if remainder == 0 { 374 return next 375 } 376 } 377 } 378 379 // We have exhausted the prefered and free set, now just pick overlapping 380 // indexes 381 var i uint 382 for i = 0; i < remainder; i++ { 383 name := structs.AllocName(a.job, a.taskGroup, i) 384 if _, used := existingNames[name]; !used { 385 next = append(next, name) 386 a.b.Set(i) 387 388 // If we have enough, return 389 remainder = n - uint(len(next)) 390 if remainder == 0 { 391 return next 392 } 393 } 394 } 395 396 return next 397 } 398 399 // Next returns the next n names for use as new placements and sets them as 400 // used. 401 func (a *allocNameIndex) Next(n uint) []string { 402 next := make([]string, 0, n) 403 404 // Get the set of unset names that can be used 405 remainder := n 406 for _, idx := range a.b.IndexesInRange(false, uint(0), uint(a.count)-1) { 407 next = append(next, structs.AllocName(a.job, a.taskGroup, uint(idx))) 408 a.b.Set(uint(idx)) 409 410 // If we have enough, return 411 remainder = n - uint(len(next)) 412 if remainder == 0 { 413 return next 414 } 415 } 416 417 // We have exhausted the free set, now just pick overlapping indexes 418 var i uint 419 for i = 0; i < remainder; i++ { 420 next = append(next, structs.AllocName(a.job, a.taskGroup, i)) 421 a.b.Set(i) 422 } 423 424 return next 425 }