github.com/clly/consul@v1.4.5/agent/consul/acl_replication_legacy_test.go (about) 1 package consul 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "os" 8 "reflect" 9 "sort" 10 "strconv" 11 "strings" 12 "testing" 13 "time" 14 15 "github.com/hashicorp/consul/agent/structs" 16 tokenStore "github.com/hashicorp/consul/agent/token" 17 "github.com/hashicorp/consul/testrpc" 18 "github.com/hashicorp/consul/testutil/retry" 19 ) 20 21 func TestACLReplication_Sorter(t *testing.T) { 22 t.Parallel() 23 acls := structs.ACLs{ 24 &structs.ACL{ID: "a"}, 25 &structs.ACL{ID: "b"}, 26 &structs.ACL{ID: "c"}, 27 } 28 29 sorter := &aclIterator{acls, 0} 30 if len := sorter.Len(); len != 3 { 31 t.Fatalf("bad: %d", len) 32 } 33 if !sorter.Less(0, 1) { 34 t.Fatalf("should be less") 35 } 36 if sorter.Less(1, 0) { 37 t.Fatalf("should not be less") 38 } 39 if !sort.IsSorted(sorter) { 40 t.Fatalf("should be sorted") 41 } 42 43 expected := structs.ACLs{ 44 &structs.ACL{ID: "b"}, 45 &structs.ACL{ID: "a"}, 46 &structs.ACL{ID: "c"}, 47 } 48 sorter.Swap(0, 1) 49 if !reflect.DeepEqual(acls, expected) { 50 t.Fatalf("bad: %v", acls) 51 } 52 if sort.IsSorted(sorter) { 53 t.Fatalf("should not be sorted") 54 } 55 sort.Sort(sorter) 56 if !sort.IsSorted(sorter) { 57 t.Fatalf("should be sorted") 58 } 59 } 60 61 func TestACLReplication_Iterator(t *testing.T) { 62 t.Parallel() 63 acls := structs.ACLs{} 64 65 iter := newACLIterator(acls) 66 if front := iter.Front(); front != nil { 67 t.Fatalf("bad: %v", front) 68 } 69 iter.Next() 70 if front := iter.Front(); front != nil { 71 t.Fatalf("bad: %v", front) 72 } 73 74 acls = structs.ACLs{ 75 &structs.ACL{ID: "a"}, 76 &structs.ACL{ID: "b"}, 77 &structs.ACL{ID: "c"}, 78 } 79 iter = newACLIterator(acls) 80 if front := iter.Front(); front != acls[0] { 81 t.Fatalf("bad: %v", front) 82 } 83 iter.Next() 84 if front := iter.Front(); front != acls[1] { 85 t.Fatalf("bad: %v", front) 86 } 87 iter.Next() 88 if front := iter.Front(); front != acls[2] { 89 t.Fatalf("bad: %v", front) 90 } 91 iter.Next() 92 if front := iter.Front(); front != nil { 93 t.Fatalf("bad: %v", front) 94 } 95 } 96 97 func TestACLReplication_reconcileACLs(t *testing.T) { 98 t.Parallel() 99 parseACLs := func(raw string) structs.ACLs { 100 var acls structs.ACLs 101 for _, key := range strings.Split(raw, "|") { 102 if len(key) == 0 { 103 continue 104 } 105 106 tuple := strings.Split(key, ":") 107 index, err := strconv.Atoi(tuple[1]) 108 if err != nil { 109 t.Fatalf("err: %v", err) 110 } 111 acl := &structs.ACL{ 112 ID: tuple[0], 113 Rules: tuple[2], 114 RaftIndex: structs.RaftIndex{ 115 ModifyIndex: uint64(index), 116 }, 117 } 118 acls = append(acls, acl) 119 } 120 return acls 121 } 122 123 parseChanges := func(changes structs.ACLRequests) string { 124 var ret string 125 for i, change := range changes { 126 if i > 0 { 127 ret += "|" 128 } 129 ret += fmt.Sprintf("%s:%s:%s", change.Op, change.ACL.ID, change.ACL.Rules) 130 } 131 return ret 132 } 133 134 tests := []struct { 135 local string 136 remote string 137 lastRemoteIndex uint64 138 expected string 139 }{ 140 // Everything empty. 141 { 142 local: "", 143 remote: "", 144 lastRemoteIndex: 0, 145 expected: "", 146 }, 147 // First time with empty local. 148 { 149 local: "", 150 remote: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 151 lastRemoteIndex: 0, 152 expected: "set:bbb:X|set:ccc:X|set:ddd:X|set:eee:X", 153 }, 154 // Remote not sorted. 155 { 156 local: "", 157 remote: "ddd:2:X|bbb:3:X|ccc:9:X|eee:11:X", 158 lastRemoteIndex: 0, 159 expected: "set:bbb:X|set:ccc:X|set:ddd:X|set:eee:X", 160 }, 161 // Neither side sorted. 162 { 163 local: "ddd:2:X|bbb:3:X|ccc:9:X|eee:11:X", 164 remote: "ccc:9:X|bbb:3:X|ddd:2:X|eee:11:X", 165 lastRemoteIndex: 0, 166 expected: "", 167 }, 168 // Fully replicated, nothing to do. 169 { 170 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 171 remote: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 172 lastRemoteIndex: 0, 173 expected: "", 174 }, 175 // Change an ACL. 176 { 177 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 178 remote: "bbb:3:X|ccc:33:Y|ddd:2:X|eee:11:X", 179 lastRemoteIndex: 0, 180 expected: "set:ccc:Y", 181 }, 182 // Change an ACL, but mask the change by the last replicated 183 // index. This isn't how things work normally, but it proves 184 // we are skipping the full compare based on the index. 185 { 186 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 187 remote: "bbb:3:X|ccc:33:Y|ddd:2:X|eee:11:X", 188 lastRemoteIndex: 33, 189 expected: "", 190 }, 191 // Empty everything out. 192 { 193 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 194 remote: "", 195 lastRemoteIndex: 0, 196 expected: "delete:bbb:X|delete:ccc:X|delete:ddd:X|delete:eee:X", 197 }, 198 // Adds on the ends and in the middle. 199 { 200 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 201 remote: "aaa:99:X|bbb:3:X|ccc:9:X|ccx:101:X|ddd:2:X|eee:11:X|fff:102:X", 202 lastRemoteIndex: 0, 203 expected: "set:aaa:X|set:ccx:X|set:fff:X", 204 }, 205 // Deletes on the ends and in the middle. 206 { 207 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 208 remote: "ccc:9:X", 209 lastRemoteIndex: 0, 210 expected: "delete:bbb:X|delete:ddd:X|delete:eee:X", 211 }, 212 // Everything. 213 { 214 local: "bbb:3:X|ccc:9:X|ddd:2:X|eee:11:X", 215 remote: "aaa:99:X|bbb:3:X|ccx:101:X|ddd:103:Y|eee:11:X|fff:102:X", 216 lastRemoteIndex: 11, 217 expected: "set:aaa:X|delete:ccc:X|set:ccx:X|set:ddd:Y|set:fff:X", 218 }, 219 } 220 for i, test := range tests { 221 local, remote := parseACLs(test.local), parseACLs(test.remote) 222 changes := reconcileLegacyACLs(local, remote, test.lastRemoteIndex) 223 if actual := parseChanges(changes); actual != test.expected { 224 t.Errorf("test case %d failed: %s", i, actual) 225 } 226 } 227 } 228 229 func TestACLReplication_updateLocalACLs_RateLimit(t *testing.T) { 230 t.Parallel() 231 dir1, s1 := testServerWithConfig(t, func(c *Config) { 232 c.Datacenter = "dc2" 233 c.ACLDatacenter = "dc1" 234 c.ACLsEnabled = true 235 c.ACLReplicationApplyLimit = 1 236 }) 237 s1.tokens.UpdateReplicationToken("secret", tokenStore.TokenSourceConfig) 238 defer os.RemoveAll(dir1) 239 defer s1.Shutdown() 240 testrpc.WaitForLeader(t, s1.RPC, "dc2") 241 242 changes := structs.ACLRequests{ 243 &structs.ACLRequest{ 244 Op: structs.ACLSet, 245 ACL: structs.ACL{ 246 ID: "secret", 247 Type: "client", 248 }, 249 }, 250 } 251 252 // Should be throttled to 1 Hz. 253 start := time.Now() 254 if _, err := s1.updateLocalLegacyACLs(changes, context.Background()); err != nil { 255 t.Fatalf("err: %v", err) 256 } 257 if dur := time.Since(start); dur < time.Second { 258 t.Fatalf("too slow: %9.6f", dur.Seconds()) 259 } 260 261 changes = append(changes, 262 &structs.ACLRequest{ 263 Op: structs.ACLSet, 264 ACL: structs.ACL{ 265 ID: "secret", 266 Type: "client", 267 }, 268 }) 269 270 // Should be throttled to 1 Hz. 271 start = time.Now() 272 if _, err := s1.updateLocalLegacyACLs(changes, context.Background()); err != nil { 273 t.Fatalf("err: %v", err) 274 } 275 if dur := time.Since(start); dur < 2*time.Second { 276 t.Fatalf("too fast: %9.6f", dur.Seconds()) 277 } 278 } 279 280 func TestACLReplication_IsACLReplicationEnabled(t *testing.T) { 281 t.Parallel() 282 // ACLs not enabled. 283 dir1, s1 := testServerWithConfig(t, func(c *Config) { 284 c.ACLDatacenter = "" 285 c.ACLsEnabled = false 286 }) 287 defer os.RemoveAll(dir1) 288 defer s1.Shutdown() 289 if s1.IsACLReplicationEnabled() { 290 t.Fatalf("should not be enabled") 291 } 292 293 // ACLs enabled but not replication. 294 dir2, s2 := testServerWithConfig(t, func(c *Config) { 295 c.Datacenter = "dc2" 296 c.ACLDatacenter = "dc1" 297 c.ACLsEnabled = true 298 }) 299 defer os.RemoveAll(dir2) 300 defer s2.Shutdown() 301 testrpc.WaitForLeader(t, s1.RPC, "dc1") 302 testrpc.WaitForLeader(t, s2.RPC, "dc2") 303 304 if s2.IsACLReplicationEnabled() { 305 t.Fatalf("should not be enabled") 306 } 307 308 // ACLs enabled with replication. 309 dir3, s3 := testServerWithConfig(t, func(c *Config) { 310 c.Datacenter = "dc2" 311 c.ACLDatacenter = "dc1" 312 c.ACLsEnabled = true 313 c.ACLTokenReplication = true 314 }) 315 defer os.RemoveAll(dir3) 316 defer s3.Shutdown() 317 testrpc.WaitForLeader(t, s3.RPC, "dc2") 318 if !s3.IsACLReplicationEnabled() { 319 t.Fatalf("should be enabled") 320 } 321 322 // ACLs enabled with replication, but inside the ACL datacenter 323 // so replication should be disabled. 324 dir4, s4 := testServerWithConfig(t, func(c *Config) { 325 c.Datacenter = "dc1" 326 c.ACLDatacenter = "dc1" 327 c.ACLsEnabled = true 328 c.ACLTokenReplication = true 329 }) 330 defer os.RemoveAll(dir4) 331 defer s4.Shutdown() 332 testrpc.WaitForLeader(t, s4.RPC, "dc1") 333 if s4.IsACLReplicationEnabled() { 334 t.Fatalf("should not be enabled") 335 } 336 } 337 338 func TestACLReplication_LegacyTokens(t *testing.T) { 339 t.Parallel() 340 dir1, s1 := testServerWithConfig(t, func(c *Config) { 341 c.ACLDatacenter = "dc1" 342 c.ACLsEnabled = true 343 c.ACLMasterToken = "root" 344 }) 345 defer os.RemoveAll(dir1) 346 defer s1.Shutdown() 347 testrpc.WaitForLeader(t, s1.RPC, "dc1") 348 client := rpcClient(t, s1) 349 defer client.Close() 350 351 dir2, s2 := testServerWithConfig(t, func(c *Config) { 352 c.Datacenter = "dc2" 353 c.ACLDatacenter = "dc1" 354 c.ACLsEnabled = true 355 c.ACLTokenReplication = true 356 c.ACLReplicationRate = 100 357 c.ACLReplicationBurst = 100 358 c.ACLReplicationApplyLimit = 1000000 359 }) 360 s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig) 361 testrpc.WaitForLeader(t, s2.RPC, "dc2") 362 defer os.RemoveAll(dir2) 363 defer s2.Shutdown() 364 365 // Try to join. 366 joinWAN(t, s2, s1) 367 testrpc.WaitForLeader(t, s1.RPC, "dc1") 368 testrpc.WaitForLeader(t, s1.RPC, "dc2") 369 370 // Create a bunch of new tokens. 371 var id string 372 for i := 0; i < 50; i++ { 373 arg := structs.ACLRequest{ 374 Datacenter: "dc1", 375 Op: structs.ACLSet, 376 ACL: structs.ACL{ 377 Name: "User token", 378 Type: structs.ACLTokenTypeClient, 379 Rules: testACLPolicy, 380 }, 381 WriteRequest: structs.WriteRequest{Token: "root"}, 382 } 383 if err := s1.RPC("ACL.Apply", &arg, &id); err != nil { 384 t.Fatalf("err: %v", err) 385 } 386 } 387 388 checkSame := func() error { 389 index, remote, err := s1.fsm.State().ACLTokenList(nil, true, true, "") 390 if err != nil { 391 return err 392 } 393 _, local, err := s2.fsm.State().ACLTokenList(nil, true, true, "") 394 if err != nil { 395 return err 396 } 397 if got, want := len(remote), len(local); got != want { 398 return fmt.Errorf("got %d remote ACLs want %d", got, want) 399 } 400 for i, token := range remote { 401 if !bytes.Equal(token.Hash, local[i].Hash) { 402 return fmt.Errorf("ACLs differ") 403 } 404 } 405 406 var status structs.ACLReplicationStatus 407 s2.aclReplicationStatusLock.RLock() 408 status = s2.aclReplicationStatus 409 s2.aclReplicationStatusLock.RUnlock() 410 if !status.Enabled || !status.Running || 411 status.ReplicatedTokenIndex != index || 412 status.SourceDatacenter != "dc1" { 413 return fmt.Errorf("ACL replication status differs") 414 } 415 416 return nil 417 } 418 // Wait for the replica to converge. 419 retry.Run(t, func(r *retry.R) { 420 if err := checkSame(); err != nil { 421 r.Fatal(err) 422 } 423 }) 424 425 // Create more new tokens. 426 for i := 0; i < 50; i++ { 427 arg := structs.ACLRequest{ 428 Datacenter: "dc1", 429 Op: structs.ACLSet, 430 ACL: structs.ACL{ 431 Name: "User token", 432 Type: structs.ACLTokenTypeClient, 433 Rules: testACLPolicy, 434 }, 435 WriteRequest: structs.WriteRequest{Token: "root"}, 436 } 437 var dontCare string 438 if err := s1.RPC("ACL.Apply", &arg, &dontCare); err != nil { 439 t.Fatalf("err: %v", err) 440 } 441 } 442 // Wait for the replica to converge. 443 retry.Run(t, func(r *retry.R) { 444 if err := checkSame(); err != nil { 445 r.Fatal(err) 446 } 447 }) 448 449 // Delete a token. 450 arg := structs.ACLRequest{ 451 Datacenter: "dc1", 452 Op: structs.ACLDelete, 453 ACL: structs.ACL{ 454 ID: id, 455 }, 456 WriteRequest: structs.WriteRequest{Token: "root"}, 457 } 458 var dontCare string 459 if err := s1.RPC("ACL.Apply", &arg, &dontCare); err != nil { 460 t.Fatalf("err: %v", err) 461 } 462 // Wait for the replica to converge. 463 retry.Run(t, func(r *retry.R) { 464 if err := checkSame(); err != nil { 465 r.Fatal(err) 466 } 467 }) 468 }