github.com/DerekStrickland/consul@v1.4.5/agent/consul/acl_replication_test.go (about) 1 package consul 2 3 import ( 4 "fmt" 5 "os" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/consul/acl" 10 "github.com/hashicorp/consul/agent/structs" 11 tokenStore "github.com/hashicorp/consul/agent/token" 12 "github.com/hashicorp/consul/testrpc" 13 "github.com/hashicorp/consul/testutil/retry" 14 "github.com/stretchr/testify/require" 15 ) 16 17 func TestACLReplication_diffACLPolicies(t *testing.T) { 18 local := structs.ACLPolicies{ 19 &structs.ACLPolicy{ 20 ID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 21 Name: "policy1", 22 Description: "policy1 - already in sync", 23 Rules: `acl = "read"`, 24 Syntax: acl.SyntaxCurrent, 25 Datacenters: nil, 26 Hash: []byte{1, 2, 3, 4}, 27 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 2}, 28 }, 29 &structs.ACLPolicy{ 30 ID: "8ea41efb-8519-4091-bc91-c42da0cda9ae", 31 Name: "policy2", 32 Description: "policy2 - updated but not changed", 33 Rules: `acl = "read"`, 34 Syntax: acl.SyntaxCurrent, 35 Datacenters: nil, 36 Hash: []byte{1, 2, 3, 4}, 37 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25}, 38 }, 39 &structs.ACLPolicy{ 40 ID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 41 Name: "policy3", 42 Description: "policy3 - updated and changed", 43 Rules: `acl = "read"`, 44 Syntax: acl.SyntaxCurrent, 45 Datacenters: nil, 46 Hash: []byte{1, 2, 3, 4}, 47 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25}, 48 }, 49 &structs.ACLPolicy{ 50 ID: "e9d33298-6490-4466-99cb-ba93af64fa76", 51 Name: "policy4", 52 Description: "policy4 - needs deleting", 53 Rules: `acl = "read"`, 54 Syntax: acl.SyntaxCurrent, 55 Datacenters: nil, 56 Hash: []byte{1, 2, 3, 4}, 57 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25}, 58 }, 59 } 60 61 remote := structs.ACLPolicyListStubs{ 62 &structs.ACLPolicyListStub{ 63 ID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 64 Name: "policy1", 65 Description: "policy1 - already in sync", 66 Datacenters: nil, 67 Hash: []byte{1, 2, 3, 4}, 68 CreateIndex: 1, 69 ModifyIndex: 2, 70 }, 71 &structs.ACLPolicyListStub{ 72 ID: "8ea41efb-8519-4091-bc91-c42da0cda9ae", 73 Name: "policy2", 74 Description: "policy2 - updated but not changed", 75 Datacenters: nil, 76 Hash: []byte{1, 2, 3, 4}, 77 CreateIndex: 1, 78 ModifyIndex: 50, 79 }, 80 &structs.ACLPolicyListStub{ 81 ID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 82 Name: "policy3", 83 Description: "policy3 - updated and changed", 84 Datacenters: nil, 85 Hash: []byte{5, 6, 7, 8}, 86 CreateIndex: 1, 87 ModifyIndex: 50, 88 }, 89 &structs.ACLPolicyListStub{ 90 ID: "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926", 91 Name: "policy5", 92 Description: "policy5 - needs adding", 93 Datacenters: nil, 94 Hash: []byte{1, 2, 3, 4}, 95 CreateIndex: 1, 96 ModifyIndex: 50, 97 }, 98 } 99 100 // Do the full diff. This full exercises the main body of the loop 101 deletions, updates := diffACLPolicies(local, remote, 28) 102 require.Len(t, updates, 2) 103 require.ElementsMatch(t, updates, []string{ 104 "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926", 105 "539f1cb6-40aa-464f-ae66-a900d26bc1b2"}) 106 107 require.Len(t, deletions, 1) 108 require.Equal(t, "e9d33298-6490-4466-99cb-ba93af64fa76", deletions[0]) 109 110 deletions, updates = diffACLPolicies(local, nil, 28) 111 require.Len(t, updates, 0) 112 require.Len(t, deletions, 4) 113 require.ElementsMatch(t, deletions, []string{ 114 "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 115 "8ea41efb-8519-4091-bc91-c42da0cda9ae", 116 "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 117 "e9d33298-6490-4466-99cb-ba93af64fa76"}) 118 119 deletions, updates = diffACLPolicies(nil, remote, 28) 120 require.Len(t, deletions, 0) 121 require.Len(t, updates, 4) 122 require.ElementsMatch(t, updates, []string{ 123 "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 124 "8ea41efb-8519-4091-bc91-c42da0cda9ae", 125 "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 126 "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926"}) 127 } 128 129 func TestACLReplication_diffACLTokens(t *testing.T) { 130 local := structs.ACLTokens{ 131 // When a just-upgraded (1.3->1.4+) secondary DC is replicating from an 132 // upgraded primary DC (1.4+), the local state for tokens predating the 133 // upgrade will lack AccessorIDs. 134 // 135 // The primary DC will lazily perform the update to assign AccessorIDs, 136 // and that new update will come across the wire locally as a new 137 // insert. 138 // 139 // We simulate that scenario here with 'token0' having no AccessorID in 140 // the secondary (local) DC and having an AccessorID assigned in the 141 // payload retrieved from the primary (remote) DC. 142 &structs.ACLToken{ 143 AccessorID: "", 144 SecretID: "5128289f-c22c-4d32-936e-7662443f1a55", 145 Description: "token0 - old and not yet upgraded", 146 Hash: []byte{1, 2, 3, 4}, 147 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 3}, 148 }, 149 &structs.ACLToken{ 150 AccessorID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 151 SecretID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 152 Description: "token1 - already in sync", 153 Hash: []byte{1, 2, 3, 4}, 154 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 2}, 155 }, 156 &structs.ACLToken{ 157 AccessorID: "8ea41efb-8519-4091-bc91-c42da0cda9ae", 158 SecretID: "8ea41efb-8519-4091-bc91-c42da0cda9ae", 159 Description: "token2 - updated but not changed", 160 Hash: []byte{1, 2, 3, 4}, 161 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25}, 162 }, 163 &structs.ACLToken{ 164 AccessorID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 165 SecretID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 166 Description: "token3 - updated and changed", 167 Hash: []byte{1, 2, 3, 4}, 168 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25}, 169 }, 170 &structs.ACLToken{ 171 AccessorID: "e9d33298-6490-4466-99cb-ba93af64fa76", 172 SecretID: "e9d33298-6490-4466-99cb-ba93af64fa76", 173 Description: "token4 - needs deleting", 174 Hash: []byte{1, 2, 3, 4}, 175 RaftIndex: structs.RaftIndex{CreateIndex: 1, ModifyIndex: 25}, 176 }, 177 } 178 179 remote := structs.ACLTokenListStubs{ 180 &structs.ACLTokenListStub{ 181 AccessorID: "72fac6a3-a014-41c8-9cb2-8d9a5e935f3d", 182 //SecretID: "5128289f-c22c-4d32-936e-7662443f1a55", (formerly) 183 Description: "token0 - old and not yet upgraded locally", 184 Hash: []byte{1, 2, 3, 4}, 185 CreateIndex: 1, 186 ModifyIndex: 3, 187 }, 188 &structs.ACLTokenListStub{ 189 AccessorID: "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 190 Description: "token1 - already in sync", 191 Hash: []byte{1, 2, 3, 4}, 192 CreateIndex: 1, 193 ModifyIndex: 2, 194 }, 195 &structs.ACLTokenListStub{ 196 AccessorID: "8ea41efb-8519-4091-bc91-c42da0cda9ae", 197 Description: "token2 - updated but not changed", 198 Hash: []byte{1, 2, 3, 4}, 199 CreateIndex: 1, 200 ModifyIndex: 50, 201 }, 202 &structs.ACLTokenListStub{ 203 AccessorID: "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 204 Description: "token3 - updated and changed", 205 Hash: []byte{5, 6, 7, 8}, 206 CreateIndex: 1, 207 ModifyIndex: 50, 208 }, 209 &structs.ACLTokenListStub{ 210 AccessorID: "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926", 211 Description: "token5 - needs adding", 212 Hash: []byte{1, 2, 3, 4}, 213 CreateIndex: 1, 214 ModifyIndex: 50, 215 }, 216 // When a 1.4+ secondary DC is replicating from a 1.4+ primary DC, 217 // tokens created using the legacy APIs will not initially have 218 // AccessorIDs assigned. That assignment is lazy (but in quick 219 // succession). 220 // 221 // The secondary (local) will see these in the api response as a stub 222 // with "" as the AccessorID. 223 // 224 // We simulate that here to verify that the secondary does the right 225 // thing by skipping them until it sees them with nonempty AccessorIDs. 226 &structs.ACLTokenListStub{ 227 AccessorID: "", 228 Description: "token6 - pending async AccessorID assignment", 229 Hash: []byte{1, 2, 3, 4}, 230 CreateIndex: 51, 231 ModifyIndex: 51, 232 }, 233 } 234 235 // Do the full diff. This full exercises the main body of the loop 236 t.Run("full-diff", func(t *testing.T) { 237 res := diffACLTokens(local, remote, 28) 238 require.Equal(t, 1, res.LocalSkipped) 239 require.Equal(t, 1, res.RemoteSkipped) 240 require.Len(t, res.LocalUpserts, 3) 241 require.ElementsMatch(t, res.LocalUpserts, []string{ 242 "72fac6a3-a014-41c8-9cb2-8d9a5e935f3d", 243 "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926", 244 "539f1cb6-40aa-464f-ae66-a900d26bc1b2"}) 245 246 require.Len(t, res.LocalDeletes, 1) 247 require.Equal(t, "e9d33298-6490-4466-99cb-ba93af64fa76", res.LocalDeletes[0]) 248 }) 249 250 t.Run("only-local", func(t *testing.T) { 251 res := diffACLTokens(local, nil, 28) 252 require.Equal(t, 1, res.LocalSkipped) 253 require.Equal(t, 0, res.RemoteSkipped) 254 require.Len(t, res.LocalUpserts, 0) 255 require.Len(t, res.LocalDeletes, 4) 256 require.ElementsMatch(t, res.LocalDeletes, []string{ 257 "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 258 "8ea41efb-8519-4091-bc91-c42da0cda9ae", 259 "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 260 "e9d33298-6490-4466-99cb-ba93af64fa76"}) 261 }) 262 263 t.Run("only-remote", func(t *testing.T) { 264 res := diffACLTokens(nil, remote, 28) 265 require.Equal(t, 0, res.LocalSkipped) 266 require.Equal(t, 1, res.RemoteSkipped) 267 require.Len(t, res.LocalDeletes, 0) 268 require.Len(t, res.LocalUpserts, 5) 269 require.ElementsMatch(t, res.LocalUpserts, []string{ 270 "72fac6a3-a014-41c8-9cb2-8d9a5e935f3d", 271 "44ef9aec-7654-4401-901b-4d4a8b3c80fc", 272 "8ea41efb-8519-4091-bc91-c42da0cda9ae", 273 "539f1cb6-40aa-464f-ae66-a900d26bc1b2", 274 "c6e8fffd-cbd9-4ecd-99fe-ab2f200c7926"}) 275 }) 276 } 277 278 func TestACLReplication_Tokens(t *testing.T) { 279 t.Parallel() 280 dir1, s1 := testServerWithConfig(t, func(c *Config) { 281 c.ACLDatacenter = "dc1" 282 c.ACLsEnabled = true 283 c.ACLMasterToken = "root" 284 }) 285 defer os.RemoveAll(dir1) 286 defer s1.Shutdown() 287 testrpc.WaitForLeader(t, s1.RPC, "dc1") 288 client := rpcClient(t, s1) 289 defer client.Close() 290 291 dir2, s2 := testServerWithConfig(t, func(c *Config) { 292 c.Datacenter = "dc2" 293 c.ACLDatacenter = "dc1" 294 c.ACLsEnabled = true 295 c.ACLTokenReplication = true 296 c.ACLReplicationRate = 100 297 c.ACLReplicationBurst = 100 298 c.ACLReplicationApplyLimit = 1000000 299 }) 300 s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig) 301 testrpc.WaitForLeader(t, s2.RPC, "dc2") 302 defer os.RemoveAll(dir2) 303 defer s2.Shutdown() 304 305 // Try to join. 306 joinWAN(t, s2, s1) 307 testrpc.WaitForLeader(t, s1.RPC, "dc1") 308 testrpc.WaitForLeader(t, s1.RPC, "dc2") 309 310 // Create a bunch of new tokens and policies 311 var tokens structs.ACLTokens 312 for i := 0; i < 50; i++ { 313 arg := structs.ACLTokenSetRequest{ 314 Datacenter: "dc1", 315 ACLToken: structs.ACLToken{ 316 Description: fmt.Sprintf("token-%d", i), 317 Policies: []structs.ACLTokenPolicyLink{ 318 structs.ACLTokenPolicyLink{ 319 ID: structs.ACLPolicyGlobalManagementID, 320 }, 321 }, 322 Local: false, 323 }, 324 WriteRequest: structs.WriteRequest{Token: "root"}, 325 } 326 var token structs.ACLToken 327 require.NoError(t, s1.RPC("ACL.TokenSet", &arg, &token)) 328 tokens = append(tokens, &token) 329 } 330 331 checkSame := func(t *retry.R) error { 332 // only account for global tokens - local tokens shouldn't be replicated 333 index, remote, err := s1.fsm.State().ACLTokenList(nil, false, true, "") 334 require.NoError(t, err) 335 _, local, err := s2.fsm.State().ACLTokenList(nil, false, true, "") 336 require.NoError(t, err) 337 338 require.Len(t, local, len(remote)) 339 for i, token := range remote { 340 require.Equal(t, token.Hash, local[i].Hash) 341 } 342 343 var status structs.ACLReplicationStatus 344 s2.aclReplicationStatusLock.RLock() 345 status = s2.aclReplicationStatus 346 s2.aclReplicationStatusLock.RUnlock() 347 if !status.Enabled || !status.Running || 348 status.ReplicationType != structs.ACLReplicateTokens || 349 status.ReplicatedTokenIndex != index || 350 status.SourceDatacenter != "dc1" { 351 return fmt.Errorf("ACL replication status differs") 352 } 353 354 return nil 355 } 356 // Wait for the replica to converge. 357 retry.Run(t, func(r *retry.R) { 358 checkSame(r) 359 }) 360 361 // add some local tokens to the secondary DC 362 // these shouldn't be deleted by replication 363 for i := 0; i < 50; i++ { 364 arg := structs.ACLTokenSetRequest{ 365 Datacenter: "dc2", 366 ACLToken: structs.ACLToken{ 367 Description: fmt.Sprintf("token-%d", i), 368 Policies: []structs.ACLTokenPolicyLink{ 369 structs.ACLTokenPolicyLink{ 370 ID: structs.ACLPolicyGlobalManagementID, 371 }, 372 }, 373 Local: true, 374 }, 375 WriteRequest: structs.WriteRequest{Token: "root"}, 376 } 377 var token structs.ACLToken 378 require.NoError(t, s2.RPC("ACL.TokenSet", &arg, &token)) 379 } 380 381 // add some local tokens to the primary DC 382 // these shouldn't be replicated to the secondary DC 383 for i := 0; i < 50; i++ { 384 arg := structs.ACLTokenSetRequest{ 385 Datacenter: "dc1", 386 ACLToken: structs.ACLToken{ 387 Description: fmt.Sprintf("token-%d", i), 388 Policies: []structs.ACLTokenPolicyLink{ 389 structs.ACLTokenPolicyLink{ 390 ID: structs.ACLPolicyGlobalManagementID, 391 }, 392 }, 393 Local: true, 394 }, 395 WriteRequest: structs.WriteRequest{Token: "root"}, 396 } 397 var token structs.ACLToken 398 require.NoError(t, s1.RPC("ACL.TokenSet", &arg, &token)) 399 } 400 401 // Update those other tokens 402 for i := 0; i < 50; i++ { 403 arg := structs.ACLTokenSetRequest{ 404 Datacenter: "dc1", 405 ACLToken: structs.ACLToken{ 406 AccessorID: tokens[i].AccessorID, 407 SecretID: tokens[i].SecretID, 408 Description: fmt.Sprintf("token-%d-modified", i), 409 Policies: []structs.ACLTokenPolicyLink{ 410 structs.ACLTokenPolicyLink{ 411 ID: structs.ACLPolicyGlobalManagementID, 412 }, 413 }, 414 Local: false, 415 }, 416 WriteRequest: structs.WriteRequest{Token: "root"}, 417 } 418 var token structs.ACLToken 419 require.NoError(t, s1.RPC("ACL.TokenSet", &arg, &token)) 420 } 421 422 // Wait for the replica to converge. 423 // this time it also verifies the local tokens from the primary were not replicated. 424 retry.Run(t, func(r *retry.R) { 425 checkSame(r) 426 }) 427 428 // verify dc2 local tokens didn't get blown away 429 _, local, err := s2.fsm.State().ACLTokenList(nil, true, false, "") 430 require.NoError(t, err) 431 require.Len(t, local, 50) 432 433 for _, token := range tokens { 434 arg := structs.ACLTokenDeleteRequest{ 435 Datacenter: "dc1", 436 TokenID: token.AccessorID, 437 WriteRequest: structs.WriteRequest{Token: "root"}, 438 } 439 440 var dontCare string 441 require.NoError(t, s1.RPC("ACL.TokenDelete", &arg, &dontCare)) 442 } 443 444 // Wait for the replica to converge. 445 retry.Run(t, func(r *retry.R) { 446 checkSame(r) 447 }) 448 } 449 450 func TestACLReplication_Policies(t *testing.T) { 451 t.Parallel() 452 dir1, s1 := testServerWithConfig(t, func(c *Config) { 453 c.ACLDatacenter = "dc1" 454 c.ACLsEnabled = true 455 c.ACLMasterToken = "root" 456 }) 457 defer os.RemoveAll(dir1) 458 defer s1.Shutdown() 459 testrpc.WaitForLeader(t, s1.RPC, "dc1") 460 client := rpcClient(t, s1) 461 defer client.Close() 462 463 dir2, s2 := testServerWithConfig(t, func(c *Config) { 464 c.Datacenter = "dc2" 465 c.ACLDatacenter = "dc1" 466 c.ACLsEnabled = true 467 c.ACLTokenReplication = false 468 c.ACLReplicationRate = 100 469 c.ACLReplicationBurst = 100 470 c.ACLReplicationApplyLimit = 1000000 471 }) 472 s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig) 473 testrpc.WaitForLeader(t, s2.RPC, "dc2") 474 defer os.RemoveAll(dir2) 475 defer s2.Shutdown() 476 477 // Try to join. 478 joinWAN(t, s2, s1) 479 testrpc.WaitForLeader(t, s1.RPC, "dc1") 480 testrpc.WaitForLeader(t, s1.RPC, "dc2") 481 482 // Create a bunch of new policies 483 var policies structs.ACLPolicies 484 for i := 0; i < 50; i++ { 485 arg := structs.ACLPolicySetRequest{ 486 Datacenter: "dc1", 487 Policy: structs.ACLPolicy{ 488 Name: fmt.Sprintf("token-%d", i), 489 Description: fmt.Sprintf("token-%d", i), 490 Rules: fmt.Sprintf(`service "app-%d" { policy = "read" }`, i), 491 }, 492 WriteRequest: structs.WriteRequest{Token: "root"}, 493 } 494 var policy structs.ACLPolicy 495 require.NoError(t, s1.RPC("ACL.PolicySet", &arg, &policy)) 496 policies = append(policies, &policy) 497 } 498 499 checkSame := func(t *retry.R) error { 500 // only account for global tokens - local tokens shouldn't be replicated 501 index, remote, err := s1.fsm.State().ACLPolicyList(nil) 502 require.NoError(t, err) 503 _, local, err := s2.fsm.State().ACLPolicyList(nil) 504 require.NoError(t, err) 505 506 require.Len(t, local, len(remote)) 507 for i, policy := range remote { 508 require.Equal(t, policy.Hash, local[i].Hash) 509 } 510 511 var status structs.ACLReplicationStatus 512 s2.aclReplicationStatusLock.RLock() 513 status = s2.aclReplicationStatus 514 s2.aclReplicationStatusLock.RUnlock() 515 if !status.Enabled || !status.Running || 516 status.ReplicationType != structs.ACLReplicatePolicies || 517 status.ReplicatedIndex != index || 518 status.SourceDatacenter != "dc1" { 519 return fmt.Errorf("ACL replication status differs") 520 } 521 522 return nil 523 } 524 // Wait for the replica to converge. 525 retry.Run(t, func(r *retry.R) { 526 checkSame(r) 527 }) 528 529 // Update those policies 530 for i := 0; i < 50; i++ { 531 arg := structs.ACLPolicySetRequest{ 532 Datacenter: "dc1", 533 Policy: structs.ACLPolicy{ 534 ID: policies[i].ID, 535 Name: fmt.Sprintf("token-%d-modified", i), 536 Description: fmt.Sprintf("token-%d-modified", i), 537 Rules: policies[i].Rules, 538 }, 539 WriteRequest: structs.WriteRequest{Token: "root"}, 540 } 541 var policy structs.ACLPolicy 542 require.NoError(t, s1.RPC("ACL.PolicySet", &arg, &policy)) 543 } 544 545 // Wait for the replica to converge. 546 // this time it also verifies the local tokens from the primary were not replicated. 547 retry.Run(t, func(r *retry.R) { 548 checkSame(r) 549 }) 550 551 for _, policy := range policies { 552 arg := structs.ACLPolicyDeleteRequest{ 553 Datacenter: "dc1", 554 PolicyID: policy.ID, 555 WriteRequest: structs.WriteRequest{Token: "root"}, 556 } 557 558 var dontCare string 559 require.NoError(t, s1.RPC("ACL.PolicyDelete", &arg, &dontCare)) 560 } 561 562 // Wait for the replica to converge. 563 retry.Run(t, func(r *retry.R) { 564 checkSame(r) 565 }) 566 } 567 568 func TestACLReplication_TokensRedacted(t *testing.T) { 569 t.Parallel() 570 dir1, s1 := testServerWithConfig(t, func(c *Config) { 571 c.ACLDatacenter = "dc1" 572 c.ACLsEnabled = true 573 c.ACLMasterToken = "root" 574 }) 575 defer os.RemoveAll(dir1) 576 defer s1.Shutdown() 577 testrpc.WaitForLeader(t, s1.RPC, "dc1") 578 client := rpcClient(t, s1) 579 defer client.Close() 580 581 // Create the ACL Write Policy 582 policyArg := structs.ACLPolicySetRequest{ 583 Datacenter: "dc1", 584 Policy: structs.ACLPolicy{ 585 Name: "token-replication-redacted", 586 Description: "token-replication-redacted", 587 Rules: `acl = "write"`, 588 }, 589 WriteRequest: structs.WriteRequest{Token: "root"}, 590 } 591 var policy structs.ACLPolicy 592 require.NoError(t, s1.RPC("ACL.PolicySet", &policyArg, &policy)) 593 594 // Create the dc2 replication token 595 tokenArg := structs.ACLTokenSetRequest{ 596 Datacenter: "dc1", 597 ACLToken: structs.ACLToken{ 598 Description: "dc2-replication", 599 Policies: []structs.ACLTokenPolicyLink{ 600 structs.ACLTokenPolicyLink{ 601 ID: policy.ID, 602 }, 603 }, 604 Local: false, 605 }, 606 WriteRequest: structs.WriteRequest{Token: "root"}, 607 } 608 609 var token structs.ACLToken 610 require.NoError(t, s1.RPC("ACL.TokenSet", &tokenArg, &token)) 611 612 dir2, s2 := testServerWithConfig(t, func(c *Config) { 613 c.Datacenter = "dc2" 614 c.ACLDatacenter = "dc1" 615 c.ACLsEnabled = true 616 c.ACLTokenReplication = true 617 c.ACLReplicationRate = 100 618 c.ACLReplicationBurst = 100 619 c.ACLReplicationApplyLimit = 1000000 620 }) 621 s2.tokens.UpdateReplicationToken(token.SecretID, tokenStore.TokenSourceConfig) 622 testrpc.WaitForLeader(t, s2.RPC, "dc2") 623 defer os.RemoveAll(dir2) 624 defer s2.Shutdown() 625 626 // Try to join. 627 joinWAN(t, s2, s1) 628 testrpc.WaitForLeader(t, s2.RPC, "dc2") 629 testrpc.WaitForLeader(t, s2.RPC, "dc1") 630 waitForNewACLs(t, s2) 631 632 // ensures replication is working ok 633 retry.Run(t, func(r *retry.R) { 634 var tokenResp structs.ACLTokenResponse 635 req := structs.ACLTokenGetRequest{ 636 Datacenter: "dc2", 637 TokenID: "root", 638 TokenIDType: structs.ACLTokenSecret, 639 QueryOptions: structs.QueryOptions{Token: "root"}, 640 } 641 err := s2.RPC("ACL.TokenRead", &req, &tokenResp) 642 require.NoError(r, err) 643 require.Equal(r, "root", tokenResp.Token.SecretID) 644 645 var status structs.ACLReplicationStatus 646 statusReq := structs.DCSpecificRequest{ 647 Datacenter: "dc2", 648 } 649 require.NoError(r, s2.RPC("ACL.ReplicationStatus", &statusReq, &status)) 650 // ensures that tokens are not being synced 651 require.True(r, status.ReplicatedTokenIndex > 0, "ReplicatedTokenIndex not greater than 0") 652 653 }) 654 655 // modify the replication policy to change to only granting read privileges 656 policyArg = structs.ACLPolicySetRequest{ 657 Datacenter: "dc1", 658 Policy: structs.ACLPolicy{ 659 ID: policy.ID, 660 Name: "token-replication-redacted", 661 Description: "token-replication-redacted", 662 Rules: `acl = "read"`, 663 }, 664 WriteRequest: structs.WriteRequest{Token: "root"}, 665 } 666 require.NoError(t, s1.RPC("ACL.PolicySet", &policyArg, &policy)) 667 668 // Create the another token so that replication will attempt to read it. 669 tokenArg = structs.ACLTokenSetRequest{ 670 Datacenter: "dc1", 671 ACLToken: structs.ACLToken{ 672 Description: "management", 673 Policies: []structs.ACLTokenPolicyLink{ 674 structs.ACLTokenPolicyLink{ 675 ID: structs.ACLPolicyGlobalManagementID, 676 }, 677 }, 678 Local: false, 679 }, 680 WriteRequest: structs.WriteRequest{Token: "root"}, 681 } 682 var token2 structs.ACLToken 683 684 // record the time right before we are touching the token 685 minErrorTime := time.Now() 686 require.NoError(t, s1.RPC("ACL.TokenSet", &tokenArg, &token2)) 687 688 retry.Run(t, func(r *retry.R) { 689 var tokenResp structs.ACLTokenResponse 690 req := structs.ACLTokenGetRequest{ 691 Datacenter: "dc2", 692 TokenID: redactedToken, 693 TokenIDType: structs.ACLTokenSecret, 694 QueryOptions: structs.QueryOptions{Token: redactedToken}, 695 } 696 err := s2.RPC("ACL.TokenRead", &req, &tokenResp) 697 // its not an error for the secret to not be found. 698 require.NoError(r, err) 699 require.Nil(r, tokenResp.Token) 700 701 var status structs.ACLReplicationStatus 702 statusReq := structs.DCSpecificRequest{ 703 Datacenter: "dc2", 704 } 705 require.NoError(r, s2.RPC("ACL.ReplicationStatus", &statusReq, &status)) 706 // ensures that tokens are not being synced 707 require.True(r, status.ReplicatedTokenIndex < token2.CreateIndex, "ReplicatedTokenIndex is not less than the token2s create index") 708 // ensures that token replication is erroring 709 require.True(r, status.LastError.After(minErrorTime), "Replication LastError not after the minErrorTime") 710 }) 711 }