github.com/argoproj/argo-cd/v3@v3.2.1/applicationset/generators/git_test.go (about) 1 package generators 2 3 import ( 4 "errors" 5 "testing" 6 7 "github.com/stretchr/testify/assert" 8 "github.com/stretchr/testify/mock" 9 "github.com/stretchr/testify/require" 10 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 "k8s.io/apimachinery/pkg/runtime" 12 "k8s.io/utils/ptr" 13 "sigs.k8s.io/controller-runtime/pkg/client/fake" 14 15 "github.com/argoproj/argo-cd/v3/applicationset/services/mocks" 16 "github.com/argoproj/argo-cd/v3/pkg/apis/application/v1alpha1" 17 ) 18 19 func Test_generateParamsFromGitFile(t *testing.T) { 20 defaultContent := []byte(` 21 foo: 22 bar: baz 23 `) 24 type args struct { 25 filePath string 26 fileContent []byte 27 values map[string]string 28 useGoTemplate bool 29 goTemplateOptions []string 30 pathParamPrefix string 31 } 32 tests := []struct { 33 name string 34 args args 35 want []map[string]any 36 wantErr bool 37 }{ 38 { 39 name: "empty file returns path parameters", 40 args: args{ 41 filePath: "path/dir/file_name.yaml", 42 fileContent: []byte(""), 43 values: map[string]string{}, 44 useGoTemplate: false, 45 }, 46 want: []map[string]any{ 47 { 48 "path": "path/dir", 49 "path.basename": "dir", 50 "path.filename": "file_name.yaml", 51 "path.basenameNormalized": "dir", 52 "path.filenameNormalized": "file-name.yaml", 53 "path[0]": "path", 54 "path[1]": "dir", 55 }, 56 }, 57 }, 58 { 59 name: "invalid json/yaml file returns error", 60 args: args{ 61 filePath: "path/dir/file_name.yaml", 62 fileContent: []byte("this is not json or yaml"), 63 values: map[string]string{}, 64 useGoTemplate: false, 65 }, 66 wantErr: true, 67 }, 68 { 69 name: "file parameters are added to params", 70 args: args{ 71 filePath: "path/dir/file_name.yaml", 72 fileContent: defaultContent, 73 values: map[string]string{}, 74 useGoTemplate: false, 75 }, 76 want: []map[string]any{ 77 { 78 "foo.bar": "baz", 79 "path": "path/dir", 80 "path.basename": "dir", 81 "path.filename": "file_name.yaml", 82 "path.basenameNormalized": "dir", 83 "path.filenameNormalized": "file-name.yaml", 84 "path[0]": "path", 85 "path[1]": "dir", 86 }, 87 }, 88 }, 89 { 90 name: "path parameter are prefixed", 91 args: args{ 92 filePath: "path/dir/file_name.yaml", 93 fileContent: defaultContent, 94 values: map[string]string{}, 95 useGoTemplate: false, 96 pathParamPrefix: "myRepo", 97 }, 98 want: []map[string]any{ 99 { 100 "foo.bar": "baz", 101 "myRepo.path": "path/dir", 102 "myRepo.path.basename": "dir", 103 "myRepo.path.filename": "file_name.yaml", 104 "myRepo.path.basenameNormalized": "dir", 105 "myRepo.path.filenameNormalized": "file-name.yaml", 106 "myRepo.path[0]": "path", 107 "myRepo.path[1]": "dir", 108 }, 109 }, 110 }, 111 { 112 name: "file parameters are added to params with go template", 113 args: args{ 114 filePath: "path/dir/file_name.yaml", 115 fileContent: defaultContent, 116 values: map[string]string{}, 117 useGoTemplate: true, 118 }, 119 want: []map[string]any{ 120 { 121 "foo": map[string]any{ 122 "bar": "baz", 123 }, 124 "path": map[string]any{ 125 "path": "path/dir", 126 "basename": "dir", 127 "filename": "file_name.yaml", 128 "basenameNormalized": "dir", 129 "filenameNormalized": "file-name.yaml", 130 "segments": []string{ 131 "path", 132 "dir", 133 }, 134 }, 135 }, 136 }, 137 }, 138 { 139 name: "path parameter are prefixed with go template", 140 args: args{ 141 filePath: "path/dir/file_name.yaml", 142 fileContent: defaultContent, 143 values: map[string]string{}, 144 useGoTemplate: true, 145 pathParamPrefix: "myRepo", 146 }, 147 want: []map[string]any{ 148 { 149 "foo": map[string]any{ 150 "bar": "baz", 151 }, 152 "myRepo": map[string]any{ 153 "path": map[string]any{ 154 "path": "path/dir", 155 "basename": "dir", 156 "filename": "file_name.yaml", 157 "basenameNormalized": "dir", 158 "filenameNormalized": "file-name.yaml", 159 "segments": []string{ 160 "path", 161 "dir", 162 }, 163 }, 164 }, 165 }, 166 }, 167 }, 168 } 169 for _, tt := range tests { 170 t.Run(tt.name, func(t *testing.T) { 171 params, err := (*GitGenerator)(nil).generateParamsFromGitFile(tt.args.filePath, tt.args.fileContent, tt.args.values, tt.args.useGoTemplate, tt.args.goTemplateOptions, tt.args.pathParamPrefix) 172 if tt.wantErr { 173 assert.Error(t, err, "GitGenerator.generateParamsFromGitFile()") 174 } else { 175 require.NoError(t, err) 176 assert.Equal(t, tt.want, params) 177 } 178 }) 179 } 180 } 181 182 func TestGitGenerateParamsFromDirectories(t *testing.T) { 183 t.Parallel() 184 185 cases := []struct { 186 name string 187 directories []v1alpha1.GitDirectoryGeneratorItem 188 pathParamPrefix string 189 repoApps []string 190 repoError error 191 values map[string]string 192 expected []map[string]any 193 expectedError error 194 }{ 195 { 196 name: "happy flow - created apps", 197 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 198 repoApps: []string{ 199 "app1", 200 "app2", 201 "app_3", 202 "p1/app4", 203 }, 204 expected: []map[string]any{ 205 {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1"}, 206 {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2", "path[0]": "app2"}, 207 {"path": "app_3", "path.basename": "app_3", "path.basenameNormalized": "app-3", "path[0]": "app_3"}, 208 }, 209 expectedError: nil, 210 }, 211 { 212 name: "It prefixes path parameters with PathParamPrefix", 213 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 214 pathParamPrefix: "myRepo", 215 repoApps: []string{ 216 "app1", 217 "app2", 218 "app_3", 219 "p1/app4", 220 }, 221 repoError: nil, 222 expected: []map[string]any{ 223 {"myRepo.path": "app1", "myRepo.path.basename": "app1", "myRepo.path.basenameNormalized": "app1", "myRepo.path[0]": "app1"}, 224 {"myRepo.path": "app2", "myRepo.path.basename": "app2", "myRepo.path.basenameNormalized": "app2", "myRepo.path[0]": "app2"}, 225 {"myRepo.path": "app_3", "myRepo.path.basename": "app_3", "myRepo.path.basenameNormalized": "app-3", "myRepo.path[0]": "app_3"}, 226 }, 227 expectedError: nil, 228 }, 229 { 230 name: "It filters application according to the paths", 231 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, 232 repoApps: []string{ 233 "app1", 234 "p1/app2", 235 "p1/p2/app3", 236 "p1/p2/p3/app4", 237 }, 238 expected: []map[string]any{ 239 {"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path[1]": "app2", "path.basenameNormalized": "app2"}, 240 {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path[2]": "app3", "path.basenameNormalized": "app3"}, 241 }, 242 expectedError: nil, 243 }, 244 { 245 name: "It filters application according to the paths with Exclude", 246 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, 247 repoApps: []string{ 248 "app1", 249 "app2", 250 "p1/app2", 251 "p1/app3", 252 "p2/app3", 253 }, 254 repoError: nil, 255 expected: []map[string]any{ 256 {"path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"}, 257 {"path": "app2", "path.basename": "app2", "path[0]": "app2", "path.basenameNormalized": "app2"}, 258 {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path[1]": "app3", "path.basenameNormalized": "app3"}, 259 }, 260 expectedError: nil, 261 }, 262 { 263 name: "Expecting same exclude behavior with different order", 264 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, 265 repoApps: []string{ 266 "app1", 267 "app2", 268 "p1/app2", 269 "p1/app3", 270 "p2/app3", 271 }, 272 repoError: nil, 273 expected: []map[string]any{ 274 {"path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"}, 275 {"path": "app2", "path.basename": "app2", "path[0]": "app2", "path.basenameNormalized": "app2"}, 276 {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path[1]": "app3", "path.basenameNormalized": "app3"}, 277 }, 278 expectedError: nil, 279 }, 280 { 281 name: "Value variable interpolation", 282 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}}, 283 repoApps: []string{ 284 "app1", 285 "p1/app2", 286 }, 287 repoError: nil, 288 values: map[string]string{ 289 "foo": "bar", 290 "aaa": "{{ path[0] }}", 291 "no-op": "{{ this-does-not-exist }}", 292 }, 293 expected: []map[string]any{ 294 {"values.foo": "bar", "values.no-op": "{{ this-does-not-exist }}", "values.aaa": "app1", "path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"}, 295 {"values.foo": "bar", "values.no-op": "{{ this-does-not-exist }}", "values.aaa": "p1", "path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path[1]": "app2", "path.basenameNormalized": "app2"}, 296 }, 297 expectedError: nil, 298 }, 299 { 300 name: "handles empty response from repo server", 301 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 302 repoApps: []string{}, 303 repoError: nil, 304 expected: []map[string]any{}, 305 expectedError: nil, 306 }, 307 { 308 name: "handles error from repo server", 309 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 310 repoApps: []string{}, 311 repoError: errors.New("error"), 312 expected: []map[string]any{}, 313 expectedError: errors.New("error generating params from git: error getting directories from repo: error"), 314 }, 315 } 316 317 for _, testCase := range cases { 318 testCaseCopy := testCase 319 320 t.Run(testCaseCopy.name, func(t *testing.T) { 321 t.Parallel() 322 323 argoCDServiceMock := mocks.Repos{} 324 325 argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError) 326 327 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 328 applicationSetInfo := v1alpha1.ApplicationSet{ 329 ObjectMeta: metav1.ObjectMeta{ 330 Name: "set", 331 }, 332 Spec: v1alpha1.ApplicationSetSpec{ 333 Generators: []v1alpha1.ApplicationSetGenerator{{ 334 Git: &v1alpha1.GitGenerator{ 335 RepoURL: "RepoURL", 336 Revision: "Revision", 337 Directories: testCaseCopy.directories, 338 PathParamPrefix: testCaseCopy.pathParamPrefix, 339 Values: testCaseCopy.values, 340 }, 341 }}, 342 }, 343 } 344 345 scheme := runtime.NewScheme() 346 err := v1alpha1.AddToScheme(scheme) 347 require.NoError(t, err) 348 appProject := v1alpha1.AppProject{} 349 350 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 351 352 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 353 354 if testCaseCopy.expectedError != nil { 355 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 356 } else { 357 require.NoError(t, err) 358 assert.Equal(t, testCaseCopy.expected, got) 359 } 360 361 argoCDServiceMock.AssertExpectations(t) 362 }) 363 } 364 } 365 366 func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) { 367 t.Parallel() 368 369 cases := []struct { 370 name string 371 directories []v1alpha1.GitDirectoryGeneratorItem 372 pathParamPrefix string 373 repoApps []string 374 repoError error 375 expected []map[string]any 376 expectedError error 377 }{ 378 { 379 name: "happy flow - created apps", 380 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 381 repoApps: []string{ 382 "app1", 383 "app2", 384 "app_3", 385 "p1/app4", 386 }, 387 repoError: nil, 388 expected: []map[string]any{ 389 { 390 "path": map[string]any{ 391 "path": "app1", 392 "basename": "app1", 393 "basenameNormalized": "app1", 394 "segments": []string{ 395 "app1", 396 }, 397 }, 398 }, 399 { 400 "path": map[string]any{ 401 "path": "app2", 402 "basename": "app2", 403 "basenameNormalized": "app2", 404 "segments": []string{ 405 "app2", 406 }, 407 }, 408 }, 409 { 410 "path": map[string]any{ 411 "path": "app_3", 412 "basename": "app_3", 413 "basenameNormalized": "app-3", 414 "segments": []string{ 415 "app_3", 416 }, 417 }, 418 }, 419 }, 420 expectedError: nil, 421 }, 422 { 423 name: "It prefixes path parameters with PathParamPrefix", 424 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 425 pathParamPrefix: "myRepo", 426 repoApps: []string{ 427 "app1", 428 "app2", 429 "app_3", 430 "p1/app4", 431 }, 432 repoError: nil, 433 expected: []map[string]any{ 434 { 435 "myRepo": map[string]any{ 436 "path": map[string]any{ 437 "path": "app1", 438 "basename": "app1", 439 "basenameNormalized": "app1", 440 "segments": []string{ 441 "app1", 442 }, 443 }, 444 }, 445 }, 446 { 447 "myRepo": map[string]any{ 448 "path": map[string]any{ 449 "path": "app2", 450 "basename": "app2", 451 "basenameNormalized": "app2", 452 "segments": []string{ 453 "app2", 454 }, 455 }, 456 }, 457 }, 458 { 459 "myRepo": map[string]any{ 460 "path": map[string]any{ 461 "path": "app_3", 462 "basename": "app_3", 463 "basenameNormalized": "app-3", 464 "segments": []string{ 465 "app_3", 466 }, 467 }, 468 }, 469 }, 470 }, 471 expectedError: nil, 472 }, 473 { 474 name: "It filters application according to the paths", 475 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, 476 repoApps: []string{ 477 "app1", 478 "p1/app2", 479 "p1/p2/app3", 480 "p1/p2/p3/app4", 481 }, 482 repoError: nil, 483 expected: []map[string]any{ 484 { 485 "path": map[string]any{ 486 "path": "p1/app2", 487 "basename": "app2", 488 "basenameNormalized": "app2", 489 "segments": []string{ 490 "p1", 491 "app2", 492 }, 493 }, 494 }, 495 { 496 "path": map[string]any{ 497 "path": "p1/p2/app3", 498 "basename": "app3", 499 "basenameNormalized": "app3", 500 "segments": []string{ 501 "p1", 502 "p2", 503 "app3", 504 }, 505 }, 506 }, 507 }, 508 expectedError: nil, 509 }, 510 { 511 name: "It filters application according to the paths with Exclude", 512 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, 513 repoApps: []string{ 514 "app1", 515 "app2", 516 "p1/app2", 517 "p1/app3", 518 "p2/app3", 519 }, 520 repoError: nil, 521 expected: []map[string]any{ 522 { 523 "path": map[string]any{ 524 "path": "app1", 525 "basename": "app1", 526 "basenameNormalized": "app1", 527 "segments": []string{ 528 "app1", 529 }, 530 }, 531 }, 532 { 533 "path": map[string]any{ 534 "path": "app2", 535 "basename": "app2", 536 "basenameNormalized": "app2", 537 "segments": []string{ 538 "app2", 539 }, 540 }, 541 }, 542 { 543 "path": map[string]any{ 544 "path": "p2/app3", 545 "basename": "app3", 546 "basenameNormalized": "app3", 547 "segments": []string{ 548 "p2", 549 "app3", 550 }, 551 }, 552 }, 553 }, 554 expectedError: nil, 555 }, 556 { 557 name: "Expecting same exclude behavior with different order", 558 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, 559 repoApps: []string{ 560 "app1", 561 "app2", 562 "p1/app2", 563 "p1/app3", 564 "p2/app3", 565 }, 566 repoError: nil, 567 expected: []map[string]any{ 568 { 569 "path": map[string]any{ 570 "path": "app1", 571 "basename": "app1", 572 "basenameNormalized": "app1", 573 "segments": []string{ 574 "app1", 575 }, 576 }, 577 }, 578 { 579 "path": map[string]any{ 580 "path": "app2", 581 "basename": "app2", 582 "basenameNormalized": "app2", 583 "segments": []string{ 584 "app2", 585 }, 586 }, 587 }, 588 { 589 "path": map[string]any{ 590 "path": "p2/app3", 591 "basename": "app3", 592 "basenameNormalized": "app3", 593 "segments": []string{ 594 "p2", 595 "app3", 596 }, 597 }, 598 }, 599 }, 600 expectedError: nil, 601 }, 602 { 603 name: "handles empty response from repo server", 604 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 605 repoApps: []string{}, 606 repoError: nil, 607 expected: []map[string]any{}, 608 expectedError: nil, 609 }, 610 { 611 name: "handles error from repo server", 612 directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 613 repoApps: []string{}, 614 repoError: errors.New("error"), 615 expected: []map[string]any{}, 616 expectedError: errors.New("error generating params from git: error getting directories from repo: error"), 617 }, 618 } 619 620 for _, testCase := range cases { 621 testCaseCopy := testCase 622 623 t.Run(testCaseCopy.name, func(t *testing.T) { 624 t.Parallel() 625 626 argoCDServiceMock := mocks.Repos{} 627 628 argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError) 629 630 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 631 applicationSetInfo := v1alpha1.ApplicationSet{ 632 ObjectMeta: metav1.ObjectMeta{ 633 Name: "set", 634 }, 635 Spec: v1alpha1.ApplicationSetSpec{ 636 GoTemplate: true, 637 Generators: []v1alpha1.ApplicationSetGenerator{{ 638 Git: &v1alpha1.GitGenerator{ 639 RepoURL: "RepoURL", 640 Revision: "Revision", 641 Directories: testCaseCopy.directories, 642 PathParamPrefix: testCaseCopy.pathParamPrefix, 643 }, 644 }}, 645 }, 646 } 647 648 scheme := runtime.NewScheme() 649 err := v1alpha1.AddToScheme(scheme) 650 require.NoError(t, err) 651 appProject := v1alpha1.AppProject{} 652 653 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 654 655 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 656 657 if testCaseCopy.expectedError != nil { 658 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 659 } else { 660 require.NoError(t, err) 661 assert.Equal(t, testCaseCopy.expected, got) 662 } 663 664 argoCDServiceMock.AssertExpectations(t) 665 }) 666 } 667 } 668 669 func TestGitGenerateParamsFromFiles(t *testing.T) { 670 t.Parallel() 671 672 cases := []struct { 673 name string 674 // files is the list of paths/globs to match 675 files []v1alpha1.GitFileGeneratorItem 676 // repoFileContents maps repo path to the literal contents of that path 677 repoFileContents map[string][]byte 678 // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value 679 repoPathsError error 680 values map[string]string 681 expected []map[string]any 682 expectedError error 683 }{ 684 { 685 name: "happy flow: create params from git files", 686 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 687 repoFileContents: map[string][]byte{ 688 "cluster-config/production/config.json": []byte(`{ 689 "cluster": { 690 "owner": "john.doe@example.com", 691 "name": "production", 692 "address": "https://kubernetes.default.svc" 693 }, 694 "key1": "val1", 695 "key2": { 696 "key2_1": "val2_1", 697 "key2_2": { 698 "key2_2_1": "val2_2_1" 699 } 700 }, 701 "key3": 123 702 }`), 703 "cluster-config/staging/config.json": []byte(`{ 704 "cluster": { 705 "owner": "foo.bar@example.com", 706 "name": "staging", 707 "address": "https://kubernetes.default.svc" 708 } 709 }`), 710 }, 711 repoPathsError: nil, 712 expected: []map[string]any{ 713 { 714 "cluster.owner": "john.doe@example.com", 715 "cluster.name": "production", 716 "cluster.address": "https://kubernetes.default.svc", 717 "key1": "val1", 718 "key2.key2_1": "val2_1", 719 "key2.key2_2.key2_2_1": "val2_2_1", 720 "key3": "123", 721 "path": "cluster-config/production", 722 "path.basename": "production", 723 "path[0]": "cluster-config", 724 "path[1]": "production", 725 "path.basenameNormalized": "production", 726 "path.filename": "config.json", 727 "path.filenameNormalized": "config.json", 728 }, 729 { 730 "cluster.owner": "foo.bar@example.com", 731 "cluster.name": "staging", 732 "cluster.address": "https://kubernetes.default.svc", 733 "path": "cluster-config/staging", 734 "path.basename": "staging", 735 "path[0]": "cluster-config", 736 "path[1]": "staging", 737 "path.basenameNormalized": "staging", 738 "path.filename": "config.json", 739 "path.filenameNormalized": "config.json", 740 }, 741 }, 742 expectedError: nil, 743 }, 744 { 745 name: "Value variable interpolation", 746 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 747 repoFileContents: map[string][]byte{ 748 "cluster-config/production/config.json": []byte(`{ 749 "cluster": { 750 "owner": "john.doe@example.com", 751 "name": "production", 752 "address": "https://kubernetes.default.svc" 753 }, 754 "key1": "val1", 755 "key2": { 756 "key2_1": "val2_1", 757 "key2_2": { 758 "key2_2_1": "val2_2_1" 759 } 760 }, 761 "key3": 123 762 }`), 763 "cluster-config/staging/config.json": []byte(`{ 764 "cluster": { 765 "owner": "foo.bar@example.com", 766 "name": "staging", 767 "address": "https://kubernetes.default.svc" 768 } 769 }`), 770 }, 771 repoPathsError: nil, 772 values: map[string]string{ 773 "aaa": "{{ cluster.owner }}", 774 "no-op": "{{ this-does-not-exist }}", 775 }, 776 expected: []map[string]any{ 777 { 778 "cluster.owner": "john.doe@example.com", 779 "cluster.name": "production", 780 "cluster.address": "https://kubernetes.default.svc", 781 "key1": "val1", 782 "key2.key2_1": "val2_1", 783 "key2.key2_2.key2_2_1": "val2_2_1", 784 "key3": "123", 785 "path": "cluster-config/production", 786 "path.basename": "production", 787 "path[0]": "cluster-config", 788 "path[1]": "production", 789 "path.basenameNormalized": "production", 790 "path.filename": "config.json", 791 "path.filenameNormalized": "config.json", 792 "values.aaa": "john.doe@example.com", 793 "values.no-op": "{{ this-does-not-exist }}", 794 }, 795 { 796 "cluster.owner": "foo.bar@example.com", 797 "cluster.name": "staging", 798 "cluster.address": "https://kubernetes.default.svc", 799 "path": "cluster-config/staging", 800 "path.basename": "staging", 801 "path[0]": "cluster-config", 802 "path[1]": "staging", 803 "path.basenameNormalized": "staging", 804 "path.filename": "config.json", 805 "path.filenameNormalized": "config.json", 806 "values.aaa": "foo.bar@example.com", 807 "values.no-op": "{{ this-does-not-exist }}", 808 }, 809 }, 810 expectedError: nil, 811 }, 812 { 813 name: "handles error during getting repo paths", 814 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 815 repoFileContents: map[string][]byte{}, 816 repoPathsError: errors.New("paths error"), 817 expected: []map[string]any{}, 818 expectedError: errors.New("error generating params from git: paths error"), 819 }, 820 { 821 name: "test invalid JSON file returns error", 822 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 823 repoFileContents: map[string][]byte{ 824 "cluster-config/production/config.json": []byte(`invalid json file`), 825 }, 826 repoPathsError: nil, 827 expected: []map[string]any{}, 828 expectedError: errors.New("error generating params from git: unable to process file 'cluster-config/production/config.json': unable to parse file: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type []map[string]interface {}"), 829 }, 830 { 831 name: "test JSON array", 832 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 833 repoFileContents: map[string][]byte{ 834 "cluster-config/production/config.json": []byte(` 835 [ 836 { 837 "cluster": { 838 "owner": "john.doe@example.com", 839 "name": "production", 840 "address": "https://kubernetes.default.svc", 841 "inner": { 842 "one" : "two" 843 } 844 } 845 }, 846 { 847 "cluster": { 848 "owner": "john.doe@example.com", 849 "name": "staging", 850 "address": "https://kubernetes.default.svc" 851 } 852 } 853 ]`), 854 }, 855 repoPathsError: nil, 856 expected: []map[string]any{ 857 { 858 "cluster.owner": "john.doe@example.com", 859 "cluster.name": "production", 860 "cluster.address": "https://kubernetes.default.svc", 861 "cluster.inner.one": "two", 862 "path": "cluster-config/production", 863 "path.basename": "production", 864 "path[0]": "cluster-config", 865 "path[1]": "production", 866 "path.basenameNormalized": "production", 867 "path.filename": "config.json", 868 "path.filenameNormalized": "config.json", 869 }, 870 { 871 "cluster.owner": "john.doe@example.com", 872 "cluster.name": "staging", 873 "cluster.address": "https://kubernetes.default.svc", 874 "path": "cluster-config/production", 875 "path.basename": "production", 876 "path[0]": "cluster-config", 877 "path[1]": "production", 878 "path.basenameNormalized": "production", 879 "path.filename": "config.json", 880 "path.filenameNormalized": "config.json", 881 }, 882 }, 883 expectedError: nil, 884 }, 885 { 886 name: "Test YAML flow", 887 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, 888 repoFileContents: map[string][]byte{ 889 "cluster-config/production/config.yaml": []byte(` 890 cluster: 891 owner: john.doe@example.com 892 name: production 893 address: https://kubernetes.default.svc 894 key1: val1 895 key2: 896 key2_1: val2_1 897 key2_2: 898 key2_2_1: val2_2_1 899 `), 900 "cluster-config/staging/config.yaml": []byte(` 901 cluster: 902 owner: foo.bar@example.com 903 name: staging 904 address: https://kubernetes.default.svc 905 `), 906 }, 907 repoPathsError: nil, 908 expected: []map[string]any{ 909 { 910 "cluster.owner": "john.doe@example.com", 911 "cluster.name": "production", 912 "cluster.address": "https://kubernetes.default.svc", 913 "key1": "val1", 914 "key2.key2_1": "val2_1", 915 "key2.key2_2.key2_2_1": "val2_2_1", 916 "path": "cluster-config/production", 917 "path.basename": "production", 918 "path[0]": "cluster-config", 919 "path[1]": "production", 920 "path.basenameNormalized": "production", 921 "path.filename": "config.yaml", 922 "path.filenameNormalized": "config.yaml", 923 }, 924 { 925 "cluster.owner": "foo.bar@example.com", 926 "cluster.name": "staging", 927 "cluster.address": "https://kubernetes.default.svc", 928 "path": "cluster-config/staging", 929 "path.basename": "staging", 930 "path[0]": "cluster-config", 931 "path[1]": "staging", 932 "path.basenameNormalized": "staging", 933 "path.filename": "config.yaml", 934 "path.filenameNormalized": "config.yaml", 935 }, 936 }, 937 expectedError: nil, 938 }, 939 { 940 name: "test YAML array", 941 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, 942 repoFileContents: map[string][]byte{ 943 "cluster-config/production/config.yaml": []byte(` 944 - cluster: 945 owner: john.doe@example.com 946 name: production 947 address: https://kubernetes.default.svc 948 inner: 949 one: two 950 - cluster: 951 owner: john.doe@example.com 952 name: staging 953 address: https://kubernetes.default.svc`), 954 }, 955 repoPathsError: nil, 956 expected: []map[string]any{ 957 { 958 "cluster.owner": "john.doe@example.com", 959 "cluster.name": "production", 960 "cluster.address": "https://kubernetes.default.svc", 961 "cluster.inner.one": "two", 962 "path": "cluster-config/production", 963 "path.basename": "production", 964 "path[0]": "cluster-config", 965 "path[1]": "production", 966 "path.basenameNormalized": "production", 967 "path.filename": "config.yaml", 968 "path.filenameNormalized": "config.yaml", 969 }, 970 { 971 "cluster.owner": "john.doe@example.com", 972 "cluster.name": "staging", 973 "cluster.address": "https://kubernetes.default.svc", 974 "path": "cluster-config/production", 975 "path.basename": "production", 976 "path[0]": "cluster-config", 977 "path[1]": "production", 978 "path.basenameNormalized": "production", 979 "path.filename": "config.yaml", 980 "path.filenameNormalized": "config.yaml", 981 }, 982 }, 983 expectedError: nil, 984 }, 985 { 986 name: "test empty YAML array", 987 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, 988 repoFileContents: map[string][]byte{ 989 "cluster-config/production/config.yaml": []byte(`[]`), 990 }, 991 repoPathsError: nil, 992 expected: []map[string]any{}, 993 expectedError: nil, 994 }, 995 } 996 997 for _, testCase := range cases { 998 testCaseCopy := testCase 999 1000 t.Run(testCaseCopy.name, func(t *testing.T) { 1001 t.Parallel() 1002 1003 argoCDServiceMock := mocks.Repos{} 1004 argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). 1005 Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError) 1006 1007 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 1008 applicationSetInfo := v1alpha1.ApplicationSet{ 1009 ObjectMeta: metav1.ObjectMeta{ 1010 Name: "set", 1011 }, 1012 Spec: v1alpha1.ApplicationSetSpec{ 1013 Generators: []v1alpha1.ApplicationSetGenerator{{ 1014 Git: &v1alpha1.GitGenerator{ 1015 RepoURL: "RepoURL", 1016 Revision: "Revision", 1017 Files: testCaseCopy.files, 1018 Values: testCaseCopy.values, 1019 }, 1020 }}, 1021 }, 1022 } 1023 1024 scheme := runtime.NewScheme() 1025 err := v1alpha1.AddToScheme(scheme) 1026 require.NoError(t, err) 1027 appProject := v1alpha1.AppProject{} 1028 1029 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 1030 1031 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 1032 1033 if testCaseCopy.expectedError != nil { 1034 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 1035 } else { 1036 require.NoError(t, err) 1037 assert.ElementsMatch(t, testCaseCopy.expected, got) 1038 } 1039 1040 argoCDServiceMock.AssertExpectations(t) 1041 }) 1042 } 1043 } 1044 1045 // TestGitGeneratorParamsFromFilesWithExcludeOptionWithNewGlobbing tests the params values generated by git file generator 1046 // when exclude option is set to true. It gives the result files based on new globbing pattern - doublestar package 1047 func TestGitGeneratorParamsFromFilesWithExcludeOptionWithNewGlobbing(t *testing.T) { 1048 t.Parallel() 1049 1050 cases := []struct { 1051 name string 1052 // files is the list of paths/globs to match 1053 files []v1alpha1.GitFileGeneratorItem 1054 // includePattern contains a list of file patterns that needs to be included 1055 includePattern []string 1056 // excludePattern contains a list of file patterns that needs to be excluded 1057 excludePattern []string 1058 // includeFiles is a map with key as absolute path to file and value as the content in bytes that satisfies the includePattern 1059 includeFiles map[string][]byte 1060 // excludeFiles is a map with key as absolute path to file and value as the content in bytes that satisfies the excludePattern 1061 // This means all the files should be excluded 1062 excludeFiles map[string][]byte 1063 // noMatchFiles contains all the files that neither match include pattern nor exclude pattern 1064 // Instead of keeping those files in the excludeFiles map, it is better to keep those files separately 1065 // in a separate field like 'noMatchFiles' to avoid confusion. 1066 noMatchFiles map[string][]byte 1067 // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value 1068 repoPathsError error 1069 values map[string]string 1070 expected []map[string]any 1071 expectedError error 1072 }{ 1073 { 1074 name: "filter files according to file-path with exclude", 1075 files: []v1alpha1.GitFileGeneratorItem{ 1076 { 1077 Path: "**/config.json", 1078 }, 1079 { 1080 Path: "p1/**/config.json", 1081 Exclude: true, 1082 }, 1083 }, 1084 includePattern: []string{"**/config.json"}, 1085 excludePattern: []string{"p1/**/config.json"}, 1086 includeFiles: map[string][]byte{ 1087 "cluster-config/production/config.json": []byte(`{ 1088 "cluster": { 1089 "owner": "john.doe@example.com", 1090 "name": "production", 1091 "address": "https://kubernetes.default.svc" 1092 }, 1093 "key1": "val1", 1094 "key2": { 1095 "key2_1": "val2_1", 1096 "key2_2": { 1097 "key2_2_1": "val2_2_1" 1098 } 1099 }, 1100 "key3": 123 1101 } 1102 `), 1103 "p1/config.json": []byte(`{ 1104 "database": { 1105 "admin": "db.admin@example.com", 1106 "name": "user-data", 1107 "host": "db.internal.local", 1108 "settings": { 1109 "replicas": 3, 1110 "backup": "daily" 1111 } 1112 } 1113 } 1114 `), 1115 "p1/p2/config.json": []byte(``), 1116 }, 1117 excludeFiles: map[string][]byte{ 1118 "p1/config.json": []byte(`{ 1119 "database": { 1120 "admin": "db.admin@example.com", 1121 "name": "user-data", 1122 "host": "db.internal.local", 1123 "settings": { 1124 "replicas": 3, 1125 "backup": "daily" 1126 } 1127 } 1128 } 1129 `), 1130 "p1/p2/config.json": []byte(``), 1131 }, 1132 repoPathsError: nil, 1133 expected: []map[string]any{ 1134 { 1135 "cluster.owner": "john.doe@example.com", 1136 "cluster.name": "production", 1137 "cluster.address": "https://kubernetes.default.svc", 1138 "key1": "val1", 1139 "key2.key2_1": "val2_1", 1140 "key2.key2_2.key2_2_1": "val2_2_1", 1141 "key3": "123", 1142 "path": "cluster-config/production", 1143 "path.basename": "production", 1144 "path[0]": "cluster-config", 1145 "path[1]": "production", 1146 "path.basenameNormalized": "production", 1147 "path.filename": "config.json", 1148 "path.filenameNormalized": "config.json", 1149 }, 1150 }, 1151 }, 1152 { 1153 name: "filter files according to multiple file-paths with exclude", 1154 files: []v1alpha1.GitFileGeneratorItem{ 1155 {Path: "**/config.json"}, 1156 {Path: "p1/app2/config.json", Exclude: true}, 1157 {Path: "p1/app3/config.json", Exclude: true}, 1158 }, 1159 includePattern: []string{"**/config.json"}, 1160 excludePattern: []string{"p1/app2/config.json", "p1/app3/config.json"}, 1161 includeFiles: map[string][]byte{ 1162 "p1/config.json": []byte(`{ 1163 "cluster": { 1164 "owner": "john.doe@example.com", 1165 "name": "production", 1166 "address": "https://kubernetes.default.svc", 1167 "inner": { 1168 "one" : "two" 1169 } 1170 } 1171 }`), 1172 "p1/app2/config.json": []byte(`{}`), 1173 "p1/app3/config.json": []byte(`{}`), 1174 }, 1175 excludeFiles: map[string][]byte{ 1176 "p1/app2/config.json": []byte(`{}`), 1177 "p1/app3/config.json": []byte(`{}`), 1178 }, 1179 repoPathsError: nil, 1180 expected: []map[string]any{ 1181 { 1182 "cluster.owner": "john.doe@example.com", 1183 "cluster.name": "production", 1184 "cluster.address": "https://kubernetes.default.svc", 1185 "cluster.inner.one": "two", 1186 "path": "p1", 1187 "path.basename": "p1", 1188 "path[0]": "p1", 1189 "path.basenameNormalized": "p1", 1190 "path.filename": "config.json", 1191 "path.filenameNormalized": "config.json", 1192 }, 1193 }, 1194 }, 1195 { 1196 name: "docs example test case to filter files according to multiple file-paths with exclude", 1197 files: []v1alpha1.GitFileGeneratorItem{{Path: "cluster-config/**/config.json"}, {Path: "cluster-config/*/dev/config.json", Exclude: true}}, 1198 includePattern: []string{"cluster-config/**/config.json"}, 1199 excludePattern: []string{"cluster-config/*/dev/config.json"}, 1200 includeFiles: map[string][]byte{ 1201 "cluster-config/engineering/prod/config.json": []byte(` 1202 cluster: 1203 owner: john.doe@example.com 1204 name: production 1205 address: https://kubernetes.default.svc 1206 `), 1207 "cluster-config/engineering/dev/config.json": []byte(` 1208 cluster: 1209 owner: foo.bar@example.com 1210 name: staging 1211 address: https://kubernetes.default.svc 1212 `), 1213 }, 1214 excludeFiles: map[string][]byte{ 1215 "cluster-config/engineering/dev/config.json": []byte(` 1216 cluster: 1217 owner: foo.bar@example.com 1218 name: staging 1219 address: https://kubernetes.default.svc 1220 `), 1221 }, 1222 repoPathsError: nil, 1223 expected: []map[string]any{ 1224 { 1225 "cluster.owner": "john.doe@example.com", 1226 "cluster.name": "production", 1227 "cluster.address": "https://kubernetes.default.svc", 1228 "path": "cluster-config/engineering/prod", 1229 "path.basename": "prod", 1230 "path[0]": "cluster-config", 1231 "path[1]": "engineering", 1232 "path[2]": "prod", 1233 "path.basenameNormalized": "prod", 1234 "path.filename": "config.json", 1235 "path.filenameNormalized": "config.json", 1236 }, 1237 }, 1238 }, 1239 { 1240 name: "testcase to verify new globbing pattern without any exclude", 1241 files: []v1alpha1.GitFileGeneratorItem{{Path: "some-path/*.yaml"}}, 1242 includePattern: []string{"some-path/*.yaml"}, 1243 excludePattern: nil, 1244 includeFiles: map[string][]byte{ 1245 "some-path/values.yaml": []byte(` 1246 cluster: 1247 owner: john.doe@example.com 1248 name: production 1249 address: https://kubernetes.default.svc 1250 `), 1251 }, 1252 excludeFiles: map[string][]byte{}, 1253 noMatchFiles: map[string][]byte{ 1254 "some-path/staging/values.yaml": []byte(` 1255 cluster: 1256 owner: foo.bar@example.com 1257 name: staging 1258 address: https://kubernetes.default.svc 1259 `), 1260 }, 1261 repoPathsError: nil, 1262 expected: []map[string]any{ 1263 { 1264 "cluster.owner": "john.doe@example.com", 1265 "cluster.name": "production", 1266 "cluster.address": "https://kubernetes.default.svc", 1267 "path": "some-path", 1268 "path.basename": "some-path", 1269 "path[0]": "some-path", 1270 "path.basenameNormalized": "some-path", 1271 "path.filename": "values.yaml", 1272 "path.filenameNormalized": "values.yaml", 1273 }, 1274 }, 1275 expectedError: nil, 1276 }, 1277 { 1278 name: "test to verify the solution for Git File Generator Problem", // https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/applicationset/Generators-Git-File-Globbing.md#git-file-generator-globbing 1279 files: []v1alpha1.GitFileGeneratorItem{{Path: "cluster-charts/*/*/values.yaml"}, {Path: "cluster-charts/*/values.yaml", Exclude: true}}, 1280 includePattern: []string{"cluster-charts/*/*/values.yaml"}, 1281 excludePattern: []string{"cluster-charts/*/values.yaml"}, 1282 includeFiles: map[string][]byte{ 1283 "cluster-charts/cluster1/mychart/values.yaml": []byte(` 1284 env: staging 1285 `), 1286 "cluster-charts/cluster1/myotherchart/values.yaml": []byte(` 1287 env: prod 1288 `), 1289 }, 1290 excludeFiles: map[string][]byte{ 1291 "cluster-charts/cluster2/values.yaml": []byte(` 1292 env: dev 1293 `), 1294 }, 1295 noMatchFiles: map[string][]byte{ 1296 "cluster-charts/cluster1/mychart/charts/mysubchart/values.yaml": []byte(` 1297 env: testing 1298 `), 1299 }, 1300 repoPathsError: nil, 1301 expected: []map[string]any{ 1302 { 1303 "env": "staging", 1304 "path": "cluster-charts/cluster1/mychart", 1305 "path.filenameNormalized": "values.yaml", 1306 "path[0]": "cluster-charts", 1307 "path[1]": "cluster1", 1308 "path[2]": "mychart", 1309 "path.basename": "mychart", 1310 "path.filename": "values.yaml", 1311 "path.basenameNormalized": "mychart", 1312 }, 1313 { 1314 "env": "prod", 1315 "path": "cluster-charts/cluster1/myotherchart", 1316 "path.filenameNormalized": "values.yaml", 1317 "path[0]": "cluster-charts", 1318 "path[1]": "cluster1", 1319 "path[2]": "myotherchart", 1320 "path.basename": "myotherchart", 1321 "path.filename": "values.yaml", 1322 "path.basenameNormalized": "myotherchart", 1323 }, 1324 }, 1325 expectedError: nil, 1326 }, 1327 } 1328 for _, testCase := range cases { 1329 testCaseCopy := testCase 1330 1331 t.Run(testCaseCopy.name, func(t *testing.T) { 1332 t.Parallel() 1333 1334 argoCDServiceMock := mocks.Repos{} 1335 1336 // IMPORTANT: we try to get the files from the repo server that matches the patterns 1337 // If we find those files also satisfy the exclude pattern, we remove them from map 1338 // This is generally done by the g.repos.GetFiles() function. 1339 // With the below mock setup, we make sure that if the GetFiles() function gets called 1340 // for a include or exclude pattern, it should always return the includeFiles or excludeFiles. 1341 for _, pattern := range testCaseCopy.excludePattern { 1342 argoCDServiceMock. 1343 On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, pattern, mock.Anything, mock.Anything). 1344 Return(testCaseCopy.excludeFiles, testCaseCopy.repoPathsError) 1345 } 1346 1347 for _, pattern := range testCaseCopy.includePattern { 1348 argoCDServiceMock. 1349 On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, pattern, mock.Anything, mock.Anything). 1350 Return(testCaseCopy.includeFiles, testCaseCopy.repoPathsError) 1351 } 1352 1353 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 1354 applicationSetInfo := v1alpha1.ApplicationSet{ 1355 ObjectMeta: metav1.ObjectMeta{ 1356 Name: "set", 1357 }, 1358 Spec: v1alpha1.ApplicationSetSpec{ 1359 Generators: []v1alpha1.ApplicationSetGenerator{{ 1360 Git: &v1alpha1.GitGenerator{ 1361 RepoURL: "RepoURL", 1362 Revision: "Revision", 1363 Files: testCaseCopy.files, 1364 Values: testCaseCopy.values, 1365 }, 1366 }}, 1367 }, 1368 } 1369 1370 scheme := runtime.NewScheme() 1371 err := v1alpha1.AddToScheme(scheme) 1372 require.NoError(t, err) 1373 appProject := v1alpha1.AppProject{} 1374 1375 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 1376 1377 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 1378 1379 if testCaseCopy.expectedError != nil { 1380 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 1381 } else { 1382 require.NoError(t, err) 1383 assert.ElementsMatch(t, testCaseCopy.expected, got) 1384 } 1385 1386 argoCDServiceMock.AssertExpectations(t) 1387 }) 1388 } 1389 } 1390 1391 // TestGitGeneratorParamsFromFilesWithExcludeOptionWithOldGlobbing tests the params values generated by git file generator 1392 // // when exclude option is set to true. It gives the result files based on old globbing pattern - git ls-files 1393 func TestGitGeneratorParamsFromFilesWithExcludeOptionWithOldGlobbing(t *testing.T) { 1394 t.Parallel() 1395 1396 cases := []struct { 1397 name string 1398 // files is the list of paths/globs to match 1399 files []v1alpha1.GitFileGeneratorItem 1400 // includePattern contains a list of file patterns that needs to be included 1401 includePattern []string 1402 // excludePattern contains a list of file patterns that needs to be excluded 1403 excludePattern []string 1404 // includeFiles is a map with key as absolute path to file and value as the content in bytes that satisfies the includePattern 1405 includeFiles map[string][]byte 1406 // excludeFiles is a map with key as absolute path to file and value as the content in bytes that satisfies the excludePattern 1407 // This means all the files should be excluded 1408 excludeFiles map[string][]byte 1409 // noMatchFiles contains all the files that neither match include pattern nor exclude pattern 1410 // Instead of keeping those files in the excludeFiles map, it is better to keep those files separately 1411 // in a separate field like 'noMatchFiles' to avoid confusion. 1412 noMatchFiles map[string][]byte 1413 // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value 1414 repoPathsError error 1415 values map[string]string 1416 expected []map[string]any 1417 expectedError error 1418 }{ 1419 { 1420 name: "filter files according to file-path with exclude", 1421 files: []v1alpha1.GitFileGeneratorItem{ 1422 { 1423 Path: "**/config.json", 1424 }, 1425 { 1426 Path: "p1/**/config.json", 1427 Exclude: true, 1428 }, 1429 }, 1430 includePattern: []string{"**/config.json"}, 1431 excludePattern: []string{"p1/**/config.json"}, 1432 includeFiles: map[string][]byte{ 1433 "cluster-config/production/config.json": []byte(`{ 1434 "cluster": { 1435 "owner": "john.doe@example.com", 1436 "name": "production", 1437 "address": "https://kubernetes.default.svc" 1438 }, 1439 "key1": "val1", 1440 "key2": { 1441 "key2_1": "val2_1", 1442 "key2_2": { 1443 "key2_2_1": "val2_2_1" 1444 } 1445 }, 1446 "key3": 123 1447 } 1448 `), 1449 "p1/config.json": []byte(`{ 1450 "database": { 1451 "admin": "db.admin@example.com", 1452 "name": "user-data", 1453 "host": "db.internal.local", 1454 "settings": { 1455 "replicas": 3, 1456 "backup": "daily" 1457 } 1458 } 1459 } 1460 `), 1461 "p1/p2/config.json": []byte(``), 1462 }, 1463 excludeFiles: map[string][]byte{ 1464 "p1/config.json": []byte(`{ 1465 "database": { 1466 "admin": "db.admin@example.com", 1467 "name": "user-data", 1468 "host": "db.internal.local", 1469 "settings": { 1470 "replicas": 3, 1471 "backup": "daily" 1472 } 1473 } 1474 } 1475 `), 1476 "p1/p2/config.json": []byte(``), 1477 }, 1478 repoPathsError: nil, 1479 expected: []map[string]any{ 1480 { 1481 "cluster.owner": "john.doe@example.com", 1482 "cluster.name": "production", 1483 "cluster.address": "https://kubernetes.default.svc", 1484 "key1": "val1", 1485 "key2.key2_1": "val2_1", 1486 "key2.key2_2.key2_2_1": "val2_2_1", 1487 "key3": "123", 1488 "path": "cluster-config/production", 1489 "path.basename": "production", 1490 "path[0]": "cluster-config", 1491 "path[1]": "production", 1492 "path.basenameNormalized": "production", 1493 "path.filename": "config.json", 1494 "path.filenameNormalized": "config.json", 1495 }, 1496 }, 1497 }, 1498 { 1499 name: "filter files according to multiple file-paths with exclude", 1500 files: []v1alpha1.GitFileGeneratorItem{ 1501 {Path: "**/config.json"}, 1502 {Path: "p1/app2/config.json", Exclude: true}, 1503 {Path: "p1/app3/config.json", Exclude: true}, 1504 }, 1505 includePattern: []string{"**/config.json"}, 1506 excludePattern: []string{"p1/app2/config.json", "p1/app3/config.json"}, 1507 includeFiles: map[string][]byte{ 1508 "p1/config.json": []byte(`{ 1509 "cluster": { 1510 "owner": "john.doe@example.com", 1511 "name": "production", 1512 "address": "https://kubernetes.default.svc", 1513 "inner": { 1514 "one" : "two" 1515 } 1516 } 1517 }`), 1518 "p1/app2/config.json": []byte(`{}`), 1519 "p1/app3/config.json": []byte(`{}`), 1520 }, 1521 excludeFiles: map[string][]byte{ 1522 "p1/app2/config.json": []byte(`{}`), 1523 "p1/app3/config.json": []byte(`{}`), 1524 }, 1525 repoPathsError: nil, 1526 expected: []map[string]any{ 1527 { 1528 "cluster.owner": "john.doe@example.com", 1529 "cluster.name": "production", 1530 "cluster.address": "https://kubernetes.default.svc", 1531 "cluster.inner.one": "two", 1532 "path": "p1", 1533 "path.basename": "p1", 1534 "path[0]": "p1", 1535 "path.basenameNormalized": "p1", 1536 "path.filename": "config.json", 1537 "path.filenameNormalized": "config.json", 1538 }, 1539 }, 1540 }, 1541 { 1542 name: "docs example test case to filter files according to multiple file-paths with exclude", 1543 files: []v1alpha1.GitFileGeneratorItem{{Path: "cluster-config/**/config.json"}, {Path: "cluster-config/*/dev/config.json", Exclude: true}}, 1544 includePattern: []string{"cluster-config/**/config.json"}, 1545 excludePattern: []string{"cluster-config/*/dev/config.json"}, 1546 includeFiles: map[string][]byte{ 1547 "cluster-config/engineering/prod/config.json": []byte(` 1548 cluster: 1549 owner: john.doe@example.com 1550 name: production 1551 address: https://kubernetes.default.svc 1552 `), 1553 "cluster-config/engineering/dev/config.json": []byte(` 1554 cluster: 1555 owner: foo.bar@example.com 1556 name: staging 1557 address: https://kubernetes.default.svc 1558 `), 1559 }, 1560 excludeFiles: map[string][]byte{ 1561 "cluster-config/engineering/dev/config.json": []byte(` 1562 cluster: 1563 owner: foo.bar@example.com 1564 name: staging 1565 address: https://kubernetes.default.svc 1566 `), 1567 }, 1568 repoPathsError: nil, 1569 expected: []map[string]any{ 1570 { 1571 "cluster.owner": "john.doe@example.com", 1572 "cluster.name": "production", 1573 "cluster.address": "https://kubernetes.default.svc", 1574 "path": "cluster-config/engineering/prod", 1575 "path.basename": "prod", 1576 "path[0]": "cluster-config", 1577 "path[1]": "engineering", 1578 "path[2]": "prod", 1579 "path.basenameNormalized": "prod", 1580 "path.filename": "config.json", 1581 "path.filenameNormalized": "config.json", 1582 }, 1583 }, 1584 }, 1585 { 1586 name: "testcase to verify new globbing pattern without any exclude", 1587 files: []v1alpha1.GitFileGeneratorItem{{Path: "some-path/*.yaml"}}, 1588 includePattern: []string{"some-path/*.yaml"}, 1589 excludePattern: nil, 1590 includeFiles: map[string][]byte{ 1591 "some-path/values.yaml": []byte(` 1592 cluster: 1593 owner: john.doe@example.com 1594 name: production 1595 address: https://kubernetes.default.svc 1596 `), 1597 "some-path/staging/values.yaml": []byte(` 1598 cluster: 1599 owner: foo.bar@example.com 1600 name: staging 1601 address: https://kubernetes.default.svc 1602 `), 1603 }, 1604 excludeFiles: map[string][]byte{}, 1605 repoPathsError: nil, 1606 expected: []map[string]any{ 1607 { 1608 "cluster.owner": "john.doe@example.com", 1609 "cluster.name": "production", 1610 "cluster.address": "https://kubernetes.default.svc", 1611 "path": "some-path", 1612 "path.basename": "some-path", 1613 "path[0]": "some-path", 1614 "path.basenameNormalized": "some-path", 1615 "path.filename": "values.yaml", 1616 "path.filenameNormalized": "values.yaml", 1617 }, 1618 { 1619 "cluster.owner": "foo.bar@example.com", 1620 "cluster.name": "staging", 1621 "cluster.address": "https://kubernetes.default.svc", 1622 "path": "some-path/staging", 1623 "path.basename": "staging", 1624 "path[0]": "some-path", 1625 "path[1]": "staging", 1626 "path.basenameNormalized": "staging", 1627 "path.filename": "values.yaml", 1628 "path.filenameNormalized": "values.yaml", 1629 }, 1630 }, 1631 expectedError: nil, 1632 }, 1633 { 1634 name: "test to verify the solution for Git File Generator Problem", 1635 files: []v1alpha1.GitFileGeneratorItem{{Path: "cluster-charts/*/*/values.yaml"}, {Path: "cluster-charts/*/values.yaml", Exclude: true}}, 1636 includePattern: []string{"cluster-charts/*/*/values.yaml"}, 1637 excludePattern: []string{"cluster-charts/*/values.yaml"}, 1638 includeFiles: map[string][]byte{ 1639 "cluster-charts/cluster1/mychart/values.yaml": []byte(` 1640 env: staging 1641 `), 1642 "cluster-charts/cluster1/myotherchart/values.yaml": []byte(` 1643 env: prod 1644 `), 1645 "cluster-charts/cluster1/mychart/charts/mysubchart/values.yaml": []byte(``), 1646 }, 1647 excludeFiles: map[string][]byte{ 1648 "cluster-charts/cluster2/values.yaml": []byte(` 1649 env: dev 1650 `), 1651 "cluster-charts/cluster1/mychart/values.yaml": []byte(` 1652 env: staging 1653 `), 1654 "cluster-charts/cluster1/myotherchart/values.yaml": []byte(` 1655 env: prod 1656 `), 1657 "cluster-charts/cluster1/mychart/charts/mysubchart/values.yaml": []byte(``), 1658 }, 1659 noMatchFiles: map[string][]byte{ 1660 "cluster-charts/cluster1/mychart/charts/mysubchart/values.yaml": []byte(` 1661 env: testing 1662 `), 1663 }, 1664 repoPathsError: nil, 1665 expected: []map[string]any{}, 1666 expectedError: nil, 1667 }, 1668 } 1669 for _, testCase := range cases { 1670 testCaseCopy := testCase 1671 1672 t.Run(testCaseCopy.name, func(t *testing.T) { 1673 t.Parallel() 1674 1675 argoCDServiceMock := mocks.Repos{} 1676 1677 // IMPORTANT: we try to get the files from the repo server that matches the patterns 1678 // If we find those files also satisfy the exclude pattern, we remove them from map 1679 // This is generally done by the g.repos.GetFiles() function. 1680 // With the below mock setup, we make sure that if the GetFiles() function gets called 1681 // for a include or exclude pattern, it should always return the includeFiles or excludeFiles. 1682 for _, pattern := range testCaseCopy.excludePattern { 1683 argoCDServiceMock. 1684 On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, pattern, mock.Anything, mock.Anything). 1685 Return(testCaseCopy.excludeFiles, testCaseCopy.repoPathsError) 1686 } 1687 1688 for _, pattern := range testCaseCopy.includePattern { 1689 argoCDServiceMock. 1690 On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, pattern, mock.Anything, mock.Anything). 1691 Return(testCaseCopy.includeFiles, testCaseCopy.repoPathsError) 1692 } 1693 1694 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 1695 applicationSetInfo := v1alpha1.ApplicationSet{ 1696 ObjectMeta: metav1.ObjectMeta{ 1697 Name: "set", 1698 }, 1699 Spec: v1alpha1.ApplicationSetSpec{ 1700 Generators: []v1alpha1.ApplicationSetGenerator{{ 1701 Git: &v1alpha1.GitGenerator{ 1702 RepoURL: "RepoURL", 1703 Revision: "Revision", 1704 Files: testCaseCopy.files, 1705 Values: testCaseCopy.values, 1706 }, 1707 }}, 1708 }, 1709 } 1710 1711 scheme := runtime.NewScheme() 1712 err := v1alpha1.AddToScheme(scheme) 1713 require.NoError(t, err) 1714 appProject := v1alpha1.AppProject{} 1715 1716 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 1717 1718 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 1719 1720 if testCaseCopy.expectedError != nil { 1721 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 1722 } else { 1723 require.NoError(t, err) 1724 assert.ElementsMatch(t, testCaseCopy.expected, got) 1725 } 1726 1727 argoCDServiceMock.AssertExpectations(t) 1728 }) 1729 } 1730 } 1731 1732 // TestGitGeneratorParamsFromFilesWithExcludeOption tests the params values generated by git file generator 1733 // when exclude option is set to true. It gives the result files based on new globbing pattern - doublestar package 1734 func TestGitGeneratorParamsFromFilesWithExcludeOptionGoTemplate(t *testing.T) { 1735 t.Parallel() 1736 1737 cases := []struct { 1738 name string 1739 // files is the list of paths/globs to match 1740 files []v1alpha1.GitFileGeneratorItem 1741 // includePattern contains a list of file patterns that needs to be included 1742 includePattern []string 1743 // excludePattern contains a list of file patterns that needs to be excluded 1744 excludePattern []string 1745 // includeFiles is a map with key as absolute path to file and value as the content in bytes that satisfies the includePattern 1746 includeFiles map[string][]byte 1747 // excludeFiles is a map with key as absolute path to file and value as the content in bytes that satisfies the excludePattern 1748 // This means all the files should be excluded 1749 excludeFiles map[string][]byte 1750 // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value 1751 repoPathsError error 1752 values map[string]string 1753 expected []map[string]any 1754 expectedError error 1755 }{ 1756 { 1757 name: "filter files according to file-path with exclude", 1758 files: []v1alpha1.GitFileGeneratorItem{ 1759 { 1760 Path: "**/config.json", 1761 }, 1762 { 1763 Path: "p1/**/config.json", 1764 Exclude: true, 1765 }, 1766 }, 1767 includePattern: []string{"**/config.json"}, 1768 excludePattern: []string{"p1/**/config.json"}, 1769 includeFiles: map[string][]byte{ 1770 "cluster-config/production/config.json": []byte(`{ 1771 "cluster": { 1772 "owner": "john.doe@example.com", 1773 "name": "production", 1774 "address": "https://kubernetes.default.svc" 1775 }, 1776 "key1": "val1", 1777 "key2": { 1778 "key2_1": "val2_1", 1779 "key2_2": { 1780 "key2_2_1": "val2_2_1" 1781 } 1782 }, 1783 "key3": 123 1784 } 1785 `), 1786 }, 1787 excludeFiles: map[string][]byte{ 1788 "p1/p2/config.json": []byte(`{ 1789 "service": { 1790 "maintainer": "dev.team@example.com", 1791 "serviceName": "auth-service", 1792 "endpoint": "http://auth.internal.svc", 1793 "config": { 1794 "retries": 5, 1795 "timeout": "30s" 1796 } 1797 } 1798 } 1799 `), 1800 }, 1801 repoPathsError: nil, 1802 expected: []map[string]any{ 1803 { 1804 "cluster": map[string]any{ 1805 "owner": "john.doe@example.com", 1806 "name": "production", 1807 "address": "https://kubernetes.default.svc", 1808 }, 1809 "key1": "val1", 1810 "key2": map[string]any{ 1811 "key2_1": "val2_1", 1812 "key2_2": map[string]any{ 1813 "key2_2_1": "val2_2_1", 1814 }, 1815 }, 1816 "key3": float64(123), 1817 "path": map[string]any{ 1818 "path": "cluster-config/production", 1819 "basename": "production", 1820 "filename": "config.json", 1821 "basenameNormalized": "production", 1822 "filenameNormalized": "config.json", 1823 "segments": []string{ 1824 "cluster-config", 1825 "production", 1826 }, 1827 }, 1828 }, 1829 }, 1830 expectedError: nil, 1831 }, 1832 { 1833 name: "filter files according to multiple file-paths with exclude", 1834 files: []v1alpha1.GitFileGeneratorItem{ 1835 {Path: "**/config.json"}, 1836 {Path: "p1/app2/config.json", Exclude: true}, 1837 {Path: "p1/app3/config.json", Exclude: true}, 1838 }, 1839 includePattern: []string{"**/config.json"}, 1840 excludePattern: []string{"p1/app2/config.json", "p1/app3/config.json"}, 1841 includeFiles: map[string][]byte{ 1842 "p1/config.json": []byte(`{ 1843 "cluster": { 1844 "owner": "john.doe@example.com", 1845 "name": "production", 1846 "address": "https://kubernetes.default.svc", 1847 "inner": { 1848 "one" : "two" 1849 } 1850 } 1851 }`), 1852 }, 1853 excludeFiles: map[string][]byte{ 1854 "p1/app2/config.json": []byte(`{ 1855 "database": { 1856 "admin": "alice.smith@example.com", 1857 "env": "staging", 1858 "url": "postgres://db.internal.svc:5432", 1859 "settings": { 1860 "replicas": 3, 1861 "backup": "enabled" 1862 } 1863 } 1864 } 1865 `), 1866 "p1/app3/config.json": []byte(`{ 1867 "storage": { 1868 "owner": "charlie.brown@example.com", 1869 "bucketName": "app-assets", 1870 "region": "us-west-2", 1871 "options": { 1872 "versioning": true, 1873 "encryption": "AES256" 1874 } 1875 } 1876 } 1877 `), 1878 }, 1879 repoPathsError: nil, 1880 expected: []map[string]any{ 1881 { 1882 "cluster": map[string]any{ 1883 "owner": "john.doe@example.com", 1884 "name": "production", 1885 "address": "https://kubernetes.default.svc", 1886 "inner": map[string]any{ 1887 "one": "two", 1888 }, 1889 }, 1890 "path": map[string]any{ 1891 "path": "p1", 1892 "basename": "p1", 1893 "filename": "config.json", 1894 "basenameNormalized": "p1", 1895 "filenameNormalized": "config.json", 1896 "segments": []string{ 1897 "p1", 1898 }, 1899 }, 1900 }, 1901 }, 1902 expectedError: nil, 1903 }, 1904 } 1905 for _, testCase := range cases { 1906 testCaseCopy := testCase 1907 1908 t.Run(testCaseCopy.name, func(t *testing.T) { 1909 t.Parallel() 1910 1911 argoCDServiceMock := mocks.Repos{} 1912 // IMPORTANT: we try to get the files from the repo server that matches the patterns 1913 // If we find those files also satisfy the exclude pattern, we remove them from map 1914 // This is generally done by the g.repos.GetFiles() function. 1915 // With the below mock setup, we make sure that if the GetFiles() function gets called 1916 // for a include or exclude pattern, it should always return the includeFiles or excludeFiles. 1917 for _, pattern := range testCaseCopy.excludePattern { 1918 argoCDServiceMock. 1919 On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, pattern, mock.Anything, mock.Anything). 1920 Return(testCaseCopy.excludeFiles, testCaseCopy.repoPathsError) 1921 } 1922 1923 for _, pattern := range testCaseCopy.includePattern { 1924 argoCDServiceMock. 1925 On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, pattern, mock.Anything, mock.Anything). 1926 Return(testCaseCopy.includeFiles, testCaseCopy.repoPathsError) 1927 } 1928 1929 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 1930 applicationSetInfo := v1alpha1.ApplicationSet{ 1931 ObjectMeta: metav1.ObjectMeta{ 1932 Name: "set", 1933 }, 1934 Spec: v1alpha1.ApplicationSetSpec{ 1935 GoTemplate: true, 1936 Generators: []v1alpha1.ApplicationSetGenerator{{ 1937 Git: &v1alpha1.GitGenerator{ 1938 RepoURL: "RepoURL", 1939 Revision: "Revision", 1940 Files: testCaseCopy.files, 1941 }, 1942 }}, 1943 }, 1944 } 1945 1946 scheme := runtime.NewScheme() 1947 err := v1alpha1.AddToScheme(scheme) 1948 require.NoError(t, err) 1949 appProject := v1alpha1.AppProject{} 1950 1951 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 1952 1953 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 1954 1955 if testCaseCopy.expectedError != nil { 1956 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 1957 } else { 1958 require.NoError(t, err) 1959 assert.ElementsMatch(t, testCaseCopy.expected, got) 1960 } 1961 1962 argoCDServiceMock.AssertExpectations(t) 1963 }) 1964 } 1965 } 1966 1967 func TestGitGenerateParamsFromFilesGoTemplate(t *testing.T) { 1968 t.Parallel() 1969 1970 cases := []struct { 1971 name string 1972 // files is the list of paths/globs to match 1973 files []v1alpha1.GitFileGeneratorItem 1974 // repoFileContents maps repo path to the literal contents of that path 1975 repoFileContents map[string][]byte 1976 // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value 1977 repoPathsError error 1978 expected []map[string]any 1979 expectedError error 1980 }{ 1981 { 1982 name: "happy flow: create params from git files", 1983 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 1984 repoFileContents: map[string][]byte{ 1985 "cluster-config/production/config.json": []byte(`{ 1986 "cluster": { 1987 "owner": "john.doe@example.com", 1988 "name": "production", 1989 "address": "https://kubernetes.default.svc" 1990 }, 1991 "key1": "val1", 1992 "key2": { 1993 "key2_1": "val2_1", 1994 "key2_2": { 1995 "key2_2_1": "val2_2_1" 1996 } 1997 }, 1998 "key3": 123 1999 }`), 2000 "cluster-config/staging/config.json": []byte(`{ 2001 "cluster": { 2002 "owner": "foo.bar@example.com", 2003 "name": "staging", 2004 "address": "https://kubernetes.default.svc" 2005 } 2006 }`), 2007 }, 2008 repoPathsError: nil, 2009 expected: []map[string]any{ 2010 { 2011 "cluster": map[string]any{ 2012 "owner": "john.doe@example.com", 2013 "name": "production", 2014 "address": "https://kubernetes.default.svc", 2015 }, 2016 "key1": "val1", 2017 "key2": map[string]any{ 2018 "key2_1": "val2_1", 2019 "key2_2": map[string]any{ 2020 "key2_2_1": "val2_2_1", 2021 }, 2022 }, 2023 "key3": float64(123), 2024 "path": map[string]any{ 2025 "path": "cluster-config/production", 2026 "basename": "production", 2027 "filename": "config.json", 2028 "basenameNormalized": "production", 2029 "filenameNormalized": "config.json", 2030 "segments": []string{ 2031 "cluster-config", 2032 "production", 2033 }, 2034 }, 2035 }, 2036 { 2037 "cluster": map[string]any{ 2038 "owner": "foo.bar@example.com", 2039 "name": "staging", 2040 "address": "https://kubernetes.default.svc", 2041 }, 2042 "path": map[string]any{ 2043 "path": "cluster-config/staging", 2044 "basename": "staging", 2045 "filename": "config.json", 2046 "basenameNormalized": "staging", 2047 "filenameNormalized": "config.json", 2048 "segments": []string{ 2049 "cluster-config", 2050 "staging", 2051 }, 2052 }, 2053 }, 2054 }, 2055 expectedError: nil, 2056 }, 2057 { 2058 name: "handles error during getting repo paths", 2059 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 2060 repoFileContents: map[string][]byte{}, 2061 repoPathsError: errors.New("paths error"), 2062 expected: []map[string]any{}, 2063 expectedError: errors.New("error generating params from git: paths error"), 2064 }, 2065 { 2066 name: "test invalid JSON file returns error", 2067 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 2068 repoFileContents: map[string][]byte{ 2069 "cluster-config/production/config.json": []byte(`invalid json file`), 2070 }, 2071 repoPathsError: nil, 2072 expected: []map[string]any{}, 2073 expectedError: errors.New("error generating params from git: unable to process file 'cluster-config/production/config.json': unable to parse file: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type []map[string]interface {}"), 2074 }, 2075 { 2076 name: "test JSON array", 2077 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, 2078 repoFileContents: map[string][]byte{ 2079 "cluster-config/production/config.json": []byte(` 2080 [ 2081 { 2082 "cluster": { 2083 "owner": "john.doe@example.com", 2084 "name": "production", 2085 "address": "https://kubernetes.default.svc", 2086 "inner": { 2087 "one" : "two" 2088 } 2089 } 2090 }, 2091 { 2092 "cluster": { 2093 "owner": "john.doe@example.com", 2094 "name": "staging", 2095 "address": "https://kubernetes.default.svc" 2096 } 2097 } 2098 ]`), 2099 }, 2100 repoPathsError: nil, 2101 expected: []map[string]any{ 2102 { 2103 "cluster": map[string]any{ 2104 "owner": "john.doe@example.com", 2105 "name": "production", 2106 "address": "https://kubernetes.default.svc", 2107 "inner": map[string]any{ 2108 "one": "two", 2109 }, 2110 }, 2111 "path": map[string]any{ 2112 "path": "cluster-config/production", 2113 "basename": "production", 2114 "filename": "config.json", 2115 "basenameNormalized": "production", 2116 "filenameNormalized": "config.json", 2117 "segments": []string{ 2118 "cluster-config", 2119 "production", 2120 }, 2121 }, 2122 }, 2123 { 2124 "cluster": map[string]any{ 2125 "owner": "john.doe@example.com", 2126 "name": "staging", 2127 "address": "https://kubernetes.default.svc", 2128 }, 2129 "path": map[string]any{ 2130 "path": "cluster-config/production", 2131 "basename": "production", 2132 "filename": "config.json", 2133 "basenameNormalized": "production", 2134 "filenameNormalized": "config.json", 2135 "segments": []string{ 2136 "cluster-config", 2137 "production", 2138 }, 2139 }, 2140 }, 2141 }, 2142 expectedError: nil, 2143 }, 2144 { 2145 name: "Test YAML flow", 2146 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, 2147 repoFileContents: map[string][]byte{ 2148 "cluster-config/production/config.yaml": []byte(` 2149 cluster: 2150 owner: john.doe@example.com 2151 name: production 2152 address: https://kubernetes.default.svc 2153 key1: val1 2154 key2: 2155 key2_1: val2_1 2156 key2_2: 2157 key2_2_1: val2_2_1 2158 `), 2159 "cluster-config/staging/config.yaml": []byte(` 2160 cluster: 2161 owner: foo.bar@example.com 2162 name: staging 2163 address: https://kubernetes.default.svc 2164 `), 2165 }, 2166 repoPathsError: nil, 2167 expected: []map[string]any{ 2168 { 2169 "cluster": map[string]any{ 2170 "owner": "john.doe@example.com", 2171 "name": "production", 2172 "address": "https://kubernetes.default.svc", 2173 }, 2174 "key1": "val1", 2175 "key2": map[string]any{ 2176 "key2_1": "val2_1", 2177 "key2_2": map[string]any{ 2178 "key2_2_1": "val2_2_1", 2179 }, 2180 }, 2181 "path": map[string]any{ 2182 "path": "cluster-config/production", 2183 "basename": "production", 2184 "filename": "config.yaml", 2185 "basenameNormalized": "production", 2186 "filenameNormalized": "config.yaml", 2187 "segments": []string{ 2188 "cluster-config", 2189 "production", 2190 }, 2191 }, 2192 }, 2193 { 2194 "cluster": map[string]any{ 2195 "owner": "foo.bar@example.com", 2196 "name": "staging", 2197 "address": "https://kubernetes.default.svc", 2198 }, 2199 "path": map[string]any{ 2200 "path": "cluster-config/staging", 2201 "basename": "staging", 2202 "filename": "config.yaml", 2203 "basenameNormalized": "staging", 2204 "filenameNormalized": "config.yaml", 2205 "segments": []string{ 2206 "cluster-config", 2207 "staging", 2208 }, 2209 }, 2210 }, 2211 }, 2212 expectedError: nil, 2213 }, 2214 { 2215 name: "test YAML array", 2216 files: []v1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, 2217 repoFileContents: map[string][]byte{ 2218 "cluster-config/production/config.yaml": []byte(` 2219 - cluster: 2220 owner: john.doe@example.com 2221 name: production 2222 address: https://kubernetes.default.svc 2223 inner: 2224 one: two 2225 - cluster: 2226 owner: john.doe@example.com 2227 name: staging 2228 address: https://kubernetes.default.svc`), 2229 }, 2230 repoPathsError: nil, 2231 expected: []map[string]any{ 2232 { 2233 "cluster": map[string]any{ 2234 "owner": "john.doe@example.com", 2235 "name": "production", 2236 "address": "https://kubernetes.default.svc", 2237 "inner": map[string]any{ 2238 "one": "two", 2239 }, 2240 }, 2241 "path": map[string]any{ 2242 "path": "cluster-config/production", 2243 "basename": "production", 2244 "filename": "config.yaml", 2245 "basenameNormalized": "production", 2246 "filenameNormalized": "config.yaml", 2247 "segments": []string{ 2248 "cluster-config", 2249 "production", 2250 }, 2251 }, 2252 }, 2253 { 2254 "cluster": map[string]any{ 2255 "owner": "john.doe@example.com", 2256 "name": "staging", 2257 "address": "https://kubernetes.default.svc", 2258 }, 2259 "path": map[string]any{ 2260 "path": "cluster-config/production", 2261 "basename": "production", 2262 "filename": "config.yaml", 2263 "basenameNormalized": "production", 2264 "filenameNormalized": "config.yaml", 2265 "segments": []string{ 2266 "cluster-config", 2267 "production", 2268 }, 2269 }, 2270 }, 2271 }, 2272 expectedError: nil, 2273 }, 2274 } 2275 2276 for _, testCase := range cases { 2277 testCaseCopy := testCase 2278 2279 t.Run(testCaseCopy.name, func(t *testing.T) { 2280 t.Parallel() 2281 2282 argoCDServiceMock := mocks.Repos{} 2283 argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). 2284 Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError) 2285 2286 gitGenerator := NewGitGenerator(&argoCDServiceMock, "") 2287 applicationSetInfo := v1alpha1.ApplicationSet{ 2288 ObjectMeta: metav1.ObjectMeta{ 2289 Name: "set", 2290 }, 2291 Spec: v1alpha1.ApplicationSetSpec{ 2292 GoTemplate: true, 2293 Generators: []v1alpha1.ApplicationSetGenerator{{ 2294 Git: &v1alpha1.GitGenerator{ 2295 RepoURL: "RepoURL", 2296 Revision: "Revision", 2297 Files: testCaseCopy.files, 2298 }, 2299 }}, 2300 }, 2301 } 2302 2303 scheme := runtime.NewScheme() 2304 err := v1alpha1.AddToScheme(scheme) 2305 require.NoError(t, err) 2306 appProject := v1alpha1.AppProject{} 2307 2308 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build() 2309 2310 got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo, client) 2311 2312 if testCaseCopy.expectedError != nil { 2313 require.EqualError(t, err, testCaseCopy.expectedError.Error()) 2314 } else { 2315 require.NoError(t, err) 2316 assert.ElementsMatch(t, testCaseCopy.expected, got) 2317 } 2318 2319 argoCDServiceMock.AssertExpectations(t) 2320 }) 2321 } 2322 } 2323 2324 func TestGitGenerator_GenerateParams(t *testing.T) { 2325 cases := []struct { 2326 name string 2327 appProject v1alpha1.AppProject 2328 directories []v1alpha1.GitDirectoryGeneratorItem 2329 pathParamPrefix string 2330 repoApps []string 2331 repoPathsError error 2332 repoFileContents map[string][]byte 2333 values map[string]string 2334 expected []map[string]any 2335 expectedError error 2336 expectedProject *string 2337 appset v1alpha1.ApplicationSet 2338 callGetDirectories bool 2339 }{ 2340 { 2341 name: "Signature Verification - ignores templated project field", 2342 repoApps: []string{ 2343 "app1", 2344 }, 2345 repoPathsError: nil, 2346 appset: v1alpha1.ApplicationSet{ 2347 ObjectMeta: metav1.ObjectMeta{ 2348 Name: "set", 2349 Namespace: "namespace", 2350 }, 2351 Spec: v1alpha1.ApplicationSetSpec{ 2352 Generators: []v1alpha1.ApplicationSetGenerator{{ 2353 Git: &v1alpha1.GitGenerator{ 2354 RepoURL: "RepoURL", 2355 Revision: "Revision", 2356 Directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 2357 PathParamPrefix: "", 2358 Values: map[string]string{ 2359 "foo": "bar", 2360 }, 2361 }, 2362 }}, 2363 Template: v1alpha1.ApplicationSetTemplate{ 2364 Spec: v1alpha1.ApplicationSpec{ 2365 Project: "{{.project}}", 2366 }, 2367 }, 2368 }, 2369 }, 2370 callGetDirectories: true, 2371 expected: []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}}, 2372 expectedError: nil, 2373 }, 2374 { 2375 name: "Signature Verification - Checks for non-templated project field", 2376 repoApps: []string{ 2377 "app1", 2378 }, 2379 repoPathsError: nil, 2380 appset: v1alpha1.ApplicationSet{ 2381 ObjectMeta: metav1.ObjectMeta{ 2382 Name: "set", 2383 Namespace: "namespace", 2384 }, 2385 Spec: v1alpha1.ApplicationSetSpec{ 2386 Generators: []v1alpha1.ApplicationSetGenerator{{ 2387 Git: &v1alpha1.GitGenerator{ 2388 RepoURL: "RepoURL", 2389 Revision: "Revision", 2390 Directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 2391 PathParamPrefix: "", 2392 Values: map[string]string{ 2393 "foo": "bar", 2394 }, 2395 }, 2396 }}, 2397 Template: v1alpha1.ApplicationSetTemplate{ 2398 Spec: v1alpha1.ApplicationSpec{ 2399 Project: "project", 2400 }, 2401 }, 2402 }, 2403 }, 2404 callGetDirectories: false, 2405 expected: []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}}, 2406 expectedError: errors.New("error getting project project: appprojects.argoproj.io \"project\" not found"), 2407 }, 2408 { 2409 name: "Project field is not templated - verify that project is passed through to repo-server as-is", 2410 repoApps: []string{ 2411 "app1", 2412 }, 2413 callGetDirectories: true, 2414 appProject: v1alpha1.AppProject{ 2415 TypeMeta: metav1.TypeMeta{}, 2416 ObjectMeta: metav1.ObjectMeta{ 2417 Name: "project", 2418 Namespace: "argocd", 2419 }, 2420 }, 2421 appset: v1alpha1.ApplicationSet{ 2422 ObjectMeta: metav1.ObjectMeta{ 2423 Name: "set", 2424 Namespace: "namespace", 2425 }, 2426 Spec: v1alpha1.ApplicationSetSpec{ 2427 Generators: []v1alpha1.ApplicationSetGenerator{{ 2428 Git: &v1alpha1.GitGenerator{ 2429 RepoURL: "RepoURL", 2430 Revision: "Revision", 2431 Directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 2432 PathParamPrefix: "", 2433 Values: map[string]string{ 2434 "foo": "bar", 2435 }, 2436 }, 2437 }}, 2438 Template: v1alpha1.ApplicationSetTemplate{ 2439 Spec: v1alpha1.ApplicationSpec{ 2440 Project: "project", 2441 }, 2442 }, 2443 }, 2444 }, 2445 expected: []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}}, 2446 expectedProject: ptr.To("project"), 2447 expectedError: nil, 2448 }, 2449 { 2450 name: "Project field is templated - verify that project is passed through to repo-server as empty string", 2451 repoApps: []string{ 2452 "app1", 2453 }, 2454 callGetDirectories: true, 2455 appset: v1alpha1.ApplicationSet{ 2456 ObjectMeta: metav1.ObjectMeta{ 2457 Name: "set", 2458 Namespace: "namespace", 2459 }, 2460 Spec: v1alpha1.ApplicationSetSpec{ 2461 Generators: []v1alpha1.ApplicationSetGenerator{{ 2462 Git: &v1alpha1.GitGenerator{ 2463 RepoURL: "RepoURL", 2464 Revision: "Revision", 2465 Directories: []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, 2466 PathParamPrefix: "", 2467 Values: map[string]string{ 2468 "foo": "bar", 2469 }, 2470 }, 2471 }}, 2472 Template: v1alpha1.ApplicationSetTemplate{ 2473 Spec: v1alpha1.ApplicationSpec{ 2474 Project: "{{.project}}", 2475 }, 2476 }, 2477 }, 2478 }, 2479 expected: []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}}, 2480 expectedProject: ptr.To(""), 2481 expectedError: nil, 2482 }, 2483 } 2484 for _, testCase := range cases { 2485 argoCDServiceMock := mocks.Repos{} 2486 2487 if testCase.callGetDirectories { 2488 var project any 2489 if testCase.expectedProject != nil { 2490 project = *testCase.expectedProject 2491 } else { 2492 project = mock.Anything 2493 } 2494 2495 argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, project, mock.Anything, mock.Anything).Return(testCase.repoApps, testCase.repoPathsError) 2496 } 2497 gitGenerator := NewGitGenerator(&argoCDServiceMock, "argocd") 2498 2499 scheme := runtime.NewScheme() 2500 err := v1alpha1.AddToScheme(scheme) 2501 require.NoError(t, err) 2502 2503 client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&testCase.appProject).Build() 2504 2505 got, err := gitGenerator.GenerateParams(&testCase.appset.Spec.Generators[0], &testCase.appset, client) 2506 2507 if testCase.expectedError != nil { 2508 require.EqualError(t, err, testCase.expectedError.Error()) 2509 } else { 2510 require.NoError(t, err) 2511 assert.Equal(t, testCase.expected, got) 2512 } 2513 2514 argoCDServiceMock.AssertExpectations(t) 2515 } 2516 }