github.com/vishnusomank/figtree@v0.1.0/figtree.go (about) 1 package figtree 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io" 8 "os" 9 "os/exec" 10 "path" 11 "path/filepath" 12 "reflect" 13 "regexp" 14 "sort" 15 "strconv" 16 "strings" 17 "unicode" 18 19 "emperror.dev/errors" 20 "github.com/coryb/walky" 21 "github.com/fatih/camelcase" 22 "gopkg.in/yaml.v3" 23 ) 24 25 type Logger interface { 26 Debugf(format string, args ...interface{}) 27 } 28 29 type nullLogger struct{} 30 31 func (*nullLogger) Debugf(string, ...interface{}) {} 32 33 var Log Logger = &nullLogger{} 34 35 func defaultApplyChangeSet(changeSet map[string]*string) error { 36 for k, v := range changeSet { 37 if v != nil { 38 os.Setenv(k, *v) 39 } else { 40 os.Unsetenv(k) 41 } 42 } 43 return nil 44 } 45 46 type CreateOption func(*FigTree) 47 48 func WithHome(home string) CreateOption { 49 return func(f *FigTree) { 50 f.home = home 51 } 52 } 53 54 func WithCwd(cwd string) CreateOption { 55 return func(f *FigTree) { 56 f.workDir = cwd 57 } 58 } 59 60 func WithEnvPrefix(env string) CreateOption { 61 return func(f *FigTree) { 62 f.envPrefix = env 63 } 64 } 65 66 func WithConfigDir(dir string) CreateOption { 67 return func(f *FigTree) { 68 f.configDir = dir 69 } 70 } 71 72 type ChangeSetFunc func(map[string]*string) error 73 74 func WithApplyChangeSet(apply ChangeSetFunc) CreateOption { 75 return func(f *FigTree) { 76 f.applyChangeSet = apply 77 } 78 } 79 80 type PreProcessor func(*yaml.Node) error 81 82 func WithPreProcessor(pp PreProcessor) CreateOption { 83 return func(f *FigTree) { 84 f.preProcessor = pp 85 } 86 } 87 88 type FilterOut func(*yaml.Node) bool 89 90 func WithFilterOut(filt FilterOut) CreateOption { 91 return func(f *FigTree) { 92 f.filterOut = filt 93 } 94 } 95 96 func defaultFilterOut(f *FigTree) FilterOut { 97 configStop := false 98 return func(config *yaml.Node) bool { 99 // if previous parse found a stop we should abort here 100 if configStop { 101 return true 102 } 103 // now check if current doc has a stop, looking for: 104 // ``` 105 // config: 106 // stop: true|false 107 // ``` 108 if pragma := walky.GetKey(config, "config"); pragma != nil { 109 if stop := walky.GetKey(pragma, "stop"); stop != nil { 110 configStop, _ = strconv.ParseBool(stop.Value) 111 } 112 } 113 // even if current doc has a stop, we should continue to 114 // process it, we dont want to process the "next" document 115 return false 116 } 117 } 118 119 func WithoutExec() CreateOption { 120 return func(f *FigTree) { 121 f.exec = false 122 } 123 } 124 125 type FigTree struct { 126 home string 127 workDir string 128 configDir string 129 envPrefix string 130 preProcessor PreProcessor 131 applyChangeSet ChangeSetFunc 132 exec bool 133 filterOut FilterOut 134 } 135 136 func NewFigTree(opts ...CreateOption) *FigTree { 137 wd, _ := os.Getwd() 138 fig := &FigTree{ 139 home: os.Getenv("HOME"), 140 workDir: wd, 141 envPrefix: "FIGTREE", 142 applyChangeSet: defaultApplyChangeSet, 143 exec: true, 144 } 145 for _, opt := range opts { 146 opt(fig) 147 } 148 return fig 149 } 150 151 func (f *FigTree) WithHome(home string) { 152 WithHome(home)(f) 153 } 154 155 func (f *FigTree) WithCwd(cwd string) { 156 WithCwd(cwd)(f) 157 } 158 159 func (f *FigTree) WithEnvPrefix(env string) { 160 WithEnvPrefix(env)(f) 161 } 162 163 func (f *FigTree) WithConfigDir(dir string) { 164 WithConfigDir(dir)(f) 165 } 166 167 func (f *FigTree) WithPreProcessor(pp PreProcessor) { 168 WithPreProcessor(pp)(f) 169 } 170 171 func (f *FigTree) WithFilterOut(filt FilterOut) { 172 WithFilterOut(filt)(f) 173 } 174 175 func (f *FigTree) WithApplyChangeSet(apply ChangeSetFunc) { 176 WithApplyChangeSet(apply)(f) 177 } 178 179 func (f *FigTree) WithIgnoreChangeSet() { 180 WithApplyChangeSet(func(_ map[string]*string) error { 181 return nil 182 })(f) 183 } 184 185 func (f *FigTree) WithoutExec() { 186 WithoutExec()(f) 187 } 188 189 func (f *FigTree) Copy() *FigTree { 190 cp := *f 191 return &cp 192 } 193 194 func (f *FigTree) LoadAllConfigs(configFile string, options interface{}) error { 195 if f.configDir != "" { 196 configFile = path.Join(f.configDir, configFile) 197 } 198 199 paths := FindParentPaths(f.home, f.workDir, configFile) 200 paths = append([]string{fmt.Sprintf("/etc/%s", configFile)}, paths...) 201 202 configSources := []ConfigSource{} 203 // iterate paths in reverse 204 for i := len(paths) - 1; i >= 0; i-- { 205 file := paths[i] 206 cs, err := f.ReadFile(file) 207 if err != nil { 208 return err 209 } 210 if cs == nil { 211 // no file contents to parse, file likely does not exist 212 continue 213 } 214 configSources = append(configSources, *cs) 215 } 216 return f.LoadAllConfigSources(configSources, options) 217 } 218 219 type ConfigSource struct { 220 Config *yaml.Node 221 Filename string 222 } 223 224 func (f *FigTree) LoadAllConfigSources(sources []ConfigSource, options interface{}) error { 225 m := NewMerger() 226 filterOut := f.filterOut 227 if filterOut == nil { 228 filterOut = defaultFilterOut(f) 229 } 230 231 for _, source := range sources { 232 // automatically skip empty configs 233 if source.Config == nil || source.Config.IsZero() { 234 continue 235 } 236 skip := filterOut(source.Config) 237 if skip { 238 continue 239 } 240 241 m.sourceFile = source.Filename 242 err := f.loadConfigSource(m, source.Config, options) 243 if err != nil { 244 return err 245 } 246 m.advance() 247 } 248 return nil 249 } 250 251 func (f *FigTree) LoadConfigSource(config *yaml.Node, source string, options interface{}) error { 252 m := NewMerger(WithSourceFile(source)) 253 return f.loadConfigSource(m, config, options) 254 } 255 256 func sourceLine(file string, node *yaml.Node) string { 257 if node.Line > 0 { 258 return fmt.Sprintf("%s:%d:%d", file, node.Line, node.Column) 259 } 260 return file 261 } 262 263 func (f *FigTree) loadConfigSource(m *Merger, config *yaml.Node, options interface{}) error { 264 if !reflect.ValueOf(options).IsValid() { 265 return errors.Errorf("options argument [%#v] is not valid", options) 266 } 267 268 var err error 269 if f.preProcessor != nil { 270 err = f.preProcessor(config) 271 if err != nil { 272 return errors.Wrapf(err, "failed to process config file %s", sourceLine(m.sourceFile, config)) 273 } 274 } 275 276 err = config.Decode(m) 277 if err != nil { 278 return errors.WithStack(walky.ErrFilename(err, m.sourceFile)) 279 } 280 281 _, err = m.mergeStructs( 282 reflect.ValueOf(options), 283 newMergeSource(walky.UnwrapDocument(config)), 284 false, 285 ) 286 if err != nil { 287 return err 288 } 289 changeSet := f.PopulateEnv(options) 290 return f.applyChangeSet(changeSet) 291 } 292 293 func (f *FigTree) LoadConfig(file string, options interface{}) error { 294 cs, err := f.ReadFile(file) 295 if err != nil { 296 return err 297 } 298 if cs == nil { 299 // no file contents to parse, file likely does not exist 300 return nil 301 } 302 return f.LoadConfigSource(cs.Config, cs.Filename, options) 303 } 304 305 // ReadFile will return a ConfigSource for given file path. If the 306 // file is executable (and WithoutExec was not used), it will execute 307 // the file and return the stdout otherwise it will return the file 308 // contents directly. 309 func (f *FigTree) ReadFile(file string) (*ConfigSource, error) { 310 absFile := file 311 if !filepath.IsAbs(file) { 312 absFile = filepath.Clean(filepath.Join(f.workDir, file)) 313 } 314 rel, err := filepath.Rel(f.workDir, absFile) 315 if err != nil { 316 rel = file 317 } 318 var node yaml.Node 319 if stat, err := os.Stat(absFile); err == nil { 320 if stat.Mode()&0o111 == 0 || !f.exec { 321 Log.Debugf("Reading config %s", absFile) 322 fh, err := os.Open(absFile) 323 if err != nil { 324 return nil, errors.Wrapf(err, "failed to open %s", rel) 325 } 326 defer fh.Close() 327 decoder := yaml.NewDecoder(fh) 328 if err := decoder.Decode(&node); err != nil && !errors.Is(err, io.EOF) { 329 return nil, errors.WithStack(walky.ErrFilename(err, file)) 330 } 331 } else { 332 Log.Debugf("Found Executable Config file: %s", absFile) 333 // it is executable, so run it and try to parse the output 334 cmd := exec.Command(absFile) 335 stdout := bytes.NewBufferString("") 336 cmd.Stdout = stdout 337 cmd.Stderr = bytes.NewBufferString("") 338 if err := cmd.Run(); err != nil { 339 return nil, errors.Wrapf(err, "%s is executable, but it failed to execute:\n%s", file, cmd.Stderr) 340 } 341 rel += "[stdout]" 342 if err := yaml.Unmarshal(stdout.Bytes(), &node); err != nil { 343 return nil, err 344 } 345 } 346 return &ConfigSource{ 347 Config: &node, 348 Filename: rel, 349 }, nil 350 } 351 return nil, nil 352 } 353 354 func FindParentPaths(homedir, cwd, fileName string) []string { 355 paths := make([]string, 0) 356 if filepath.IsAbs(fileName) { 357 // dont recursively look for files when fileName is an abspath 358 _, err := os.Stat(fileName) 359 if err == nil { 360 paths = append(paths, fileName) 361 } 362 return paths 363 } 364 365 // special case if homedir is not in current path then check there anyway 366 if homedir != "" && !strings.HasPrefix(cwd, homedir) { 367 file := path.Join(homedir, fileName) 368 if _, err := os.Stat(file); err == nil { 369 paths = append(paths, filepath.FromSlash(file)) 370 } 371 } 372 373 var dir string 374 for _, part := range strings.Split(cwd, string(os.PathSeparator)) { 375 if part == "" && dir == "" { 376 dir = "/" 377 } else { 378 dir = path.Join(dir, part) 379 } 380 file := path.Join(dir, fileName) 381 if _, err := os.Stat(file); err == nil { 382 paths = append(paths, filepath.FromSlash(file)) 383 } 384 } 385 return paths 386 } 387 388 func (f *FigTree) FindParentPaths(fileName string) []string { 389 return FindParentPaths(f.home, f.workDir, fileName) 390 } 391 392 var camelCaseWords = regexp.MustCompile("[0-9A-Za-z]+") 393 394 func camelCase(name string) string { 395 words := camelCaseWords.FindAllString(name, -1) 396 for i, word := range words { 397 words[i] = strings.Title(word) 398 } 399 return strings.Join(words, "") 400 } 401 402 type Merger struct { 403 sourceFile string 404 preserveMap map[string]struct{} 405 Config ConfigOptions `json:"config,omitempty" yaml:"config,omitempty"` 406 ignore []string 407 } 408 409 type MergeOption func(*Merger) 410 411 func WithSourceFile(source string) MergeOption { 412 return func(m *Merger) { 413 m.sourceFile = source 414 } 415 } 416 417 func PreserveMap(keys ...string) MergeOption { 418 return func(m *Merger) { 419 for _, key := range keys { 420 m.preserveMap[key] = struct{}{} 421 } 422 } 423 } 424 425 func NewMerger(options ...MergeOption) *Merger { 426 m := &Merger{ 427 sourceFile: "merge", 428 preserveMap: make(map[string]struct{}), 429 } 430 for _, opt := range options { 431 opt(m) 432 } 433 return m 434 } 435 436 // advance will move all the current overwrite properties to 437 // the ignore properties, then reset the overwrite properties. 438 // This is used after a document has be processed so the next 439 // document does not modify overwritten fields. 440 func (m *Merger) advance() { 441 for _, overwrite := range m.Config.Overwrite { 442 found := false 443 for _, ignore := range m.ignore { 444 if ignore == overwrite { 445 found = true 446 break 447 } 448 } 449 if !found { 450 m.ignore = append(m.ignore, overwrite) 451 } 452 } 453 m.Config.Overwrite = nil 454 } 455 456 // Merge will attempt to merge the data from src into dst. src and dst may each 457 // be either a map or a struct. Structs do not need to have the same structure, 458 // but any field name that exists in both structs will must be the same type. 459 func Merge(dst, src interface{}) error { 460 dstValue := reflect.ValueOf(dst) 461 if dstValue.Kind() == reflect.Struct { 462 return errors.New("dst argument cannot be a struct (should be *struct)") 463 } 464 m := NewMerger() 465 _, err := m.mergeStructs(dstValue, newMergeSource(reflect.ValueOf(src)), false) 466 return err 467 } 468 469 // MakeMergeStruct will take multiple structs and return a pointer to a zero value for the 470 // anonymous struct that has all the public fields from all the structs merged into one struct. 471 // If there are multiple structs with the same field names, the first appearance of that name 472 // will be used. 473 func MakeMergeStruct(structs ...interface{}) interface{} { 474 m := NewMerger() 475 return m.MakeMergeStruct(structs...) 476 } 477 478 func (m *Merger) MakeMergeStruct(structs ...interface{}) interface{} { 479 values := []reflect.Value{} 480 for _, data := range structs { 481 values = append(values, reflect.ValueOf(data)) 482 } 483 return m.makeMergeStruct(values...).Interface() 484 } 485 486 func inlineField(field reflect.StructField) bool { 487 if tag := field.Tag.Get("figtree"); tag != "" { 488 return strings.HasSuffix(tag, ",inline") 489 } 490 if tag := field.Tag.Get("yaml"); tag != "" { 491 return strings.HasSuffix(tag, ",inline") 492 } 493 return false 494 } 495 496 func (m *Merger) makeMergeStruct(values ...reflect.Value) reflect.Value { 497 foundFields := map[string]reflect.StructField{} 498 for i := 0; i < len(values); i++ { 499 v := values[i] 500 if v.Kind() == reflect.Ptr { 501 v = v.Elem() 502 } 503 typ := v.Type() 504 var field reflect.StructField 505 if typ.Kind() == reflect.Struct { 506 for j := 0; j < typ.NumField(); j++ { 507 field = typ.Field(j) 508 if field.PkgPath != "" { 509 // unexported field, skip 510 continue 511 } 512 513 field.Name = CanonicalFieldName(field) 514 515 if f, ok := foundFields[field.Name]; ok { 516 if f.Type.Kind() == reflect.Struct && field.Type.Kind() == reflect.Struct { 517 if fName, fieldName := f.Type.Name(), field.Type.Name(); fName == "" || fieldName == "" || fName != fieldName { 518 // we have 2 fields with the same name and they are both structs, so we need 519 // to merge the existing struct with the new one in case they are different 520 newval := m.makeMergeStruct(reflect.New(f.Type).Elem(), reflect.New(field.Type).Elem()).Elem() 521 f.Type = newval.Type() 522 foundFields[field.Name] = f 523 } 524 } 525 // field already found, skip 526 continue 527 } 528 if inlineField(field) { 529 // insert inline after this value, it will have a higher 530 // "type" priority than later values 531 values = append(values[:i+1], append([]reflect.Value{v.Field(j)}, values[i+1:]...)...) 532 continue 533 } 534 foundFields[field.Name] = field 535 } 536 } else if typ.Kind() == reflect.Map { 537 for _, key := range v.MapKeys() { 538 keyval := reflect.ValueOf(v.MapIndex(key).Interface()) 539 if _, ok := m.preserveMap[key.String()]; !ok { 540 if keyval.Kind() == reflect.Ptr && keyval.Elem().Kind() == reflect.Map { 541 keyval = m.makeMergeStruct(keyval.Elem()) 542 } else if keyval.Kind() == reflect.Map { 543 keyval = m.makeMergeStruct(keyval).Elem() 544 } 545 } 546 var t reflect.Type 547 if !keyval.IsValid() { 548 // this nonsense is to create a generic `interface{}` type. There is 549 // probably an easier to do this, but it eludes me at the moment. 550 var dummy interface{} 551 t = reflect.ValueOf(&dummy).Elem().Type() 552 } else { 553 t = reflect.ValueOf(keyval.Interface()).Type() 554 } 555 field = reflect.StructField{ 556 Name: camelCase(key.String()), 557 Type: t, 558 Tag: reflect.StructTag(fmt.Sprintf(`json:"%s" yaml:"%s"`, key.String(), key.String())), 559 } 560 if f, ok := foundFields[field.Name]; ok { 561 if f.Type.Kind() == reflect.Struct && t.Kind() == reflect.Struct { 562 if fName, tName := f.Type.Name(), t.Name(); fName == "" || tName == "" || fName != tName { 563 // we have 2 fields with the same name and they are both structs, so we need 564 // to merge the existig struct with the new one in case they are different 565 newval := m.makeMergeStruct(reflect.New(f.Type).Elem(), reflect.New(t).Elem()).Elem() 566 f.Type = newval.Type() 567 foundFields[field.Name] = f 568 } 569 } 570 // field already found, skip 571 continue 572 } 573 foundFields[field.Name] = field 574 } 575 } 576 } 577 578 fields := []reflect.StructField{} 579 for _, value := range foundFields { 580 fields = append(fields, value) 581 } 582 sort.Slice(fields, func(i, j int) bool { 583 return fields[i].Name < fields[j].Name 584 }) 585 newType := reflect.StructOf(fields) 586 return reflect.New(newType) 587 } 588 589 func (m *Merger) mapToStruct(src reflect.Value) (reflect.Value, error) { 590 if src.Kind() != reflect.Map { 591 return reflect.Value{}, nil 592 } 593 594 dest := m.makeMergeStruct(src) 595 if dest.Kind() == reflect.Ptr { 596 dest = dest.Elem() 597 } 598 599 for _, key := range src.MapKeys() { 600 structFieldName := camelCase(key.String()) 601 keyval := reflect.ValueOf(src.MapIndex(key).Interface()) 602 // skip invalid (ie nil) key values 603 if !keyval.IsValid() { 604 continue 605 } 606 switch { 607 case keyval.Kind() == reflect.Ptr && keyval.Elem().Kind() == reflect.Map: 608 keyval, err := m.mapToStruct(keyval.Elem()) 609 if err != nil { 610 return reflect.Value{}, err 611 } 612 _, err = m.mergeStructs(dest.FieldByName(structFieldName), newMergeSource(reflect.ValueOf(keyval.Addr().Interface())), false) 613 if err != nil { 614 return reflect.Value{}, err 615 } 616 case keyval.Kind() == reflect.Map: 617 keyval, err := m.mapToStruct(keyval) 618 if err != nil { 619 return reflect.Value{}, err 620 } 621 _, err = m.mergeStructs(dest.FieldByName(structFieldName), newMergeSource(reflect.ValueOf(keyval.Interface())), false) 622 if err != nil { 623 return reflect.Value{}, err 624 } 625 default: 626 dest.FieldByName(structFieldName).Set(reflect.ValueOf(keyval.Interface())) 627 } 628 } 629 return dest, nil 630 } 631 632 func structToMap(src mergeSource) (mergeSource, error) { 633 if !src.isStruct() { 634 return src, nil 635 } 636 637 newMap := reflect.ValueOf(map[string]interface{}{}) 638 639 reflectedStruct, _, err := src.reflect() 640 if err != nil { 641 return mergeSource{}, err 642 } 643 typ := reflectedStruct.Type() 644 645 for i := 0; i < typ.NumField(); i++ { 646 structField := typ.Field(i) 647 if structField.PkgPath != "" { 648 // skip private fields 649 continue 650 } 651 name := yamlFieldName(structField) 652 newMap.SetMapIndex(reflect.ValueOf(name), reflectedStruct.Field(i)) 653 } 654 655 return newMergeSource(newMap), nil 656 } 657 658 type ConfigOptions struct { 659 Overwrite []string `json:"overwrite,omitempty" yaml:"overwrite,omitempty"` 660 } 661 662 func yamlFieldName(sf reflect.StructField) string { 663 if tag, ok := sf.Tag.Lookup("yaml"); ok { 664 // with yaml:"foobar,omitempty" 665 // we just want to the "foobar" part 666 parts := strings.Split(tag, ",") 667 if parts[0] != "" && parts[0] != "-" { 668 return parts[0] 669 } 670 } 671 // guess the field name from reversing camel case 672 // so "FooBar" becomes "foo-bar" 673 parts := camelcase.Split(sf.Name) 674 for i := range parts { 675 parts[i] = strings.ToLower(parts[i]) 676 } 677 return strings.Join(parts, "-") 678 } 679 680 // CanonicalFieldName will return the the field name that will be used with 681 // merging maps and structs where the name casing/formatting may not 682 // be consistent. If the field uses tag `figtree:",name=MyName"` then 683 // that name will be used instead of the default contention. 684 func CanonicalFieldName(sf reflect.StructField) string { 685 if tag, ok := sf.Tag.Lookup("figtree"); ok { 686 for _, part := range strings.Split(tag, ",") { 687 if strings.HasPrefix(part, "name=") { 688 return strings.TrimPrefix(part, "name=") 689 } 690 } 691 } 692 693 // For consistency with YAML data, determine a canonical field name 694 // based on the YAML tag. Do not rely on the Go struct field name unless 695 // there is no YAML tag. 696 return camelCase(yamlFieldName(sf)) 697 } 698 699 func (m *Merger) mustOverwrite(name string) bool { 700 for _, prop := range m.Config.Overwrite { 701 if name == prop { 702 return true 703 } 704 } 705 return false 706 } 707 708 func (m *Merger) mustIgnore(name string) bool { 709 for _, prop := range m.ignore { 710 if name == prop { 711 return true 712 } 713 } 714 return false 715 } 716 717 func isZeroOrDefaultOption(v reflect.Value) bool { 718 if option := toOption(v); option != nil { 719 // an option can only be `zero` if it is undefined 720 if !option.IsDefined() { 721 return true 722 } 723 return option.IsDefault() 724 } 725 return false 726 } 727 728 func toOption(v reflect.Value) option { 729 v = indirect(v) 730 if !v.IsValid() { 731 return nil 732 } 733 if !v.CanAddr() { 734 tmp := reflect.New(v.Type()).Elem() 735 tmp.Set(v) 736 v = tmp 737 } 738 if option, ok := v.Addr().Interface().(option); ok { 739 return option 740 } 741 return nil 742 } 743 744 func indirect(v reflect.Value) reflect.Value { 745 for v.Kind() == reflect.Pointer { 746 v = v.Elem() 747 } 748 return v 749 } 750 751 func uninterface(v reflect.Value) reflect.Value { 752 if v.Kind() == reflect.Interface { 753 return v.Elem() 754 } 755 return v 756 } 757 758 func isZero(v reflect.Value) bool { 759 v = indirect(v) 760 if !v.IsValid() { 761 return true 762 } 763 if option := toOption(v); option != nil { 764 return !option.IsDefined() 765 } 766 return reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) 767 } 768 769 func isSame(v1, v2 reflect.Value) bool { 770 v1Valid := v1.IsValid() 771 v2Valid := v2.IsValid() 772 if !v1Valid && !v2Valid { 773 return true 774 } 775 if !v1Valid || !v2Valid { 776 return false 777 } 778 return reflect.DeepEqual(v1.Interface(), v2.Interface()) 779 } 780 781 type notAssignableError struct { 782 dstType reflect.Type 783 srcType reflect.Type 784 sourceLocation SourceLocation 785 } 786 787 func (e notAssignableError) Error() string { 788 return fmt.Sprintf("%s: %s is not assignable to %s", e.sourceLocation, e.srcType, e.dstType) 789 } 790 791 var stringType = reflect.ValueOf("").Type() 792 793 type assignOptions struct { 794 // Overwrite will attempt to replace the destination with the source 795 // even if the dest is already a valid (non-zero, non-default) value. 796 Overwrite bool 797 798 // srcIsDefault is used internally when we are provided src as an Option 799 // and we unwrap the option and recursively assign the raw value to dest. 800 srcIsDefault bool 801 // destIsDefault is used internally when we are provided dest as an Option 802 // and we unwrap the option and recursively try to assign src to the raw 803 // value. 804 destIsDefault bool 805 // sourceLocation is used internally to track the source file 806 // name/line/column when that data is available. This is set 807 // when we recursively call assignValue after unwrapping src Option. 808 sourceLocation SourceLocation 809 } 810 811 // assignValue will attempt to assign src to dest. If there is no errors, the 812 // bool return value will indicate if the assignment happened, which will be 813 // false when the trying to assign to a non-zero, or non-default value (without 814 // Overwrite set) 815 func (m *Merger) assignValue(dest reflect.Value, src mergeSource, opts assignOptions) (bool, error) { 816 reflectedSrc, coord, err := src.reflect() 817 if err != nil { 818 return false, walky.ErrFilename(err, m.sourceFile) 819 } 820 Log.Debugf("assignValue: %#v to %#v [opts: %#v]\n", reflectedSrc, dest, opts) 821 if !dest.IsValid() || !reflectedSrc.IsValid() { 822 return false, nil 823 } 824 825 // Not much we can do here if dest is unsettable, this will happen if 826 // dest comes from a map without copying first. This is a programmer error. 827 if !dest.CanSet() { 828 return false, errors.Errorf("Cannot assign %#v to unsettable value %#v", reflectedSrc, dest) 829 } 830 831 // if we have a pointer value, deref (and create if nil) 832 if dest.Kind() == reflect.Pointer { 833 if dest.IsNil() { 834 dest.Set(reflect.New(dest.Type().Elem())) 835 } 836 dest = dest.Elem() 837 } 838 839 // if src is a pointer, deref, return if nil and not overwriting 840 if reflectedSrc.Kind() == reflect.Pointer { 841 reflectedSrc = reflectedSrc.Elem() 842 // reflectedSrc might be invalid if it was Nil so lets handle that now 843 if !reflectedSrc.IsValid() { 844 if opts.Overwrite { 845 dest.Set(reflectedSrc) 846 return true, nil 847 } 848 return false, nil 849 } 850 } 851 852 // check to see if we can convert src to dest type before we check to see 853 // if is assignable. We cannot assign float32 to float64, but we can 854 // convert float32 to float64 and then assign. Note we skip conversion 855 // to strings since almost anything can be converted to a string 856 if dest.Kind() != reflect.String && reflectedSrc.CanConvert(dest.Type()) { 857 reflectedSrc = reflectedSrc.Convert(dest.Type()) 858 } 859 860 // if the source is an option, get the raw value of the option 861 // and try to assign that to the dest. assignValue does not require 862 // the source to be addressable, but in order to check for the option 863 // interface we might have to make the source addressable via a copy. 864 addressableSrc := reflectedSrc 865 if !addressableSrc.CanAddr() { 866 addressableSrc = reflect.New(reflectedSrc.Type()).Elem() 867 addressableSrc.Set(reflectedSrc) 868 } 869 if option := toOption(addressableSrc); option != nil { 870 srcOptionValue := reflect.ValueOf(option.GetValue()) 871 opts.sourceLocation = option.GetSource() 872 opts.srcIsDefault = option.IsDefault() 873 return m.assignValue(dest, newMergeSource(srcOptionValue), opts) 874 } 875 876 // if dest is an option type, then try to assign directly to the 877 // raw option value and then populate the option object 878 if dest.CanAddr() { 879 if option := toOption(dest); option != nil { 880 destOptionValue := reflect.ValueOf(option.GetValue()) 881 if !destOptionValue.IsValid() { 882 // this will happen when we have an Option[any], and 883 // GetValue returns nil as the default value 884 if _, ok := dest.Interface().(Option[any]); ok { 885 // since we want an `any` we should be good with 886 // just creating the src type 887 destOptionValue = reflect.New(reflectedSrc.Type()).Elem() 888 } 889 } 890 if !destOptionValue.CanSet() { 891 destOptionValue = reflect.New(destOptionValue.Type()).Elem() 892 } 893 opts.destIsDefault = option.IsDefault() 894 ok, err := m.assignValue(destOptionValue, src, opts) 895 if err != nil { 896 return false, err 897 } 898 if ok { 899 if err := option.SetValue(destOptionValue.Interface()); err != nil { 900 return false, err 901 } 902 source := opts.sourceLocation 903 if source.Name == "" { 904 source.Name = m.sourceFile 905 } 906 if coord != nil { 907 source.Location = coord 908 } 909 option.SetSource(source) 910 } 911 return ok, nil 912 } 913 } 914 915 // if we are assigning to a yaml.Node then try to preserve the raw 916 // yaml.Node input, otherwise encode the src into the Node. 917 if node, ok := dest.Interface().(yaml.Node); ok { 918 if src.node != nil { 919 dest.Set(reflect.ValueOf(*src.node)) 920 return true, nil 921 } 922 if err := node.Encode(reflectedSrc.Interface()); err != nil { 923 return false, errors.WithStack(err) 924 } 925 dest.Set(reflect.ValueOf(node)) 926 return true, nil 927 } 928 929 if reflectedSrc.Type().AssignableTo(dest.Type()) { 930 shouldAssignDest := opts.Overwrite || isZero(dest) || (opts.destIsDefault && !opts.srcIsDefault) 931 if shouldAssignDest { 932 switch reflectedSrc.Kind() { 933 case reflect.Map: 934 // maps are mutable, so create a brand new shiny one 935 dup := reflect.New(reflectedSrc.Type()).Elem() 936 ok, err := m.mergeMaps(dup, src, opts.Overwrite) 937 if err != nil { 938 return false, err 939 } 940 if ok { 941 dest.Set(dup) 942 } 943 return ok, nil 944 case reflect.Slice: 945 if reflectedSrc.IsNil() { 946 dest.Set(reflectedSrc) 947 } else { 948 // slices are mutable, so create a brand new shiny one 949 cp := reflect.MakeSlice(reflectedSrc.Type(), reflectedSrc.Len(), reflectedSrc.Len()) 950 reflect.Copy(cp, reflectedSrc) 951 dest.Set(cp) 952 } 953 default: 954 dest.Set(reflectedSrc) 955 } 956 return true, nil 957 } 958 return false, nil 959 } 960 961 if dest.Kind() == reflect.Bool && reflectedSrc.Kind() == reflect.String { 962 b, err := strconv.ParseBool(reflectedSrc.Interface().(string)) 963 if err != nil { 964 return false, errors.Wrapf(err, "%s is not assignable to %s, invalid bool value %#v", reflectedSrc.Type(), dest.Type(), reflectedSrc) 965 } 966 dest.Set(reflect.ValueOf(b)) 967 return true, nil 968 } 969 970 if dest.Kind() == reflect.String && reflectedSrc.Kind() != reflect.String && stringType.AssignableTo(dest.Type()) { 971 switch reflectedSrc.Kind() { 972 case reflect.Array, reflect.Slice, reflect.Map: 973 return false, errors.WithStack( 974 notAssignableError{ 975 srcType: reflectedSrc.Type(), 976 dstType: dest.Type(), 977 sourceLocation: NewSource(m.sourceFile, WithLocation(coord)), 978 }, 979 ) 980 case reflect.Struct: 981 // we generally dont want to assign structs to a string 982 // unless that struct is an option struct in which case 983 // we use convert the value 984 if option := toOption(reflectedSrc); option != nil { 985 dest.Set(reflect.ValueOf(fmt.Sprint(option.GetValue()))) 986 } 987 return false, errors.WithStack( 988 notAssignableError{ 989 srcType: reflectedSrc.Type(), 990 dstType: dest.Type(), 991 sourceLocation: NewSource(m.sourceFile, WithLocation(coord)), 992 }, 993 ) 994 default: 995 // if we have a scalar node we want to convert to a string, just use 996 // the literal value tokenized from the document, this will 997 // allow values like `False` to be preserved as a case-sensitive 998 // string rather than being converted to a bool, the back to a string. 999 if src.node != nil && src.node.Kind == yaml.ScalarNode { 1000 dest.Set(reflect.ValueOf(src.node.Value)) 1001 } else { 1002 dest.Set(reflect.ValueOf(fmt.Sprint(reflectedSrc.Interface()))) 1003 } 1004 } 1005 return true, nil 1006 } 1007 1008 if !isSpecial(dest) && dest.CanAddr() { 1009 meth := dest.Addr().MethodByName("UnmarshalYAML") 1010 if meth.IsValid() { 1011 if src.node != nil { 1012 if err := src.node.Decode(dest.Addr().Interface()); err != nil { 1013 return false, errors.WithStack(walky.ErrFilename(walky.NewYAMLError(err, src.node), m.sourceFile)) 1014 } 1015 } else { 1016 // we know we have an UnmarshalYAML function, so use yaml 1017 // to convert to/from between random types since we can't 1018 // do it with reflection alone here. 1019 content, err := yaml.Marshal(reflectedSrc.Interface()) 1020 if err != nil { 1021 return false, errors.WithStack(err) 1022 } 1023 if err := yaml.Unmarshal(content, dest.Addr().Interface()); err != nil { 1024 return false, errors.WithStack(err) 1025 } 1026 } 1027 return true, nil 1028 } 1029 } 1030 1031 // if we have a collection don't proceed to attempt to unmarshal direct 1032 // from the yaml.Node ... collections are process per item, rather than 1033 // as a whole. 1034 if isCollection(dest) { 1035 return false, errors.WithStack( 1036 notAssignableError{ 1037 srcType: reflectedSrc.Type(), 1038 dstType: dest.Type(), 1039 sourceLocation: NewSource(m.sourceFile, WithLocation(coord)), 1040 }, 1041 ) 1042 } 1043 1044 // if we are still here, try one last-ditch effort to see if we can convert, 1045 // this time allowing things to convert to `string` types is okay since we 1046 // have exhausted all other assignment options above. 1047 if reflectedSrc.CanConvert(dest.Type()) { 1048 shouldAssignDest := opts.Overwrite || isZero(dest) || (opts.destIsDefault && !opts.srcIsDefault) 1049 if shouldAssignDest { 1050 reflectedSrc = reflectedSrc.Convert(dest.Type()) 1051 dest.Set(reflectedSrc) 1052 return true, nil 1053 } 1054 return false, nil 1055 } 1056 1057 return false, errors.WithStack( 1058 notAssignableError{ 1059 srcType: reflectedSrc.Type(), 1060 dstType: dest.Type(), 1061 sourceLocation: NewSource(m.sourceFile, WithLocation(coord)), 1062 }, 1063 ) 1064 } 1065 1066 type mergeSource struct { 1067 reflected reflect.Value 1068 node *yaml.Node 1069 coord *FileCoordinate 1070 } 1071 1072 func newMergeSource(src any) mergeSource { 1073 switch cast := src.(type) { 1074 case reflect.Value: 1075 cast = uninterface(indirect(cast)) 1076 if cast.IsValid() && cast.CanInterface() { 1077 // handle the edge case that someone has passed in 1078 // reflect.ValueOf(*yaml.Node) rather than *yaml.Node directly 1079 if node, ok := cast.Interface().(yaml.Node); ok { 1080 return mergeSource{ 1081 node: walky.Indirect(&node), 1082 } 1083 } 1084 } 1085 return mergeSource{ 1086 reflected: cast, 1087 } 1088 case *yaml.Node: 1089 return mergeSource{ 1090 node: walky.Indirect(cast), 1091 } 1092 } 1093 panic(fmt.Sprintf("Unknown type: %T", src)) 1094 } 1095 1096 func (ms *mergeSource) reflect() (reflect.Value, *FileCoordinate, error) { 1097 if ms.reflected.IsValid() && !ms.reflected.IsZero() { 1098 return ms.reflected, ms.coord, nil 1099 } 1100 if ms.node != nil { 1101 if ms.node.Line != 0 || ms.node.Column != 0 { 1102 ms.coord = &FileCoordinate{ 1103 Line: ms.node.Line, 1104 Column: ms.node.Column, 1105 } 1106 } 1107 var val any 1108 err := ms.node.Decode(&val) 1109 if err != nil { 1110 return reflect.Value{}, nil, errors.WithStack(walky.NewYAMLError(err, ms.node)) 1111 } 1112 ms.reflected = uninterface(reflect.ValueOf(&val).Elem()) 1113 } 1114 return ms.reflected, ms.coord, nil 1115 } 1116 1117 func (ms *mergeSource) isMap() bool { 1118 if ms.node != nil { 1119 return ms.node.Kind == yaml.MappingNode 1120 } 1121 return ms.reflected.Kind() == reflect.Map 1122 } 1123 1124 func (ms *mergeSource) isStruct() bool { 1125 if ms.node != nil { 1126 return false 1127 } 1128 return ms.reflected.Kind() == reflect.Struct 1129 } 1130 1131 func (ms *mergeSource) isList() bool { 1132 if ms.node != nil { 1133 return ms.node.Kind == yaml.SequenceNode 1134 } 1135 switch ms.reflected.Kind() { 1136 case reflect.Array, reflect.Slice: 1137 return true 1138 } 1139 return false 1140 } 1141 1142 func (ms *mergeSource) isZero() bool { 1143 if ms.node != nil { 1144 // values directly from config files cannot be 'zero' 1145 // ie `foo: false` is still non-zero even though the 1146 // value is the zero value (false) 1147 return false 1148 } 1149 return isZero(ms.reflected) 1150 } 1151 1152 func (ms *mergeSource) isValid() bool { 1153 if ms.node != nil { 1154 return !ms.node.IsZero() 1155 } 1156 return ms.reflected.IsValid() 1157 } 1158 1159 func (ms *mergeSource) len() int { 1160 if ms.node != nil { 1161 if ms.node.Kind == yaml.MappingNode || ms.node.Kind == yaml.SequenceNode { 1162 return len(ms.node.Content) 1163 } 1164 return 0 1165 } 1166 return ms.reflected.Len() 1167 } 1168 1169 func (ms *mergeSource) foreachField(f func(key string, value mergeSource, anonymous bool) error) error { 1170 if ms.node != nil { 1171 return walky.RangeMap(ms.node, func(keyNode, valueNode *yaml.Node) error { 1172 return f(keyNode.Value, newMergeSource(valueNode), false) 1173 }) 1174 } 1175 switch ms.reflected.Kind() { 1176 case reflect.Struct: 1177 for i := 0; i < ms.reflected.NumField(); i++ { 1178 structField := ms.reflected.Type().Field(i) 1179 field := ms.reflected.Field(i) 1180 field = uninterface(indirect(field)) 1181 fieldName := yamlFieldName(structField) 1182 if structField.PkgPath != "" && !structField.Anonymous { 1183 continue 1184 } 1185 err := f(fieldName, newMergeSource(field), structField.Anonymous) 1186 if err != nil { 1187 return err 1188 } 1189 } 1190 return nil 1191 case reflect.Map: 1192 for _, key := range ms.reflected.MapKeys() { 1193 val := ms.reflected.MapIndex(key) 1194 if val.IsValid() { 1195 val = uninterface(indirect(val)) 1196 } 1197 err := f(key.String(), newMergeSource(val), false) 1198 if err != nil { 1199 return err 1200 } 1201 } 1202 return nil 1203 } 1204 1205 return errors.Errorf("expected struct, got %s", ms.reflected.Kind()) 1206 } 1207 1208 func (ms *mergeSource) foreachKey(f func(key reflect.Value, value mergeSource) error) error { 1209 if ms.node != nil { 1210 return walky.RangeMap(ms.node, func(keyNode, valueNode *yaml.Node) error { 1211 newMS := newMergeSource(keyNode) 1212 key, _, err := newMS.reflect() 1213 if err != nil { 1214 return err 1215 } 1216 return f(key, newMergeSource(valueNode)) 1217 }) 1218 } 1219 if ms.reflected.Kind() == reflect.Map { 1220 for _, key := range ms.reflected.MapKeys() { 1221 val := ms.reflected.MapIndex(key) 1222 val = uninterface(indirect(val)) 1223 err := f(key, newMergeSource(val)) 1224 if err != nil { 1225 return err 1226 } 1227 } 1228 return nil 1229 } 1230 return errors.Errorf("not map") 1231 } 1232 1233 func (ms *mergeSource) foreach(f func(ix int, item mergeSource) error) error { 1234 if ms.node != nil { 1235 for i := 0; i < len(ms.node.Content); i += 1 { 1236 if err := f(i, newMergeSource(ms.node.Content[i])); err != nil { 1237 return err 1238 } 1239 } 1240 return nil 1241 } 1242 switch ms.reflected.Kind() { 1243 case reflect.Slice, reflect.Array: 1244 for i := 0; i < ms.reflected.Len(); i++ { 1245 item := ms.reflected.Index(i) 1246 item = uninterface(indirect(item)) 1247 if err := f(i, newMergeSource(item)); err != nil { 1248 return err 1249 } 1250 } 1251 return nil 1252 } 1253 return errors.Errorf("not slice or array") 1254 } 1255 1256 type fieldYAML struct { 1257 StructField reflect.StructField 1258 Value reflect.Value 1259 } 1260 1261 // populateYAMLMaps will collect a map by field name where 1262 // those field names are converted to a common name used in YAML 1263 // documents so we can easily merge fields and maps together from 1264 // multiple sources. 1265 func populateYAMLMaps(v reflect.Value) map[string]fieldYAML { 1266 fieldsByYAML := make(map[string]fieldYAML) 1267 if v.Kind() != reflect.Struct { 1268 return fieldsByYAML 1269 } 1270 for i := 0; i < v.NumField(); i++ { 1271 fieldType := v.Type().Field(i) 1272 yamlName := yamlFieldName(fieldType) 1273 if _, ok := fieldsByYAML[yamlName]; !ok { 1274 fieldsByYAML[yamlName] = fieldYAML{ 1275 StructField: fieldType, 1276 Value: v.Field(i), 1277 } 1278 } 1279 } 1280 1281 for i := 0; i < v.NumField(); i++ { 1282 fieldType := v.Type().Field(i) 1283 if !fieldType.Anonymous { 1284 continue 1285 } 1286 1287 fv := v.Field(i) 1288 if fv.Kind() == reflect.Pointer && fv.IsNil() { 1289 // skip nil pointers for embedded structs 1290 continue 1291 } 1292 if indirect(fv).Type().Kind() == reflect.Struct { 1293 anonFields := populateYAMLMaps(indirect(fv)) 1294 for k, v := range anonFields { 1295 if _, ok := fieldsByYAML[k]; !ok { 1296 fieldsByYAML[k] = v 1297 } 1298 } 1299 } 1300 } 1301 return fieldsByYAML 1302 } 1303 1304 func (m *Merger) mergeStructs(dst reflect.Value, src mergeSource, overwrite bool) (changed bool, err error) { 1305 dst = indirect(dst) 1306 1307 if dst.Kind() == reflect.Interface { 1308 realDst := dst.Elem() 1309 if realDst.IsValid() { 1310 newDst := reflect.New(realDst.Type()).Elem() 1311 newDst.Set(realDst) 1312 defer func(orig reflect.Value) { 1313 if changed { 1314 orig.Set(newDst) 1315 } 1316 }(dst) 1317 dst = newDst 1318 } 1319 } 1320 1321 if dst.Kind() == reflect.Map { 1322 return m.mergeMaps(dst, src, overwrite) 1323 } 1324 1325 if !dst.IsValid() || !src.isValid() { 1326 Log.Debugf("Valid: dst:%v src:%t", dst.IsValid(), src.isValid()) 1327 return false, nil 1328 } 1329 1330 // We first collect maps of struct fields by the yamlized name 1331 // so we can easily compare maps and structs by common names 1332 dstFieldsByYAML := populateYAMLMaps(dst) 1333 1334 err = src.foreachField(func(fieldName string, srcField mergeSource, anon bool) error { 1335 if m.mustIgnore(fieldName) { 1336 return nil 1337 } 1338 1339 dstFieldByYAML, ok := dstFieldsByYAML[fieldName] 1340 if !ok { 1341 if anon { 1342 // this is an embedded struct, and the destination does not contain 1343 // the same embedded struct, so try to merge the embedded struct 1344 // directly with the destination 1345 ok, err := m.mergeStructs(dst, srcField, m.mustOverwrite(fieldName)) 1346 if err != nil { 1347 return errors.WithStack(err) 1348 } 1349 changed = changed || ok 1350 } 1351 // if original value does not have the same struct field 1352 // then just skip this field. 1353 return nil 1354 } 1355 1356 // PkgPath is empty for upper case (exported) field names. 1357 if dstFieldByYAML.StructField.PkgPath != "" { 1358 // unexported field, skipping 1359 return nil 1360 } 1361 1362 dstField := dstFieldByYAML.Value 1363 1364 fieldChanged := false 1365 if dstField.Kind() == reflect.Interface { 1366 realDstField := dstField.Elem() 1367 if realDstField.IsValid() { 1368 newDstField := reflect.New(realDstField.Type()).Elem() 1369 newDstField.Set(realDstField) 1370 defer func(orig reflect.Value) { 1371 if fieldChanged { 1372 orig.Set(newDstField) 1373 } 1374 }(dstField) 1375 dstField = newDstField 1376 } 1377 } 1378 1379 // if we have a pointer value, deref (and create if nil) 1380 if dstField.Kind() == reflect.Pointer { 1381 if dstField.IsNil() { 1382 newField := reflect.New(dstField.Type().Elem()) 1383 defer func(origField reflect.Value) { 1384 if fieldChanged { 1385 origField.Set(newField) 1386 } 1387 }(dstField) 1388 dstField = newField 1389 } 1390 dstField = dstField.Elem() 1391 } 1392 1393 val, _, err := srcField.reflect() 1394 if err != nil { 1395 return walky.ErrFilename(err, m.sourceFile) 1396 } 1397 1398 shouldAssign := (isZero(dstField) && !srcField.isZero() || (isZeroOrDefaultOption(dstField) && !isZeroOrDefaultOption(val))) || (overwrite || m.mustOverwrite(fieldName)) 1399 1400 var assignErr error 1401 if shouldAssign && !isSame(dstField, val) { 1402 fieldChanged, assignErr = m.assignValue(dstField, srcField, assignOptions{ 1403 Overwrite: overwrite || m.mustOverwrite(fieldName), 1404 }) 1405 // if this is a notAssignableError then we want 1406 // to continue down to try to investigate more complex 1407 // types. For example we will get here when we try to 1408 // assign ListStringOption to []string or []interface 1409 // where we want to iterate below for each StringOption. 1410 var naErr notAssignableError 1411 if assignErr != nil && !errors.As(assignErr, &naErr) { 1412 return assignErr 1413 } 1414 changed = changed || fieldChanged 1415 if fieldChanged { 1416 return nil 1417 } 1418 } 1419 switch dstField.Kind() { 1420 case reflect.Map: 1421 Log.Debugf("Merging Map: %#v to %#v [overwrite: %t]", val, dstField, overwrite || m.mustOverwrite(fieldName)) 1422 ok, err := m.mergeStructs(dstField, srcField, overwrite || m.mustOverwrite(fieldName)) 1423 if err != nil { 1424 return errors.WithStack(err) 1425 } 1426 fieldChanged = fieldChanged || ok 1427 changed = changed || ok 1428 return nil 1429 case reflect.Slice, reflect.Array: 1430 Log.Debugf("Merging %#v to %#v [overwrite: %t]", val, dstField, overwrite || m.mustOverwrite(fieldName)) 1431 merged, ok, err := m.mergeArrays(dstField, srcField, overwrite || m.mustOverwrite(fieldName)) 1432 if err != nil { 1433 return err 1434 } 1435 if ok { 1436 dstField.Set(merged) 1437 } 1438 fieldChanged = fieldChanged || ok 1439 changed = changed || ok 1440 return nil 1441 case reflect.Struct: 1442 // only merge structs if they are not special structs (options or yaml.Node): 1443 if !isSpecial(dstField) { 1444 Log.Debugf("Merging Struct: %#v to %#v [overwrite: %t]", val, dstField, overwrite || m.mustOverwrite(fieldName)) 1445 ok, err := m.mergeStructs(dstField, srcField, overwrite || m.mustOverwrite(fieldName)) 1446 if err != nil { 1447 return errors.WithStack(err) 1448 } 1449 fieldChanged = fieldChanged || ok 1450 changed = changed || ok 1451 return nil 1452 } 1453 } 1454 return assignErr 1455 }) 1456 if err != nil { 1457 return changed, walky.ErrFilename(err, m.sourceFile) 1458 } 1459 return changed, nil 1460 } 1461 1462 func (m *Merger) mergeMaps(dst reflect.Value, src mergeSource, overwrite bool) (bool, error) { 1463 if src.isStruct() { 1464 var err error 1465 src, err = structToMap(src) 1466 if err != nil { 1467 return false, err 1468 } 1469 } 1470 if !src.isMap() { 1471 return false, nil 1472 } 1473 if overwrite { 1474 // truncate all the keys 1475 for _, key := range dst.MapKeys() { 1476 // setting to zero value is a "delete" operation 1477 dst.SetMapIndex(key, reflect.Value{}) 1478 } 1479 } 1480 1481 changed := false 1482 err := src.foreachKey(func(key reflect.Value, value mergeSource) error { 1483 if !dst.MapIndex(key).IsValid() { 1484 dstElem := reflect.New(dst.Type().Elem()).Elem() 1485 ok, err := m.assignValue(dstElem, value, assignOptions{ 1486 Overwrite: overwrite, 1487 }) 1488 if option := toOption(dstElem); option != nil { 1489 loc := option.GetSource() 1490 if loc.Location == nil { 1491 _, coord, err := value.reflect() 1492 if err != nil { 1493 return walky.ErrFilename(err, m.sourceFile) 1494 } 1495 loc.Location = coord 1496 } 1497 if loc.Name == "" { 1498 loc.Name = m.sourceFile 1499 } 1500 option.SetSource(loc) 1501 } 1502 var assignErr notAssignableError 1503 if err != nil && !errors.As(err, &assignErr) { 1504 return err 1505 } else if err == nil { 1506 if dst.IsNil() { 1507 if !dst.CanSet() { 1508 // TODO: Should this be an error? 1509 return nil 1510 } 1511 dst.Set(reflect.MakeMap(dst.Type())) 1512 } 1513 Log.Debugf("Setting %v to %#v", key.Interface(), dstElem.Interface()) 1514 dst.SetMapIndex(key, dstElem) 1515 changed = changed || ok 1516 return nil 1517 } 1518 } 1519 1520 if dst.IsNil() { 1521 // nil map here, we need to create one 1522 newMap := reflect.MakeMap(dst.Type()) 1523 dst.Set(newMap) 1524 } 1525 if !dst.MapIndex(key).IsValid() { 1526 newVal := reflect.New(dst.Type().Elem()).Elem() 1527 dst.SetMapIndex(key, newVal) 1528 changed = true 1529 } 1530 dstVal := reflect.ValueOf(dst.MapIndex(key).Interface()) 1531 dstValKind := dstVal.Kind() 1532 switch { 1533 case dstValKind == reflect.Map: 1534 Log.Debugf("Merging: %#v to %#v", value, dstVal) 1535 ok, err := m.mergeStructs(dstVal, value, overwrite || m.mustOverwrite(key.String())) 1536 if err != nil { 1537 return errors.WithStack(err) 1538 } 1539 changed = changed || ok 1540 return nil 1541 case dstValKind == reflect.Struct && !isSpecial(dstVal): 1542 Log.Debugf("Merging: %#v to %#v", value, dstVal) 1543 if !dstVal.CanAddr() { 1544 // we can't address dstVal so we need to make a new value 1545 // outside the map, merge into the new value, then 1546 // set the map key to the new value 1547 newVal := reflect.New(dstVal.Type()).Elem() 1548 newVal.Set(dstVal) 1549 ok, err := m.mergeStructs(newVal, value, overwrite || m.mustOverwrite(key.String())) 1550 if err != nil { 1551 return errors.WithStack(err) 1552 } 1553 if ok { 1554 dst.SetMapIndex(key, newVal) 1555 changed = true 1556 } 1557 return nil 1558 } 1559 ok, err := m.mergeStructs(dstVal, value, overwrite || m.mustOverwrite(key.String())) 1560 if err != nil { 1561 return errors.WithStack(err) 1562 } 1563 changed = changed || ok 1564 return nil 1565 case dstValKind == reflect.Slice, dstValKind == reflect.Array: 1566 Log.Debugf("Merging: %#v to %#v", value, dstVal) 1567 merged, ok, err := m.mergeArrays(dstVal, value, overwrite || m.mustOverwrite(key.String())) 1568 if err != nil { 1569 return err 1570 } 1571 if ok { 1572 dst.SetMapIndex(key, merged) 1573 } 1574 changed = changed || ok 1575 default: 1576 if !isZero(dstVal) { 1577 return nil 1578 } 1579 reflected, _, err := value.reflect() 1580 if err != nil { 1581 return walky.ErrFilename(err, m.sourceFile) 1582 } 1583 if !reflected.IsValid() { 1584 return nil 1585 } 1586 if !dstVal.IsValid() || reflected.Type().AssignableTo(dstVal.Type()) { 1587 dst.SetMapIndex(key, reflected) 1588 } else { 1589 if srcOption := toOption(reflected); srcOption != nil { 1590 dst.SetMapIndex(key, reflect.ValueOf(srcOption.GetValue())) 1591 return nil 1592 } 1593 settableDstVal := reflect.New(dstVal.Type()).Elem() 1594 settableDstVal.Set(dstVal) 1595 ok, err := m.assignValue(settableDstVal, value, assignOptions{ 1596 Overwrite: overwrite || m.mustOverwrite(key.String()), 1597 }) 1598 if err != nil { 1599 return errors.WithStack(err) 1600 } 1601 if ok { 1602 dst.SetMapIndex(key, settableDstVal) 1603 changed = true 1604 return nil 1605 } 1606 return errors.Errorf("map value %T is not assignable to %T", reflected.Interface(), dstVal.Interface()) 1607 } 1608 } 1609 return nil 1610 }) 1611 if err != nil { 1612 return changed, walky.ErrFilename(err, m.sourceFile) 1613 } 1614 return changed, nil 1615 } 1616 1617 func isCollection(dst reflect.Value) bool { 1618 if !dst.IsValid() { 1619 return false 1620 } 1621 switch dst.Kind() { 1622 case reflect.Map, reflect.Slice, reflect.Array: 1623 return true 1624 case reflect.Struct: 1625 return !isSpecial(dst) 1626 } 1627 return false 1628 } 1629 1630 // isSpecial returns true if the value is an Option, slice of Options 1631 // map of Options or a yaml.Node. 1632 func isSpecial(dst reflect.Value) bool { 1633 if !dst.IsValid() { 1634 return false 1635 } 1636 if option := toOption(dst); option != nil { 1637 return true 1638 } 1639 if _, ok := dst.Interface().(yaml.Node); ok { 1640 return true 1641 } 1642 return false 1643 } 1644 1645 func (m *Merger) mergeArrays(dst reflect.Value, src mergeSource, overwrite bool) (reflect.Value, bool, error) { 1646 var cp reflect.Value 1647 switch dst.Type().Kind() { 1648 case reflect.Slice: 1649 if overwrite { 1650 // overwriting so just make a new slice 1651 cp = reflect.MakeSlice(dst.Type(), 0, 0) 1652 } else { 1653 cp = reflect.MakeSlice(dst.Type(), dst.Len(), dst.Len()) 1654 reflect.Copy(cp, dst) 1655 } 1656 case reflect.Array: 1657 // arrays are copied, not passed by reference, so we dont need to copy 1658 cp = dst 1659 } 1660 1661 if !src.isList() { 1662 reflectedSrc, coord, err := src.reflect() 1663 if err != nil { 1664 return reflect.Value{}, false, walky.ErrFilename(err, m.sourceFile) 1665 } 1666 if !reflectedSrc.IsValid() { 1667 // if this is a nil interface data then 1668 // we don't care that we cant assign it to a 1669 // list, it is a no-op anyway. 1670 return cp, false, nil 1671 } 1672 return reflect.Value{}, false, errors.WithStack( 1673 notAssignableError{ 1674 srcType: reflectedSrc.Type(), 1675 dstType: dst.Type(), 1676 sourceLocation: NewSource(m.sourceFile, WithLocation(coord)), 1677 }, 1678 ) 1679 } 1680 1681 // when dst is an empty array we dont want to dedup those elements, they 1682 // should all be directly assigned. We only want to dedup when merging 1683 // in arrays from alternate sources, not the original source. 1684 skipDedup := false 1685 if cp.Len() == 0 { 1686 skipDedup = true 1687 } 1688 1689 var zero interface{} 1690 changed := overwrite 1691 err := src.foreach(func(ix int, item mergeSource) error { 1692 reflected, _, err := item.reflect() 1693 if err != nil { 1694 return walky.ErrFilename(err, m.sourceFile) 1695 } 1696 if dst.Kind() == reflect.Array { 1697 if dst.Len() <= ix { 1698 // truncate arrays, we cannot append 1699 return nil 1700 } 1701 dstElem := dst.Index(ix) 1702 if isZeroOrDefaultOption(dstElem) || dstElem.IsZero() || overwrite { 1703 ok, err := m.assignValue(dstElem, item, assignOptions{ 1704 Overwrite: overwrite, 1705 }) 1706 if err != nil { 1707 return err 1708 } 1709 changed = changed || ok 1710 } 1711 return nil 1712 } 1713 1714 // if src or dst's are options we need to compare the 1715 // values to determine if we need to skip inserting this 1716 // element 1717 compareValue := reflected 1718 if nOption := toOption(reflected); nOption != nil { 1719 if !nOption.IsDefined() { 1720 return nil 1721 } 1722 compareValue = reflect.ValueOf(nOption.GetValue()) 1723 } 1724 1725 if !compareValue.IsValid() || reflect.DeepEqual(compareValue.Interface(), zero) { 1726 return nil 1727 } 1728 1729 if !skipDedup { 1730 for i := 0; i < cp.Len(); i++ { 1731 destElem := cp.Index(i) 1732 if destOption := toOption(destElem); destOption != nil { 1733 destElem = reflect.ValueOf(destOption.GetValue()) 1734 } 1735 1736 // try to assign the input to a tmp value so all the normal 1737 // conversions happen before we compare it to existing elements. 1738 // Otherwise we might end up with extra dups in the array 1739 // that are the same value 1740 if destElem.CanInterface() { 1741 // ensure the destElem is not a nil value for a pointer type 1742 // .. in which case this will return a zero Value, which 1743 // cannot have `Type` called on it. 1744 if !reflect.ValueOf(destElem.Interface()).IsValid() { 1745 continue 1746 } 1747 tmpVal := reflect.New(reflect.ValueOf(destElem.Interface()).Type()).Elem() 1748 _, err := m.assignValue(tmpVal, item, assignOptions{}) 1749 if err == nil { 1750 if reflect.DeepEqual(destElem.Interface(), tmpVal.Interface()) { 1751 return nil 1752 } 1753 } 1754 } 1755 } 1756 } 1757 1758 dstElem := reflect.New(cp.Type().Elem()).Elem() 1759 dstKind := dstElem.Kind() 1760 switch { 1761 case dstKind == reflect.Map, (dstKind == reflect.Struct && !isSpecial(dstElem)): 1762 Log.Debugf("Merging: %#v to %#v", reflected, dstElem) 1763 ok, err := m.mergeStructs(dstElem, item, overwrite) 1764 if err != nil { 1765 return errors.WithStack(err) 1766 } 1767 changed = changed || ok 1768 case dstKind == reflect.Slice, dstKind == reflect.Array: 1769 Log.Debugf("Merging: %#v to %#v", reflected, dstElem) 1770 merged, ok, err := m.mergeArrays(dstElem, item, overwrite) 1771 if err != nil { 1772 return err 1773 } 1774 if ok { 1775 dstElem.Set(merged) 1776 } 1777 changed = changed || ok 1778 default: 1779 ok, err := m.assignValue(dstElem, item, assignOptions{ 1780 Overwrite: overwrite, 1781 }) 1782 if err != nil { 1783 return err 1784 } 1785 changed = changed || ok 1786 } 1787 1788 cp = reflect.Append(cp, dstElem) 1789 return nil 1790 }) 1791 if err != nil { 1792 return reflect.Value{}, false, err 1793 } 1794 return cp, changed, nil 1795 } 1796 1797 func (f *FigTree) formatEnvName(name string) string { 1798 name = fmt.Sprintf("%s_%s", f.envPrefix, strings.ToUpper(name)) 1799 1800 return strings.Map(func(r rune) rune { 1801 if unicode.IsDigit(r) || unicode.IsLetter(r) { 1802 return r 1803 } 1804 return '_' 1805 }, name) 1806 } 1807 1808 func (f *FigTree) formatEnvValue(value reflect.Value) (string, bool) { 1809 switch t := value.Interface().(type) { 1810 case string: 1811 return t, true 1812 case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool: 1813 return fmt.Sprintf("%v", t), true 1814 default: 1815 switch value.Kind() { 1816 case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: 1817 if value.IsNil() { 1818 return "", false 1819 } 1820 } 1821 if t == nil { 1822 return "", false 1823 } 1824 type definable interface { 1825 IsDefined() bool 1826 } 1827 if def, ok := t.(definable); ok { 1828 // skip fields that are not defined 1829 if !def.IsDefined() { 1830 return "", false 1831 } 1832 } 1833 type gettable interface { 1834 GetValue() interface{} 1835 } 1836 if get, ok := t.(gettable); ok { 1837 return fmt.Sprintf("%v", get.GetValue()), true 1838 } else { 1839 if b, err := json.Marshal(t); err == nil { 1840 val := strings.TrimSpace(string(b)) 1841 if val == "null" { 1842 return "", true 1843 } 1844 return val, true 1845 } 1846 } 1847 } 1848 return "", false 1849 } 1850 1851 func (f *FigTree) PopulateEnv(data interface{}) (changeSet map[string]*string) { 1852 changeSet = make(map[string]*string) 1853 1854 options := reflect.ValueOf(data) 1855 if options.Kind() == reflect.Ptr { 1856 options = reflect.ValueOf(options.Elem().Interface()) 1857 } 1858 if options.Kind() == reflect.Map { 1859 for _, key := range options.MapKeys() { 1860 if strKey, ok := key.Interface().(string); ok { 1861 // first chunk up string so that `foo-bar` becomes ["foo", "bar"] 1862 parts := strings.FieldsFunc(strKey, func(r rune) bool { 1863 return !unicode.IsLetter(r) && !unicode.IsNumber(r) 1864 }) 1865 // now for each chunk split again on camelcase so ["fooBar", "baz"] 1866 // becomes ["foo", "Bar", "baz"] 1867 allParts := []string{} 1868 for _, part := range parts { 1869 allParts = append(allParts, camelcase.Split(part)...) 1870 } 1871 1872 name := strings.Join(allParts, "_") 1873 envName := f.formatEnvName(name) 1874 val, ok := f.formatEnvValue(options.MapIndex(key)) 1875 if ok { 1876 changeSet[envName] = &val 1877 } else { 1878 changeSet[envName] = nil 1879 } 1880 } 1881 } 1882 } else if options.Kind() == reflect.Struct { 1883 for i := 0; i < options.NumField(); i++ { 1884 structField := options.Type().Field(i) 1885 // PkgPath is empty for upper case (exported) field names. 1886 if structField.PkgPath != "" { 1887 // unexported field, skipping 1888 continue 1889 } 1890 1891 envNames := []string{strings.Join(camelcase.Split(structField.Name), "_")} 1892 formatName := true 1893 if tag := structField.Tag.Get("figtree"); tag != "" { 1894 if strings.Contains(tag, ",inline") { 1895 // if we have a tag like: `figtree:",inline"` then we 1896 // want to the field as a top level member and not serialize 1897 // the raw struct to json, so just recurse here 1898 nestedEnvSet := f.PopulateEnv(options.Field(i).Interface()) 1899 for k, v := range nestedEnvSet { 1900 changeSet[k] = v 1901 } 1902 continue 1903 } 1904 if strings.Contains(tag, ",raw") { 1905 formatName = false 1906 } 1907 // next look for `figtree:"env,..."` to set the env name to that 1908 parts := strings.Split(tag, ",") 1909 if len(parts) > 0 { 1910 // if the env name is "-" then we should not populate this data into the env 1911 if parts[0] == "-" { 1912 continue 1913 } 1914 for _, part := range parts { 1915 if strings.HasPrefix(part, "name=") { 1916 continue 1917 } 1918 envNames = strings.Split(part, ";") 1919 break 1920 } 1921 } 1922 } 1923 for _, name := range envNames { 1924 envName := name 1925 if formatName { 1926 envName = f.formatEnvName(name) 1927 } 1928 val, ok := f.formatEnvValue(options.Field(i)) 1929 if ok { 1930 changeSet[envName] = &val 1931 } else { 1932 changeSet[envName] = nil 1933 } 1934 } 1935 } 1936 } 1937 1938 return changeSet 1939 }