github.com/diptanu/nomad@v0.5.7-0.20170516172507-d72e86cbe3d9/command/agent/config_parse.go (about) 1 package agent 2 3 import ( 4 "bytes" 5 "fmt" 6 "io" 7 "os" 8 "path/filepath" 9 "time" 10 11 "github.com/hashicorp/go-multierror" 12 "github.com/hashicorp/hcl" 13 "github.com/hashicorp/hcl/hcl/ast" 14 "github.com/hashicorp/nomad/nomad/structs/config" 15 "github.com/mitchellh/mapstructure" 16 ) 17 18 // ParseConfigFile parses the given path as a config file. 19 func ParseConfigFile(path string) (*Config, error) { 20 path, err := filepath.Abs(path) 21 if err != nil { 22 return nil, err 23 } 24 25 f, err := os.Open(path) 26 if err != nil { 27 return nil, err 28 } 29 defer f.Close() 30 31 config, err := ParseConfig(f) 32 if err != nil { 33 return nil, err 34 } 35 36 return config, nil 37 } 38 39 // ParseConfig parses the config from the given io.Reader. 40 // 41 // Due to current internal limitations, the entire contents of the 42 // io.Reader will be copied into memory first before parsing. 43 func ParseConfig(r io.Reader) (*Config, error) { 44 // Copy the reader into an in-memory buffer first since HCL requires it. 45 var buf bytes.Buffer 46 if _, err := io.Copy(&buf, r); err != nil { 47 return nil, err 48 } 49 50 // Parse the buffer 51 root, err := hcl.Parse(buf.String()) 52 if err != nil { 53 return nil, fmt.Errorf("error parsing: %s", err) 54 } 55 buf.Reset() 56 57 // Top-level item should be a list 58 list, ok := root.Node.(*ast.ObjectList) 59 if !ok { 60 return nil, fmt.Errorf("error parsing: root should be an object") 61 } 62 63 var config Config 64 if err := parseConfig(&config, list); err != nil { 65 return nil, fmt.Errorf("error parsing 'config': %v", err) 66 } 67 68 return &config, nil 69 } 70 71 func parseConfig(result *Config, list *ast.ObjectList) error { 72 // Check for invalid keys 73 valid := []string{ 74 "region", 75 "datacenter", 76 "name", 77 "data_dir", 78 "log_level", 79 "bind_addr", 80 "enable_debug", 81 "ports", 82 "addresses", 83 "interfaces", 84 "advertise", 85 "client", 86 "server", 87 "telemetry", 88 "leave_on_interrupt", 89 "leave_on_terminate", 90 "enable_syslog", 91 "syslog_facility", 92 "disable_update_check", 93 "disable_anonymous_signature", 94 "atlas", 95 "consul", 96 "vault", 97 "tls", 98 "http_api_response_headers", 99 } 100 if err := checkHCLKeys(list, valid); err != nil { 101 return multierror.Prefix(err, "config:") 102 } 103 104 // Decode the full thing into a map[string]interface for ease 105 var m map[string]interface{} 106 if err := hcl.DecodeObject(&m, list); err != nil { 107 return err 108 } 109 delete(m, "ports") 110 delete(m, "addresses") 111 delete(m, "interfaces") 112 delete(m, "advertise") 113 delete(m, "client") 114 delete(m, "server") 115 delete(m, "telemetry") 116 delete(m, "atlas") 117 delete(m, "consul") 118 delete(m, "vault") 119 delete(m, "tls") 120 delete(m, "http_api_response_headers") 121 122 // Decode the rest 123 if err := mapstructure.WeakDecode(m, result); err != nil { 124 return err 125 } 126 127 // Parse ports 128 if o := list.Filter("ports"); len(o.Items) > 0 { 129 if err := parsePorts(&result.Ports, o); err != nil { 130 return multierror.Prefix(err, "ports ->") 131 } 132 } 133 134 // Parse addresses 135 if o := list.Filter("addresses"); len(o.Items) > 0 { 136 if err := parseAddresses(&result.Addresses, o); err != nil { 137 return multierror.Prefix(err, "addresses ->") 138 } 139 } 140 141 // Parse advertise 142 if o := list.Filter("advertise"); len(o.Items) > 0 { 143 if err := parseAdvertise(&result.AdvertiseAddrs, o); err != nil { 144 return multierror.Prefix(err, "advertise ->") 145 } 146 } 147 148 // Parse client config 149 if o := list.Filter("client"); len(o.Items) > 0 { 150 if err := parseClient(&result.Client, o); err != nil { 151 return multierror.Prefix(err, "client ->") 152 } 153 } 154 155 // Parse server config 156 if o := list.Filter("server"); len(o.Items) > 0 { 157 if err := parseServer(&result.Server, o); err != nil { 158 return multierror.Prefix(err, "server ->") 159 } 160 } 161 162 // Parse telemetry config 163 if o := list.Filter("telemetry"); len(o.Items) > 0 { 164 if err := parseTelemetry(&result.Telemetry, o); err != nil { 165 return multierror.Prefix(err, "telemetry ->") 166 } 167 } 168 169 // Parse atlas config 170 if o := list.Filter("atlas"); len(o.Items) > 0 { 171 if err := parseAtlas(&result.Atlas, o); err != nil { 172 return multierror.Prefix(err, "atlas ->") 173 } 174 } 175 176 // Parse the consul config 177 if o := list.Filter("consul"); len(o.Items) > 0 { 178 if err := parseConsulConfig(&result.Consul, o); err != nil { 179 return multierror.Prefix(err, "consul ->") 180 } 181 } 182 183 // Parse the vault config 184 if o := list.Filter("vault"); len(o.Items) > 0 { 185 if err := parseVaultConfig(&result.Vault, o); err != nil { 186 return multierror.Prefix(err, "vault ->") 187 } 188 } 189 190 // Parse the TLS config 191 if o := list.Filter("tls"); len(o.Items) > 0 { 192 if err := parseTLSConfig(&result.TLSConfig, o); err != nil { 193 return multierror.Prefix(err, "tls ->") 194 } 195 } 196 197 // Parse out http_api_response_headers fields. These are in HCL as a list so 198 // we need to iterate over them and merge them. 199 if headersO := list.Filter("http_api_response_headers"); len(headersO.Items) > 0 { 200 for _, o := range headersO.Elem().Items { 201 var m map[string]interface{} 202 if err := hcl.DecodeObject(&m, o.Val); err != nil { 203 return err 204 } 205 if err := mapstructure.WeakDecode(m, &result.HTTPAPIResponseHeaders); err != nil { 206 return err 207 } 208 } 209 } 210 211 return nil 212 } 213 214 func parsePorts(result **Ports, list *ast.ObjectList) error { 215 list = list.Elem() 216 if len(list.Items) > 1 { 217 return fmt.Errorf("only one 'ports' block allowed") 218 } 219 220 // Get our ports object 221 listVal := list.Items[0].Val 222 223 // Check for invalid keys 224 valid := []string{ 225 "http", 226 "rpc", 227 "serf", 228 } 229 if err := checkHCLKeys(listVal, valid); err != nil { 230 return err 231 } 232 233 var m map[string]interface{} 234 if err := hcl.DecodeObject(&m, listVal); err != nil { 235 return err 236 } 237 238 var ports Ports 239 if err := mapstructure.WeakDecode(m, &ports); err != nil { 240 return err 241 } 242 *result = &ports 243 return nil 244 } 245 246 func parseAddresses(result **Addresses, list *ast.ObjectList) error { 247 list = list.Elem() 248 if len(list.Items) > 1 { 249 return fmt.Errorf("only one 'addresses' block allowed") 250 } 251 252 // Get our addresses object 253 listVal := list.Items[0].Val 254 255 // Check for invalid keys 256 valid := []string{ 257 "http", 258 "rpc", 259 "serf", 260 } 261 if err := checkHCLKeys(listVal, valid); err != nil { 262 return err 263 } 264 265 var m map[string]interface{} 266 if err := hcl.DecodeObject(&m, listVal); err != nil { 267 return err 268 } 269 270 var addresses Addresses 271 if err := mapstructure.WeakDecode(m, &addresses); err != nil { 272 return err 273 } 274 *result = &addresses 275 return nil 276 } 277 278 func parseAdvertise(result **AdvertiseAddrs, list *ast.ObjectList) error { 279 list = list.Elem() 280 if len(list.Items) > 1 { 281 return fmt.Errorf("only one 'advertise' block allowed") 282 } 283 284 // Get our advertise object 285 listVal := list.Items[0].Val 286 287 // Check for invalid keys 288 valid := []string{ 289 "http", 290 "rpc", 291 "serf", 292 } 293 if err := checkHCLKeys(listVal, valid); err != nil { 294 return err 295 } 296 297 var m map[string]interface{} 298 if err := hcl.DecodeObject(&m, listVal); err != nil { 299 return err 300 } 301 302 var advertise AdvertiseAddrs 303 if err := mapstructure.WeakDecode(m, &advertise); err != nil { 304 return err 305 } 306 *result = &advertise 307 return nil 308 } 309 310 func parseClient(result **ClientConfig, list *ast.ObjectList) error { 311 list = list.Elem() 312 if len(list.Items) > 1 { 313 return fmt.Errorf("only one 'client' block allowed") 314 } 315 316 // Get our client object 317 obj := list.Items[0] 318 319 // Value should be an object 320 var listVal *ast.ObjectList 321 if ot, ok := obj.Val.(*ast.ObjectType); ok { 322 listVal = ot.List 323 } else { 324 return fmt.Errorf("client value: should be an object") 325 } 326 327 // Check for invalid keys 328 valid := []string{ 329 "enabled", 330 "state_dir", 331 "alloc_dir", 332 "servers", 333 "node_class", 334 "options", 335 "meta", 336 "chroot_env", 337 "network_interface", 338 "network_speed", 339 "cpu_total_compute", 340 "max_kill_timeout", 341 "client_max_port", 342 "client_min_port", 343 "reserved", 344 "stats", 345 "gc_interval", 346 "gc_disk_usage_threshold", 347 "gc_inode_usage_threshold", 348 "gc_parallel_destroys", 349 "no_host_uuid", 350 } 351 if err := checkHCLKeys(listVal, valid); err != nil { 352 return err 353 } 354 355 var m map[string]interface{} 356 if err := hcl.DecodeObject(&m, listVal); err != nil { 357 return err 358 } 359 360 delete(m, "options") 361 delete(m, "meta") 362 delete(m, "chroot_env") 363 delete(m, "reserved") 364 delete(m, "stats") 365 366 var config ClientConfig 367 dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ 368 DecodeHook: mapstructure.StringToTimeDurationHookFunc(), 369 WeaklyTypedInput: true, 370 Result: &config, 371 }) 372 if err != nil { 373 return err 374 } 375 if err := dec.Decode(m); err != nil { 376 return err 377 } 378 379 // Parse out options fields. These are in HCL as a list so we need to 380 // iterate over them and merge them. 381 if optionsO := listVal.Filter("options"); len(optionsO.Items) > 0 { 382 for _, o := range optionsO.Elem().Items { 383 var m map[string]interface{} 384 if err := hcl.DecodeObject(&m, o.Val); err != nil { 385 return err 386 } 387 if err := mapstructure.WeakDecode(m, &config.Options); err != nil { 388 return err 389 } 390 } 391 } 392 393 // Parse out options meta. These are in HCL as a list so we need to 394 // iterate over them and merge them. 395 if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { 396 for _, o := range metaO.Elem().Items { 397 var m map[string]interface{} 398 if err := hcl.DecodeObject(&m, o.Val); err != nil { 399 return err 400 } 401 if err := mapstructure.WeakDecode(m, &config.Meta); err != nil { 402 return err 403 } 404 } 405 } 406 407 // Parse out chroot_env fields. These are in HCL as a list so we need to 408 // iterate over them and merge them. 409 if chrootEnvO := listVal.Filter("chroot_env"); len(chrootEnvO.Items) > 0 { 410 for _, o := range chrootEnvO.Elem().Items { 411 var m map[string]interface{} 412 if err := hcl.DecodeObject(&m, o.Val); err != nil { 413 return err 414 } 415 if err := mapstructure.WeakDecode(m, &config.ChrootEnv); err != nil { 416 return err 417 } 418 } 419 } 420 421 // Parse reserved config 422 if o := listVal.Filter("reserved"); len(o.Items) > 0 { 423 if err := parseReserved(&config.Reserved, o); err != nil { 424 return multierror.Prefix(err, "reserved ->") 425 } 426 } 427 428 *result = &config 429 return nil 430 } 431 432 func parseReserved(result **Resources, list *ast.ObjectList) error { 433 list = list.Elem() 434 if len(list.Items) > 1 { 435 return fmt.Errorf("only one 'reserved' block allowed") 436 } 437 438 // Get our reserved object 439 obj := list.Items[0] 440 441 // Value should be an object 442 var listVal *ast.ObjectList 443 if ot, ok := obj.Val.(*ast.ObjectType); ok { 444 listVal = ot.List 445 } else { 446 return fmt.Errorf("client value: should be an object") 447 } 448 449 // Check for invalid keys 450 valid := []string{ 451 "cpu", 452 "memory", 453 "disk", 454 "iops", 455 "reserved_ports", 456 } 457 if err := checkHCLKeys(listVal, valid); err != nil { 458 return err 459 } 460 461 var m map[string]interface{} 462 if err := hcl.DecodeObject(&m, listVal); err != nil { 463 return err 464 } 465 466 var reserved Resources 467 if err := mapstructure.WeakDecode(m, &reserved); err != nil { 468 return err 469 } 470 if err := reserved.ParseReserved(); err != nil { 471 return err 472 } 473 474 *result = &reserved 475 return nil 476 } 477 478 func parseServer(result **ServerConfig, list *ast.ObjectList) error { 479 list = list.Elem() 480 if len(list.Items) > 1 { 481 return fmt.Errorf("only one 'server' block allowed") 482 } 483 484 // Get our server object 485 obj := list.Items[0] 486 487 // Value should be an object 488 var listVal *ast.ObjectList 489 if ot, ok := obj.Val.(*ast.ObjectType); ok { 490 listVal = ot.List 491 } else { 492 return fmt.Errorf("client value: should be an object") 493 } 494 495 // Check for invalid keys 496 valid := []string{ 497 "enabled", 498 "bootstrap_expect", 499 "data_dir", 500 "protocol_version", 501 "num_schedulers", 502 "enabled_schedulers", 503 "node_gc_threshold", 504 "eval_gc_threshold", 505 "job_gc_threshold", 506 "heartbeat_grace", 507 "start_join", 508 "retry_join", 509 "retry_max", 510 "retry_interval", 511 "rejoin_after_leave", 512 "encrypt", 513 } 514 if err := checkHCLKeys(listVal, valid); err != nil { 515 return err 516 } 517 518 var m map[string]interface{} 519 if err := hcl.DecodeObject(&m, listVal); err != nil { 520 return err 521 } 522 523 var config ServerConfig 524 if err := mapstructure.WeakDecode(m, &config); err != nil { 525 return err 526 } 527 528 *result = &config 529 return nil 530 } 531 532 func parseTelemetry(result **Telemetry, list *ast.ObjectList) error { 533 list = list.Elem() 534 if len(list.Items) > 1 { 535 return fmt.Errorf("only one 'telemetry' block allowed") 536 } 537 538 // Get our telemetry object 539 listVal := list.Items[0].Val 540 541 // Check for invalid keys 542 valid := []string{ 543 "statsite_address", 544 "statsd_address", 545 "disable_hostname", 546 "use_node_name", 547 "collection_interval", 548 "publish_allocation_metrics", 549 "publish_node_metrics", 550 "datadog_address", 551 "circonus_api_token", 552 "circonus_api_app", 553 "circonus_api_url", 554 "circonus_submission_interval", 555 "circonus_submission_url", 556 "circonus_check_id", 557 "circonus_check_force_metric_activation", 558 "circonus_check_instance_id", 559 "circonus_check_search_tag", 560 "circonus_check_display_name", 561 "circonus_check_tags", 562 "circonus_broker_id", 563 "circonus_broker_select_tag", 564 } 565 if err := checkHCLKeys(listVal, valid); err != nil { 566 return err 567 } 568 569 var m map[string]interface{} 570 if err := hcl.DecodeObject(&m, listVal); err != nil { 571 return err 572 } 573 574 var telemetry Telemetry 575 if err := mapstructure.WeakDecode(m, &telemetry); err != nil { 576 return err 577 } 578 if telemetry.CollectionInterval != "" { 579 if dur, err := time.ParseDuration(telemetry.CollectionInterval); err != nil { 580 return fmt.Errorf("error parsing value of %q: %v", "collection_interval", err) 581 } else { 582 telemetry.collectionInterval = dur 583 } 584 } 585 *result = &telemetry 586 return nil 587 } 588 589 func parseAtlas(result **AtlasConfig, list *ast.ObjectList) error { 590 list = list.Elem() 591 if len(list.Items) > 1 { 592 return fmt.Errorf("only one 'atlas' block allowed") 593 } 594 595 // Get our atlas object 596 listVal := list.Items[0].Val 597 598 // Check for invalid keys 599 valid := []string{ 600 "infrastructure", 601 "token", 602 "join", 603 "endpoint", 604 } 605 if err := checkHCLKeys(listVal, valid); err != nil { 606 return err 607 } 608 609 var m map[string]interface{} 610 if err := hcl.DecodeObject(&m, listVal); err != nil { 611 return err 612 } 613 614 var atlas AtlasConfig 615 if err := mapstructure.WeakDecode(m, &atlas); err != nil { 616 return err 617 } 618 *result = &atlas 619 return nil 620 } 621 622 func parseConsulConfig(result **config.ConsulConfig, list *ast.ObjectList) error { 623 list = list.Elem() 624 if len(list.Items) > 1 { 625 return fmt.Errorf("only one 'consul' block allowed") 626 } 627 628 // Get our Consul object 629 listVal := list.Items[0].Val 630 631 // Check for invalid keys 632 valid := []string{ 633 "address", 634 "auth", 635 "auto_advertise", 636 "ca_file", 637 "cert_file", 638 "checks_use_advertise", 639 "client_auto_join", 640 "client_service_name", 641 "key_file", 642 "server_auto_join", 643 "server_service_name", 644 "ssl", 645 "timeout", 646 "token", 647 "verify_ssl", 648 } 649 650 if err := checkHCLKeys(listVal, valid); err != nil { 651 return err 652 } 653 654 var m map[string]interface{} 655 if err := hcl.DecodeObject(&m, listVal); err != nil { 656 return err 657 } 658 659 consulConfig := config.DefaultConsulConfig() 660 dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ 661 DecodeHook: mapstructure.StringToTimeDurationHookFunc(), 662 WeaklyTypedInput: true, 663 Result: &consulConfig, 664 }) 665 if err != nil { 666 return err 667 } 668 if err := dec.Decode(m); err != nil { 669 return err 670 } 671 672 *result = consulConfig 673 return nil 674 } 675 676 func parseTLSConfig(result **config.TLSConfig, list *ast.ObjectList) error { 677 list = list.Elem() 678 if len(list.Items) > 1 { 679 return fmt.Errorf("only one 'tls' block allowed") 680 } 681 682 // Get the TLS object 683 listVal := list.Items[0].Val 684 685 valid := []string{ 686 "http", 687 "rpc", 688 "verify_server_hostname", 689 "ca_file", 690 "cert_file", 691 "key_file", 692 "verify_https_client", 693 } 694 695 if err := checkHCLKeys(listVal, valid); err != nil { 696 return err 697 } 698 699 var m map[string]interface{} 700 if err := hcl.DecodeObject(&m, listVal); err != nil { 701 return err 702 } 703 704 var tlsConfig config.TLSConfig 705 if err := mapstructure.WeakDecode(m, &tlsConfig); err != nil { 706 return err 707 } 708 *result = &tlsConfig 709 return nil 710 } 711 712 func parseVaultConfig(result **config.VaultConfig, list *ast.ObjectList) error { 713 list = list.Elem() 714 if len(list.Items) > 1 { 715 return fmt.Errorf("only one 'vault' block allowed") 716 } 717 718 // Get our Vault object 719 listVal := list.Items[0].Val 720 721 // Check for invalid keys 722 valid := []string{ 723 "address", 724 "allow_unauthenticated", 725 "enabled", 726 "task_token_ttl", 727 "ca_file", 728 "ca_path", 729 "cert_file", 730 "create_from_role", 731 "key_file", 732 "tls_server_name", 733 "tls_skip_verify", 734 "token", 735 } 736 737 if err := checkHCLKeys(listVal, valid); err != nil { 738 return err 739 } 740 741 var m map[string]interface{} 742 if err := hcl.DecodeObject(&m, listVal); err != nil { 743 return err 744 } 745 746 vaultConfig := config.DefaultVaultConfig() 747 dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ 748 DecodeHook: mapstructure.StringToTimeDurationHookFunc(), 749 WeaklyTypedInput: true, 750 Result: &vaultConfig, 751 }) 752 if err != nil { 753 return err 754 } 755 if err := dec.Decode(m); err != nil { 756 return err 757 } 758 759 *result = vaultConfig 760 return nil 761 } 762 763 func checkHCLKeys(node ast.Node, valid []string) error { 764 var list *ast.ObjectList 765 switch n := node.(type) { 766 case *ast.ObjectList: 767 list = n 768 case *ast.ObjectType: 769 list = n.List 770 default: 771 return fmt.Errorf("cannot check HCL keys of type %T", n) 772 } 773 774 validMap := make(map[string]struct{}, len(valid)) 775 for _, v := range valid { 776 validMap[v] = struct{}{} 777 } 778 779 var result error 780 for _, item := range list.Items { 781 key := item.Keys[0].Token.Value().(string) 782 if _, ok := validMap[key]; !ok { 783 result = multierror.Append(result, fmt.Errorf( 784 "invalid key: %s", key)) 785 } 786 } 787 788 return result 789 }