github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/doc-generator/parser.go (about)

     1  package main
     2  
     3  import (
     4  	"flag"
     5  	"fmt"
     6  	"reflect"
     7  	"regexp"
     8  	"strings"
     9  	"unicode"
    10  
    11  	"github.com/grafana/dskit/flagext"
    12  	"github.com/pkg/errors"
    13  	"github.com/prometheus/common/model"
    14  	"github.com/weaveworks/common/logging"
    15  )
    16  
    17  var (
    18  	yamlFieldNameParser   = regexp.MustCompile("^[^,]+")
    19  	yamlFieldInlineParser = regexp.MustCompile("^[^,]*,inline$")
    20  )
    21  
    22  type configBlock struct {
    23  	name          string
    24  	desc          string
    25  	entries       []*configEntry
    26  	flagsPrefix   string
    27  	flagsPrefixes []string
    28  }
    29  
    30  func (b *configBlock) Add(entry *configEntry) {
    31  	b.entries = append(b.entries, entry)
    32  }
    33  
    34  type configEntry struct {
    35  	kind     string
    36  	name     string
    37  	required bool
    38  
    39  	// In case the kind is "block"
    40  	block     *configBlock
    41  	blockDesc string
    42  	root      bool
    43  
    44  	// In case the kind is "field"
    45  	fieldFlag    string
    46  	fieldDesc    string
    47  	fieldType    string
    48  	fieldDefault string
    49  }
    50  
    51  type rootBlock struct {
    52  	name       string
    53  	desc       string
    54  	structType reflect.Type
    55  }
    56  
    57  func parseFlags(cfg flagext.Registerer) map[uintptr]*flag.Flag {
    58  	fs := flag.NewFlagSet("", flag.PanicOnError)
    59  	cfg.RegisterFlags(fs)
    60  
    61  	flags := map[uintptr]*flag.Flag{}
    62  	fs.VisitAll(func(f *flag.Flag) {
    63  		// Skip deprecated flags
    64  		if f.Value.String() == "deprecated" {
    65  			return
    66  		}
    67  
    68  		ptr := reflect.ValueOf(f.Value).Pointer()
    69  		flags[ptr] = f
    70  	})
    71  
    72  	return flags
    73  }
    74  
    75  func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Flag) ([]*configBlock, error) {
    76  	blocks := []*configBlock{}
    77  
    78  	// If the input block is nil it means we're generating the doc for the top-level block
    79  	if block == nil {
    80  		block = &configBlock{}
    81  		blocks = append(blocks, block)
    82  	}
    83  
    84  	// The input config is expected to be addressable.
    85  	if reflect.TypeOf(cfg).Kind() != reflect.Ptr {
    86  		t := reflect.TypeOf(cfg)
    87  		return nil, fmt.Errorf("%s is a %s while a %s is expected", t, t.Kind(), reflect.Ptr)
    88  	}
    89  
    90  	// The input config is expected to be a pointer to struct.
    91  	v := reflect.ValueOf(cfg).Elem()
    92  	t := v.Type()
    93  
    94  	if v.Kind() != reflect.Struct {
    95  		return nil, fmt.Errorf("%s is a %s while a %s is expected", v, v.Kind(), reflect.Struct)
    96  	}
    97  
    98  	for i := 0; i < t.NumField(); i++ {
    99  		field := t.Field(i)
   100  		fieldValue := v.FieldByIndex(field.Index)
   101  
   102  		// Skip fields explicitly marked as "hidden" in the doc
   103  		if isFieldHidden(field) {
   104  			continue
   105  		}
   106  
   107  		// Skip fields not exported via yaml (unless they're inline)
   108  		fieldName := getFieldName(field)
   109  		if fieldName == "" && !isFieldInline(field) {
   110  			continue
   111  		}
   112  
   113  		// Skip field types which are non configurable
   114  		if field.Type.Kind() == reflect.Func {
   115  			continue
   116  		}
   117  
   118  		// Skip deprecated fields we're still keeping for backward compatibility
   119  		// reasons (by convention we prefix them by UnusedFlag)
   120  		if strings.HasPrefix(field.Name, "UnusedFlag") {
   121  			continue
   122  		}
   123  
   124  		// Handle custom fields in vendored libs upon which we have no control.
   125  		fieldEntry, err := getCustomFieldEntry(field, fieldValue, flags)
   126  		if err != nil {
   127  			return nil, err
   128  		}
   129  		if fieldEntry != nil {
   130  			block.Add(fieldEntry)
   131  			continue
   132  		}
   133  
   134  		// Recursively re-iterate if it's a struct
   135  		if field.Type.Kind() == reflect.Struct {
   136  			// Check whether the sub-block is a root config block
   137  			rootName, rootDesc, isRoot := isRootBlock(field.Type)
   138  
   139  			// Since we're going to recursively iterate, we need to create a new sub
   140  			// block and pass it to the doc generation function.
   141  			var subBlock *configBlock
   142  
   143  			if !isFieldInline(field) {
   144  				var blockName string
   145  				var blockDesc string
   146  
   147  				if isRoot {
   148  					blockName = rootName
   149  					blockDesc = rootDesc
   150  				} else {
   151  					blockName = fieldName
   152  					blockDesc = getFieldDescription(field, "")
   153  				}
   154  
   155  				subBlock = &configBlock{
   156  					name: blockName,
   157  					desc: blockDesc,
   158  				}
   159  
   160  				block.Add(&configEntry{
   161  					kind:      "block",
   162  					name:      fieldName,
   163  					required:  isFieldRequired(field),
   164  					block:     subBlock,
   165  					blockDesc: blockDesc,
   166  					root:      isRoot,
   167  				})
   168  
   169  				if isRoot {
   170  					blocks = append(blocks, subBlock)
   171  				}
   172  			} else {
   173  				subBlock = block
   174  			}
   175  
   176  			// Recursively generate the doc for the sub-block
   177  			otherBlocks, err := parseConfig(subBlock, fieldValue.Addr().Interface(), flags)
   178  			if err != nil {
   179  				return nil, err
   180  			}
   181  
   182  			blocks = append(blocks, otherBlocks...)
   183  			continue
   184  		}
   185  
   186  		fieldType, err := getFieldType(field.Type)
   187  		if err != nil {
   188  			return nil, errors.Wrapf(err, "config=%s.%s", t.PkgPath(), t.Name())
   189  		}
   190  
   191  		fieldFlag, err := getFieldFlag(field, fieldValue, flags)
   192  		if err != nil {
   193  			return nil, errors.Wrapf(err, "config=%s.%s", t.PkgPath(), t.Name())
   194  		}
   195  		if fieldFlag == nil {
   196  			block.Add(&configEntry{
   197  				kind:      "field",
   198  				name:      fieldName,
   199  				required:  isFieldRequired(field),
   200  				fieldDesc: getFieldDescription(field, ""),
   201  				fieldType: fieldType,
   202  			})
   203  			continue
   204  		}
   205  
   206  		block.Add(&configEntry{
   207  			kind:         "field",
   208  			name:         fieldName,
   209  			required:     isFieldRequired(field),
   210  			fieldFlag:    fieldFlag.Name,
   211  			fieldDesc:    getFieldDescription(field, fieldFlag.Usage),
   212  			fieldType:    fieldType,
   213  			fieldDefault: fieldFlag.DefValue,
   214  		})
   215  	}
   216  
   217  	return blocks, nil
   218  }
   219  
   220  func getFieldName(field reflect.StructField) string {
   221  	name := field.Name
   222  	tag := field.Tag.Get("yaml")
   223  
   224  	// If the tag is not specified, then an exported field can be
   225  	// configured via the field name (lowercase), while an unexported
   226  	// field can't be configured.
   227  	if tag == "" {
   228  		if unicode.IsLower(rune(name[0])) {
   229  			return ""
   230  		}
   231  
   232  		return strings.ToLower(name)
   233  	}
   234  
   235  	// Parse the field name
   236  	fieldName := yamlFieldNameParser.FindString(tag)
   237  	if fieldName == "-" {
   238  		return ""
   239  	}
   240  
   241  	return fieldName
   242  }
   243  
   244  func getFieldType(t reflect.Type) (string, error) {
   245  	// Handle custom data types used in the config
   246  	switch t.String() {
   247  	case "*url.URL":
   248  		return "url", nil
   249  	case "time.Duration":
   250  		return "duration", nil
   251  	case "cortex.moduleName":
   252  		return "string", nil
   253  	case "flagext.StringSliceCSV":
   254  		return "string", nil
   255  	case "flagext.CIDRSliceCSV":
   256  		return "string", nil
   257  	case "[]*relabel.Config":
   258  		return "relabel_config...", nil
   259  	}
   260  
   261  	// Fallback to auto-detection of built-in data types
   262  	switch t.Kind() {
   263  	case reflect.Bool:
   264  		return "boolean", nil
   265  
   266  	case reflect.Int:
   267  		fallthrough
   268  	case reflect.Int8:
   269  		fallthrough
   270  	case reflect.Int16:
   271  		fallthrough
   272  	case reflect.Int32:
   273  		fallthrough
   274  	case reflect.Int64:
   275  		fallthrough
   276  	case reflect.Uint:
   277  		fallthrough
   278  	case reflect.Uint8:
   279  		fallthrough
   280  	case reflect.Uint16:
   281  		fallthrough
   282  	case reflect.Uint32:
   283  		fallthrough
   284  	case reflect.Uint64:
   285  		return "int", nil
   286  
   287  	case reflect.Float32:
   288  		fallthrough
   289  	case reflect.Float64:
   290  		return "float", nil
   291  
   292  	case reflect.String:
   293  		return "string", nil
   294  
   295  	case reflect.Slice:
   296  		// Get the type of elements
   297  		elemType, err := getFieldType(t.Elem())
   298  		if err != nil {
   299  			return "", err
   300  		}
   301  
   302  		return "list of " + elemType, nil
   303  
   304  	case reflect.Map:
   305  		return fmt.Sprintf("map of %s to %s", t.Key(), t.Elem().String()), nil
   306  
   307  	default:
   308  		return "", fmt.Errorf("unsupported data type %s", t.Kind())
   309  	}
   310  }
   311  
   312  func getFieldFlag(field reflect.StructField, fieldValue reflect.Value, flags map[uintptr]*flag.Flag) (*flag.Flag, error) {
   313  	if isAbsentInCLI(field) {
   314  		return nil, nil
   315  	}
   316  	fieldPtr := fieldValue.Addr().Pointer()
   317  	fieldFlag, ok := flags[fieldPtr]
   318  	if !ok {
   319  		return nil, fmt.Errorf("unable to find CLI flag for '%s' config entry", field.Name)
   320  	}
   321  
   322  	return fieldFlag, nil
   323  }
   324  
   325  func getCustomFieldEntry(field reflect.StructField, fieldValue reflect.Value, flags map[uintptr]*flag.Flag) (*configEntry, error) {
   326  	if field.Type == reflect.TypeOf(logging.Level{}) || field.Type == reflect.TypeOf(logging.Format{}) {
   327  		fieldFlag, err := getFieldFlag(field, fieldValue, flags)
   328  		if err != nil {
   329  			return nil, err
   330  		}
   331  
   332  		return &configEntry{
   333  			kind:         "field",
   334  			name:         getFieldName(field),
   335  			required:     isFieldRequired(field),
   336  			fieldFlag:    fieldFlag.Name,
   337  			fieldDesc:    fieldFlag.Usage,
   338  			fieldType:    "string",
   339  			fieldDefault: fieldFlag.DefValue,
   340  		}, nil
   341  	}
   342  	if field.Type == reflect.TypeOf(flagext.URLValue{}) {
   343  		fieldFlag, err := getFieldFlag(field, fieldValue, flags)
   344  		if err != nil {
   345  			return nil, err
   346  		}
   347  
   348  		return &configEntry{
   349  			kind:         "field",
   350  			name:         getFieldName(field),
   351  			required:     isFieldRequired(field),
   352  			fieldFlag:    fieldFlag.Name,
   353  			fieldDesc:    fieldFlag.Usage,
   354  			fieldType:    "url",
   355  			fieldDefault: fieldFlag.DefValue,
   356  		}, nil
   357  	}
   358  	if field.Type == reflect.TypeOf(flagext.Secret{}) {
   359  		fieldFlag, err := getFieldFlag(field, fieldValue, flags)
   360  		if err != nil {
   361  			return nil, err
   362  		}
   363  
   364  		return &configEntry{
   365  			kind:         "field",
   366  			name:         getFieldName(field),
   367  			required:     isFieldRequired(field),
   368  			fieldFlag:    fieldFlag.Name,
   369  			fieldDesc:    fieldFlag.Usage,
   370  			fieldType:    "string",
   371  			fieldDefault: fieldFlag.DefValue,
   372  		}, nil
   373  	}
   374  	if field.Type == reflect.TypeOf(model.Duration(0)) {
   375  		fieldFlag, err := getFieldFlag(field, fieldValue, flags)
   376  		if err != nil {
   377  			return nil, err
   378  		}
   379  
   380  		return &configEntry{
   381  			kind:         "field",
   382  			name:         getFieldName(field),
   383  			required:     isFieldRequired(field),
   384  			fieldFlag:    fieldFlag.Name,
   385  			fieldDesc:    fieldFlag.Usage,
   386  			fieldType:    "duration",
   387  			fieldDefault: fieldFlag.DefValue,
   388  		}, nil
   389  	}
   390  	if field.Type == reflect.TypeOf(flagext.Time{}) {
   391  		fieldFlag, err := getFieldFlag(field, fieldValue, flags)
   392  		if err != nil {
   393  			return nil, err
   394  		}
   395  
   396  		return &configEntry{
   397  			kind:         "field",
   398  			name:         getFieldName(field),
   399  			required:     isFieldRequired(field),
   400  			fieldFlag:    fieldFlag.Name,
   401  			fieldDesc:    fieldFlag.Usage,
   402  			fieldType:    "time",
   403  			fieldDefault: fieldFlag.DefValue,
   404  		}, nil
   405  	}
   406  
   407  	return nil, nil
   408  }
   409  
   410  func isFieldHidden(f reflect.StructField) bool {
   411  	return getDocTagFlag(f, "hidden")
   412  }
   413  
   414  func isAbsentInCLI(f reflect.StructField) bool {
   415  	return getDocTagFlag(f, "nocli")
   416  }
   417  
   418  func isFieldRequired(f reflect.StructField) bool {
   419  	return getDocTagFlag(f, "required")
   420  }
   421  
   422  func isFieldInline(f reflect.StructField) bool {
   423  	return yamlFieldInlineParser.MatchString(f.Tag.Get("yaml"))
   424  }
   425  
   426  func getFieldDescription(f reflect.StructField, fallback string) string {
   427  	if desc := getDocTagValue(f, "description"); desc != "" {
   428  		return desc
   429  	}
   430  
   431  	return fallback
   432  }
   433  
   434  func isRootBlock(t reflect.Type) (string, string, bool) {
   435  	for _, rootBlock := range rootBlocks {
   436  		if t == rootBlock.structType {
   437  			return rootBlock.name, rootBlock.desc, true
   438  		}
   439  	}
   440  
   441  	return "", "", false
   442  }
   443  
   444  func getDocTagFlag(f reflect.StructField, name string) bool {
   445  	cfg := parseDocTag(f)
   446  	_, ok := cfg[name]
   447  	return ok
   448  }
   449  
   450  func getDocTagValue(f reflect.StructField, name string) string {
   451  	cfg := parseDocTag(f)
   452  	return cfg[name]
   453  }
   454  
   455  func parseDocTag(f reflect.StructField) map[string]string {
   456  	cfg := map[string]string{}
   457  	tag := f.Tag.Get("doc")
   458  
   459  	if tag == "" {
   460  		return cfg
   461  	}
   462  
   463  	for _, entry := range strings.Split(tag, "|") {
   464  		parts := strings.SplitN(entry, "=", 2)
   465  
   466  		switch len(parts) {
   467  		case 1:
   468  			cfg[parts[0]] = ""
   469  		case 2:
   470  			cfg[parts[0]] = parts[1]
   471  		}
   472  	}
   473  
   474  	return cfg
   475  }