github.com/jhump/protoreflect@v1.16.0/desc/protoparse/parser.go (about)

     1  package protoparse
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"io/ioutil"
     8  	"os"
     9  	"path/filepath"
    10  	"sort"
    11  	"strings"
    12  
    13  	"github.com/bufbuild/protocompile"
    14  	ast2 "github.com/bufbuild/protocompile/ast"
    15  	"github.com/bufbuild/protocompile/linker"
    16  	"github.com/bufbuild/protocompile/options"
    17  	"github.com/bufbuild/protocompile/parser"
    18  	"github.com/bufbuild/protocompile/protoutil"
    19  	"github.com/bufbuild/protocompile/reporter"
    20  	"github.com/bufbuild/protocompile/sourceinfo"
    21  	"github.com/bufbuild/protocompile/walk"
    22  	"google.golang.org/protobuf/proto"
    23  	"google.golang.org/protobuf/reflect/protoreflect"
    24  	"google.golang.org/protobuf/types/descriptorpb"
    25  
    26  	"github.com/jhump/protoreflect/desc"
    27  	"github.com/jhump/protoreflect/desc/internal"
    28  	"github.com/jhump/protoreflect/desc/protoparse/ast"
    29  )
    30  
    31  // FileAccessor is an abstraction for opening proto source files. It takes the
    32  // name of the file to open and returns either the input reader or an error.
    33  type FileAccessor func(filename string) (io.ReadCloser, error)
    34  
    35  // FileContentsFromMap returns a FileAccessor that uses the given map of file
    36  // contents. This allows proto source files to be constructed in memory and
    37  // easily supplied to a parser. The map keys are the paths to the proto source
    38  // files, and the values are the actual proto source contents.
    39  func FileContentsFromMap(files map[string]string) FileAccessor {
    40  	return func(filename string) (io.ReadCloser, error) {
    41  		contents, ok := files[filename]
    42  		if !ok {
    43  			// Try changing path separators since user-provided
    44  			// map may use different separators.
    45  			contents, ok = files[filepath.ToSlash(filename)]
    46  			if !ok {
    47  				return nil, os.ErrNotExist
    48  			}
    49  		}
    50  		return ioutil.NopCloser(strings.NewReader(contents)), nil
    51  	}
    52  }
    53  
    54  // Parser parses proto source into descriptors.
    55  type Parser struct {
    56  	// The paths used to search for dependencies that are referenced in import
    57  	// statements in proto source files. If no import paths are provided then
    58  	// "." (current directory) is assumed to be the only import path.
    59  	//
    60  	// This setting is only used during ParseFiles operations. Since calls to
    61  	// ParseFilesButDoNotLink do not link, there is no need to load and parse
    62  	// dependencies.
    63  	ImportPaths []string
    64  
    65  	// If true, the supplied file names/paths need not necessarily match how the
    66  	// files are referenced in import statements. The parser will attempt to
    67  	// match import statements to supplied paths, "guessing" the import paths
    68  	// for the files. Note that this inference is not perfect and link errors
    69  	// could result. It works best when all proto files are organized such that
    70  	// a single import path can be inferred (e.g. all files under a single tree
    71  	// with import statements all being relative to the root of this tree).
    72  	InferImportPaths bool
    73  
    74  	// LookupImport is a function that accepts a filename and
    75  	// returns a file descriptor, which will be consulted when resolving imports.
    76  	// This allows a compiled Go proto in another Go module to be referenced
    77  	// in the proto(s) being parsed.
    78  	//
    79  	// In the event of a filename collision, Accessor is consulted first,
    80  	// then LookupImport is consulted, and finally the well-known protos
    81  	// are used.
    82  	//
    83  	// For example, in order to automatically look up compiled Go protos that
    84  	// have been imported and be able to use them as imports, set this to
    85  	// desc.LoadFileDescriptor.
    86  	LookupImport func(string) (*desc.FileDescriptor, error)
    87  
    88  	// LookupImportProto has the same functionality as LookupImport, however it returns
    89  	// a FileDescriptorProto instead of a FileDescriptor.
    90  	LookupImportProto func(string) (*descriptorpb.FileDescriptorProto, error)
    91  
    92  	// Used to create a reader for a given filename, when loading proto source
    93  	// file contents. If unset, os.Open is used. If ImportPaths is also empty
    94  	// then relative paths are will be relative to the process's current working
    95  	// directory.
    96  	Accessor FileAccessor
    97  
    98  	// If true, the resulting file descriptors will retain source code info,
    99  	// that maps elements to their location in the source files as well as
   100  	// includes comments found during parsing (and attributed to elements of
   101  	// the source file).
   102  	IncludeSourceCodeInfo bool
   103  
   104  	// If true, the results from ParseFilesButDoNotLink will be passed through
   105  	// some additional validations. But only constraints that do not require
   106  	// linking can be checked. These include proto2 vs. proto3 language features,
   107  	// looking for incorrect usage of reserved names or tags, and ensuring that
   108  	// fields have unique tags and that enum values have unique numbers (unless
   109  	// the enum allows aliases).
   110  	ValidateUnlinkedFiles bool
   111  
   112  	// If true, the results from ParseFilesButDoNotLink will have options
   113  	// interpreted. Any uninterpretable options (including any custom options or
   114  	// options that refer to message and enum types, which can only be
   115  	// interpreted after linking) will be left in uninterpreted_options. Also,
   116  	// the "default" pseudo-option for fields can only be interpreted for scalar
   117  	// fields, excluding enums. (Interpreting default values for enum fields
   118  	// requires resolving enum names, which requires linking.)
   119  	InterpretOptionsInUnlinkedFiles bool
   120  
   121  	// A custom reporter of syntax and link errors. If not specified, the
   122  	// default reporter just returns the reported error, which causes parsing
   123  	// to abort after encountering a single error.
   124  	//
   125  	// The reporter is not invoked for system or I/O errors, only for syntax and
   126  	// link errors.
   127  	ErrorReporter ErrorReporter
   128  
   129  	// A custom reporter of warnings. If not specified, warning messages are ignored.
   130  	WarningReporter WarningReporter
   131  }
   132  
   133  // ParseFiles parses the named files into descriptors. The returned slice has
   134  // the same number of entries as the give filenames, in the same order. So the
   135  // first returned descriptor corresponds to the first given name, and so on.
   136  //
   137  // All dependencies for all specified files (including transitive dependencies)
   138  // must be accessible via the parser's Accessor or a link error will occur. The
   139  // exception to this rule is that files can import standard Google-provided
   140  // files -- e.g. google/protobuf/*.proto -- without needing to supply sources
   141  // for these files. Like protoc, this parser has a built-in version of these
   142  // files it can use if they aren't explicitly supplied.
   143  //
   144  // If the Parser has no ErrorReporter set and a syntax or link error occurs,
   145  // parsing will abort with the first such error encountered. If there is an
   146  // ErrorReporter configured and it returns non-nil, parsing will abort with the
   147  // error it returns. If syntax or link errors are encountered but the configured
   148  // ErrorReporter always returns nil, the parse fails with ErrInvalidSource.
   149  func (p Parser) ParseFiles(filenames ...string) ([]*desc.FileDescriptor, error) {
   150  	srcInfoMode := protocompile.SourceInfoNone
   151  	if p.IncludeSourceCodeInfo {
   152  		srcInfoMode = protocompile.SourceInfoExtraComments
   153  	}
   154  	rep := newReporter(p.ErrorReporter, p.WarningReporter)
   155  	res, srcSpanAddr := p.getResolver(filenames)
   156  
   157  	if p.InferImportPaths {
   158  		// we must first compile everything to protos
   159  		results, err := parseToProtosRecursive(res, filenames, reporter.NewHandler(rep), srcSpanAddr)
   160  		if err != nil {
   161  			return nil, err
   162  		}
   163  		// then we can infer import paths
   164  		var rewritten map[string]string
   165  		results, rewritten = fixupFilenames(results)
   166  		if len(rewritten) > 0 {
   167  			for i := range filenames {
   168  				if replace, ok := rewritten[filenames[i]]; ok {
   169  					filenames[i] = replace
   170  				}
   171  			}
   172  		}
   173  		resolverFromResults := protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) {
   174  			res, ok := results[path]
   175  			if !ok {
   176  				return protocompile.SearchResult{}, os.ErrNotExist
   177  			}
   178  			return protocompile.SearchResult{ParseResult: noCloneParseResult{res}}, nil
   179  		})
   180  		res = protocompile.CompositeResolver{resolverFromResults, res}
   181  	}
   182  
   183  	c := protocompile.Compiler{
   184  		Resolver:       res,
   185  		MaxParallelism: 1,
   186  		SourceInfoMode: srcInfoMode,
   187  		Reporter:       rep,
   188  	}
   189  	results, err := c.Compile(context.Background(), filenames...)
   190  	if err != nil {
   191  		return nil, err
   192  	}
   193  
   194  	fds := make([]protoreflect.FileDescriptor, len(results))
   195  	alreadySeen := make(map[string]struct{}, len(results))
   196  	for i, res := range results {
   197  		removeDynamicExtensions(res, alreadySeen)
   198  		fds[i] = res
   199  	}
   200  	return desc.WrapFiles(fds)
   201  }
   202  
   203  type noCloneParseResult struct {
   204  	parser.Result
   205  }
   206  
   207  func (r noCloneParseResult) Clone() parser.Result {
   208  	// protocompile will clone parser.Result to make sure it can't be shared
   209  	// with other compilation operations (which would not be thread-safe).
   210  	// However, this parse result cannot be shared with another compile
   211  	// operation. That means the clone is unnecessary; so we skip it, to avoid
   212  	// the associated performance costs.
   213  	return r.Result
   214  }
   215  
   216  // ParseFilesButDoNotLink parses the named files into descriptor protos. The
   217  // results are just protos, not fully-linked descriptors. It is possible that
   218  // descriptors are invalid and still be returned in parsed form without error
   219  // due to the fact that the linking step is skipped (and thus many validation
   220  // steps omitted).
   221  //
   222  // There are a few side effects to not linking the descriptors:
   223  //  1. No options will be interpreted. Options can refer to extensions or have
   224  //     message and enum types. Without linking, these extension and type
   225  //     references are not resolved, so the options may not be interpretable.
   226  //     So all options will appear in UninterpretedOption fields of the various
   227  //     descriptor options messages.
   228  //  2. Type references will not be resolved. This means that the actual type
   229  //     names in the descriptors may be unqualified and even relative to the
   230  //     scope in which the type reference appears. This goes for fields that
   231  //     have message and enum types. It also applies to methods and their
   232  //     references to request and response message types.
   233  //  3. Type references are not known. For non-scalar fields, until the type
   234  //     name is resolved (during linking), it is not known whether the type
   235  //     refers to a message or an enum. So all fields with such type references
   236  //     will not have their Type set, only the TypeName.
   237  //
   238  // This method will still validate the syntax of parsed files. If the parser's
   239  // ValidateUnlinkedFiles field is true, additional checks, beyond syntax will
   240  // also be performed.
   241  //
   242  // If the Parser has no ErrorReporter set and a syntax error occurs, parsing
   243  // will abort with the first such error encountered. If there is an
   244  // ErrorReporter configured and it returns non-nil, parsing will abort with the
   245  // error it returns. If syntax errors are encountered but the configured
   246  // ErrorReporter always returns nil, the parse fails with ErrInvalidSource.
   247  func (p Parser) ParseFilesButDoNotLink(filenames ...string) ([]*descriptorpb.FileDescriptorProto, error) {
   248  	rep := newReporter(p.ErrorReporter, p.WarningReporter)
   249  	p.ImportPaths = nil // not used for this "do not link" operation.
   250  	res, _ := p.getResolver(filenames)
   251  	results, err := parseToProtos(res, filenames, reporter.NewHandler(rep), p.ValidateUnlinkedFiles)
   252  	if err != nil {
   253  		return nil, err
   254  	}
   255  
   256  	if p.InferImportPaths {
   257  		resultsMap := make(map[string]parser.Result, len(results))
   258  		for _, res := range results {
   259  			resultsMap[res.FileDescriptorProto().GetName()] = res
   260  		}
   261  		var rewritten map[string]string
   262  		resultsMap, rewritten = fixupFilenames(resultsMap)
   263  		if len(rewritten) > 0 {
   264  			for i := range filenames {
   265  				if replace, ok := rewritten[filenames[i]]; ok {
   266  					filenames[i] = replace
   267  				}
   268  			}
   269  		}
   270  		for i := range filenames {
   271  			results[i] = resultsMap[filenames[i]]
   272  		}
   273  	}
   274  
   275  	protos := make([]*descriptorpb.FileDescriptorProto, len(results))
   276  	for i, res := range results {
   277  		protos[i] = res.FileDescriptorProto()
   278  		var optsIndex sourceinfo.OptionIndex
   279  		if p.InterpretOptionsInUnlinkedFiles {
   280  			var err error
   281  			optsIndex, err = options.InterpretUnlinkedOptions(res)
   282  			if err != nil {
   283  				return nil, err
   284  			}
   285  			removeDynamicExtensionsFromProto(protos[i])
   286  		}
   287  		if p.IncludeSourceCodeInfo {
   288  			protos[i].SourceCodeInfo = sourceinfo.GenerateSourceInfo(res.AST(), optsIndex, sourceinfo.WithExtraComments())
   289  		}
   290  	}
   291  
   292  	return protos, nil
   293  }
   294  
   295  // ParseToAST parses the named files into ASTs, or Abstract Syntax Trees. This
   296  // is for consumers of proto files that don't care about compiling the files to
   297  // descriptors, but care deeply about a non-lossy structured representation of
   298  // the source (since descriptors are lossy). This includes formatting tools and
   299  // possibly linters, too.
   300  //
   301  // If the requested filenames include standard imports (such as
   302  // "google/protobuf/empty.proto") and no source is provided, the corresponding
   303  // AST in the returned slice will be nil. These standard imports are only
   304  // available for use as descriptors; no source is available unless it is
   305  // provided by the configured Accessor.
   306  //
   307  // If the Parser has no ErrorReporter set and a syntax error occurs, parsing
   308  // will abort with the first such error encountered. If there is an
   309  // ErrorReporter configured and it returns non-nil, parsing will abort with the
   310  // error it returns. If syntax errors are encountered but the configured
   311  // ErrorReporter always returns nil, the parse fails with ErrInvalidSource.
   312  func (p Parser) ParseToAST(filenames ...string) ([]*ast.FileNode, error) {
   313  	rep := newReporter(p.ErrorReporter, p.WarningReporter)
   314  	res, _ := p.getResolver(filenames)
   315  	asts, _, err := parseToASTs(res, filenames, reporter.NewHandler(rep))
   316  	if err != nil {
   317  		return nil, err
   318  	}
   319  	results := make([]*ast.FileNode, len(asts))
   320  	for i := range asts {
   321  		if asts[i] == nil {
   322  			// should not be possible but...
   323  			return nil, fmt.Errorf("resolver did not produce source for %v", filenames[i])
   324  		}
   325  		results[i] = convertAST(asts[i])
   326  	}
   327  	return results, nil
   328  }
   329  
   330  func parseToAST(res protocompile.Resolver, filename string, rep *reporter.Handler) (*ast2.FileNode, parser.Result, error) {
   331  	searchResult, err := res.FindFileByPath(filename)
   332  	if err != nil {
   333  		_ = rep.HandleError(err)
   334  		return nil, nil, rep.Error()
   335  	}
   336  	switch {
   337  	case searchResult.ParseResult != nil:
   338  		return nil, searchResult.ParseResult, nil
   339  	case searchResult.Proto != nil:
   340  		return nil, parser.ResultWithoutAST(searchResult.Proto), nil
   341  	case searchResult.Desc != nil:
   342  		return nil, parser.ResultWithoutAST(protoutil.ProtoFromFileDescriptor(searchResult.Desc)), nil
   343  	case searchResult.AST != nil:
   344  		return searchResult.AST, nil, nil
   345  	case searchResult.Source != nil:
   346  		astRoot, err := parser.Parse(filename, searchResult.Source, rep)
   347  		return astRoot, nil, err
   348  	default:
   349  		_ = rep.HandleError(fmt.Errorf("resolver did not produce a result for %v", filename))
   350  		return nil, nil, rep.Error()
   351  	}
   352  }
   353  
   354  func parseToASTs(res protocompile.Resolver, filenames []string, rep *reporter.Handler) ([]*ast2.FileNode, []parser.Result, error) {
   355  	asts := make([]*ast2.FileNode, len(filenames))
   356  	results := make([]parser.Result, len(filenames))
   357  	for i := range filenames {
   358  		asts[i], results[i], _ = parseToAST(res, filenames[i], rep)
   359  		if rep.ReporterError() != nil {
   360  			break
   361  		}
   362  	}
   363  	return asts, results, rep.Error()
   364  }
   365  
   366  func parseToProtos(res protocompile.Resolver, filenames []string, rep *reporter.Handler, validate bool) ([]parser.Result, error) {
   367  	asts, results, err := parseToASTs(res, filenames, rep)
   368  	if err != nil {
   369  		return nil, err
   370  	}
   371  	for i := range results {
   372  		if results[i] != nil {
   373  			continue
   374  		}
   375  		var err error
   376  		results[i], err = parser.ResultFromAST(asts[i], validate, rep)
   377  		if err != nil {
   378  			return nil, err
   379  		}
   380  	}
   381  	return results, nil
   382  }
   383  
   384  func parseToProtosRecursive(res protocompile.Resolver, filenames []string, rep *reporter.Handler, srcSpanAddr *ast2.SourceSpan) (map[string]parser.Result, error) {
   385  	results := make(map[string]parser.Result, len(filenames))
   386  	for _, filename := range filenames {
   387  		if err := parseToProtoRecursive(res, filename, rep, srcSpanAddr, results); err != nil {
   388  			return results, err
   389  		}
   390  	}
   391  	return results, rep.Error()
   392  }
   393  
   394  func parseToProtoRecursive(res protocompile.Resolver, filename string, rep *reporter.Handler, srcSpanAddr *ast2.SourceSpan, results map[string]parser.Result) error {
   395  	if _, ok := results[filename]; ok {
   396  		// already processed this one
   397  		return nil
   398  	}
   399  	results[filename] = nil // placeholder entry
   400  
   401  	astRoot, parseResult, err := parseToAST(res, filename, rep)
   402  	if err != nil {
   403  		return err
   404  	}
   405  	if parseResult == nil {
   406  		parseResult, err = parser.ResultFromAST(astRoot, true, rep)
   407  		if err != nil {
   408  			return err
   409  		}
   410  	}
   411  	results[filename] = parseResult
   412  
   413  	if astRoot != nil {
   414  		// We have an AST, so we use it to recursively examine imports.
   415  		for _, decl := range astRoot.Decls {
   416  			imp, ok := decl.(*ast2.ImportNode)
   417  			if !ok {
   418  				continue
   419  			}
   420  			err := func() error {
   421  				orig := *srcSpanAddr
   422  				*srcSpanAddr = astRoot.NodeInfo(imp.Name)
   423  				defer func() {
   424  					*srcSpanAddr = orig
   425  				}()
   426  
   427  				return parseToProtoRecursive(res, imp.Name.AsString(), rep, srcSpanAddr, results)
   428  			}()
   429  			if err != nil {
   430  				return err
   431  			}
   432  		}
   433  		return nil
   434  	}
   435  
   436  	// Without an AST, we must recursively examine the proto. This makes it harder
   437  	// (but not necessarily impossible) to get the source location of the import.
   438  	fd := parseResult.FileDescriptorProto()
   439  	for i, dep := range fd.Dependency {
   440  		path := []int32{internal.File_dependencyTag, int32(i)}
   441  		err := func() error {
   442  			orig := *srcSpanAddr
   443  			found := false
   444  			for _, loc := range fd.GetSourceCodeInfo().GetLocation() {
   445  				if pathsEqual(loc.Path, path) {
   446  					start := SourcePos{
   447  						Filename: dep,
   448  						Line:     int(loc.Span[0]),
   449  						Col:      int(loc.Span[1]),
   450  					}
   451  					var end SourcePos
   452  					if len(loc.Span) > 3 {
   453  						end = SourcePos{
   454  							Filename: dep,
   455  							Line:     int(loc.Span[2]),
   456  							Col:      int(loc.Span[3]),
   457  						}
   458  					} else {
   459  						end = SourcePos{
   460  							Filename: dep,
   461  							Line:     int(loc.Span[0]),
   462  							Col:      int(loc.Span[2]),
   463  						}
   464  					}
   465  					*srcSpanAddr = ast2.NewSourceSpan(start, end)
   466  					found = true
   467  					break
   468  				}
   469  			}
   470  			if !found {
   471  				*srcSpanAddr = ast2.UnknownSpan(dep)
   472  			}
   473  			defer func() {
   474  				*srcSpanAddr = orig
   475  			}()
   476  
   477  			return parseToProtoRecursive(res, dep, rep, srcSpanAddr, results)
   478  		}()
   479  		if err != nil {
   480  			return err
   481  		}
   482  	}
   483  	return nil
   484  }
   485  
   486  func pathsEqual(a, b []int32) bool {
   487  	if len(a) != len(b) {
   488  		return false
   489  	}
   490  	for i := range a {
   491  		if a[i] != b[i] {
   492  			return false
   493  		}
   494  	}
   495  	return true
   496  }
   497  
   498  func newReporter(errRep ErrorReporter, warnRep WarningReporter) reporter.Reporter {
   499  	if errRep != nil {
   500  		delegate := errRep
   501  		errRep = func(err ErrorWithPos) error {
   502  			if _, ok := err.(ErrorWithSourcePos); !ok {
   503  				err = toErrorWithSourcePos(err)
   504  			}
   505  			return delegate(err)
   506  		}
   507  	}
   508  	if warnRep != nil {
   509  		delegate := warnRep
   510  		warnRep = func(err ErrorWithPos) {
   511  			if _, ok := err.(ErrorWithSourcePos); !ok {
   512  				err = toErrorWithSourcePos(err)
   513  			}
   514  			delegate(err)
   515  		}
   516  	}
   517  	return reporter.NewReporter(errRep, warnRep)
   518  }
   519  
   520  func (p Parser) getResolver(filenames []string) (protocompile.Resolver, *ast2.SourceSpan) {
   521  	var srcSpan ast2.SourceSpan
   522  	accessor := p.Accessor
   523  	if accessor == nil {
   524  		accessor = func(name string) (io.ReadCloser, error) {
   525  			return os.Open(name)
   526  		}
   527  	}
   528  	sourceResolver := &protocompile.SourceResolver{
   529  		Accessor: func(filename string) (io.ReadCloser, error) {
   530  			in, err := accessor(filename)
   531  			if err != nil {
   532  				if !strings.Contains(err.Error(), filename) {
   533  					// errors that don't include the filename that failed are no bueno
   534  					err = errorWithFilename{filename: filename, underlying: err}
   535  				}
   536  				if srcSpan != nil {
   537  					err = reporter.Error(srcSpan, err)
   538  				}
   539  			}
   540  			return in, err
   541  		},
   542  		ImportPaths: p.ImportPaths,
   543  	}
   544  	var importResolver protocompile.CompositeResolver
   545  	if p.LookupImport != nil {
   546  		importResolver = append(importResolver, protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) {
   547  			fd, err := p.LookupImport(path)
   548  			if err != nil {
   549  				return protocompile.SearchResult{}, err
   550  			}
   551  			return protocompile.SearchResult{Desc: fd.UnwrapFile()}, nil
   552  		}))
   553  	}
   554  	if p.LookupImportProto != nil {
   555  		importResolver = append(importResolver, protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) {
   556  			fd, err := p.LookupImportProto(path)
   557  			if err != nil {
   558  				return protocompile.SearchResult{}, err
   559  			}
   560  			return protocompile.SearchResult{Proto: fd}, nil
   561  		}))
   562  	}
   563  	backupResolver := protocompile.WithStandardImports(importResolver)
   564  	return protocompile.CompositeResolver{
   565  		sourceResolver,
   566  		protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) {
   567  			return backupResolver.FindFileByPath(path)
   568  		}),
   569  	}, &srcSpan
   570  }
   571  
   572  func fixupFilenames(protos map[string]parser.Result) (revisedProtos map[string]parser.Result, rewrittenPaths map[string]string) {
   573  	// In the event that the given filenames (keys in the supplied map) do not
   574  	// match the actual paths used in 'import' statements in the files, we try
   575  	// to revise names in the protos so that they will match and be linkable.
   576  	revisedProtos = make(map[string]parser.Result, len(protos))
   577  	rewrittenPaths = make(map[string]string, len(protos))
   578  
   579  	protoPaths := map[string]struct{}{}
   580  	// TODO: this is O(n^2) but could likely be O(n) with a clever data structure (prefix tree that is indexed backwards?)
   581  	importCandidates := map[string]map[string]struct{}{}
   582  	candidatesAvailable := map[string]struct{}{}
   583  	for name := range protos {
   584  		candidatesAvailable[name] = struct{}{}
   585  		for _, f := range protos {
   586  			for _, imp := range f.FileDescriptorProto().Dependency {
   587  				if strings.HasSuffix(name, imp) || strings.HasSuffix(imp, name) {
   588  					candidates := importCandidates[imp]
   589  					if candidates == nil {
   590  						candidates = map[string]struct{}{}
   591  						importCandidates[imp] = candidates
   592  					}
   593  					candidates[name] = struct{}{}
   594  				}
   595  			}
   596  		}
   597  	}
   598  	for imp, candidates := range importCandidates {
   599  		// if we found multiple possible candidates, use the one that is an exact match
   600  		// if it exists, and otherwise, guess that it's the shortest path (fewest elements)
   601  		var best string
   602  		for c := range candidates {
   603  			if _, ok := candidatesAvailable[c]; !ok {
   604  				// already used this candidate and re-written its filename accordingly
   605  				continue
   606  			}
   607  			if c == imp {
   608  				// exact match!
   609  				best = c
   610  				break
   611  			}
   612  			if best == "" {
   613  				best = c
   614  			} else {
   615  				// NB: We can't actually tell which file is supposed to match
   616  				// this import. So we prefer the longest name. On a tie, we
   617  				// choose the lexically earliest match.
   618  				minLen := strings.Count(best, string(filepath.Separator))
   619  				cLen := strings.Count(c, string(filepath.Separator))
   620  				if cLen > minLen || (cLen == minLen && c < best) {
   621  					best = c
   622  				}
   623  			}
   624  		}
   625  		if best != "" {
   626  			if len(best) > len(imp) {
   627  				prefix := best[:len(best)-len(imp)]
   628  				protoPaths[prefix] = struct{}{}
   629  			}
   630  			f := protos[best]
   631  			f.FileDescriptorProto().Name = proto.String(imp)
   632  			revisedProtos[imp] = f
   633  			rewrittenPaths[best] = imp
   634  			delete(candidatesAvailable, best)
   635  
   636  			// If other candidates are actually references to the same file, remove them.
   637  			for c := range candidates {
   638  				if _, ok := candidatesAvailable[c]; !ok {
   639  					// already used this candidate and re-written its filename accordingly
   640  					continue
   641  				}
   642  				possibleDup := protos[c]
   643  				prevName := possibleDup.FileDescriptorProto().Name
   644  				possibleDup.FileDescriptorProto().Name = proto.String(imp)
   645  				if !proto.Equal(f.FileDescriptorProto(), protos[c].FileDescriptorProto()) {
   646  					// not equal: restore name and look at next one
   647  					possibleDup.FileDescriptorProto().Name = prevName
   648  					continue
   649  				}
   650  				// This file used a different name but was actually the same file. So
   651  				// we prune it from the set.
   652  				rewrittenPaths[c] = imp
   653  				delete(candidatesAvailable, c)
   654  				if len(c) > len(imp) {
   655  					prefix := c[:len(c)-len(imp)]
   656  					protoPaths[prefix] = struct{}{}
   657  				}
   658  			}
   659  		}
   660  	}
   661  
   662  	if len(candidatesAvailable) == 0 {
   663  		return revisedProtos, rewrittenPaths
   664  	}
   665  
   666  	if len(protoPaths) == 0 {
   667  		for c := range candidatesAvailable {
   668  			revisedProtos[c] = protos[c]
   669  		}
   670  		return revisedProtos, rewrittenPaths
   671  	}
   672  
   673  	// Any remaining candidates are entry-points (not imported by others), so
   674  	// the best bet to "fixing" their file name is to see if they're in one of
   675  	// the proto paths we found, and if so strip that prefix.
   676  	protoPathStrs := make([]string, len(protoPaths))
   677  	i := 0
   678  	for p := range protoPaths {
   679  		protoPathStrs[i] = p
   680  		i++
   681  	}
   682  	sort.Strings(protoPathStrs)
   683  	// we look at paths in reverse order, so we'll use a longer proto path if
   684  	// there is more than one match
   685  	for c := range candidatesAvailable {
   686  		var imp string
   687  		for i := len(protoPathStrs) - 1; i >= 0; i-- {
   688  			p := protoPathStrs[i]
   689  			if strings.HasPrefix(c, p) {
   690  				imp = c[len(p):]
   691  				break
   692  			}
   693  		}
   694  		if imp != "" {
   695  			f := protos[c]
   696  			f.FileDescriptorProto().Name = proto.String(imp)
   697  			f.FileNode()
   698  			revisedProtos[imp] = f
   699  			rewrittenPaths[c] = imp
   700  		} else {
   701  			revisedProtos[c] = protos[c]
   702  		}
   703  	}
   704  
   705  	return revisedProtos, rewrittenPaths
   706  }
   707  
   708  func removeDynamicExtensions(fd protoreflect.FileDescriptor, alreadySeen map[string]struct{}) {
   709  	if _, ok := alreadySeen[fd.Path()]; ok {
   710  		// already processed
   711  		return
   712  	}
   713  	alreadySeen[fd.Path()] = struct{}{}
   714  	res, ok := fd.(linker.Result)
   715  	if ok {
   716  		removeDynamicExtensionsFromProto(res.FileDescriptorProto())
   717  	}
   718  	// also remove extensions from dependencies
   719  	for i, length := 0, fd.Imports().Len(); i < length; i++ {
   720  		removeDynamicExtensions(fd.Imports().Get(i).FileDescriptor, alreadySeen)
   721  	}
   722  }
   723  
   724  func removeDynamicExtensionsFromProto(fd *descriptorpb.FileDescriptorProto) {
   725  	// protocompile returns descriptors with dynamic extension fields for custom options.
   726  	// But protoparse only used known custom options and everything else defined in the
   727  	// sources would be stored as unrecognized fields. So to bridge the difference in
   728  	// behavior, we need to remove custom options from the given file and add them back
   729  	// via serializing-then-de-serializing them back into the options messages. That way,
   730  	// statically known options will be properly typed and others will be unrecognized.
   731  	//
   732  	// This is best effort. So if an error occurs, we'll still return a result, but it
   733  	// may include a dynamic extension.
   734  	fd.Options = removeDynamicExtensionsFromOptions(fd.Options)
   735  	_ = walk.DescriptorProtos(fd, func(_ protoreflect.FullName, msg proto.Message) error {
   736  		switch msg := msg.(type) {
   737  		case *descriptorpb.DescriptorProto:
   738  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   739  			for _, extr := range msg.ExtensionRange {
   740  				extr.Options = removeDynamicExtensionsFromOptions(extr.Options)
   741  			}
   742  		case *descriptorpb.FieldDescriptorProto:
   743  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   744  		case *descriptorpb.OneofDescriptorProto:
   745  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   746  		case *descriptorpb.EnumDescriptorProto:
   747  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   748  		case *descriptorpb.EnumValueDescriptorProto:
   749  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   750  		case *descriptorpb.ServiceDescriptorProto:
   751  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   752  		case *descriptorpb.MethodDescriptorProto:
   753  			msg.Options = removeDynamicExtensionsFromOptions(msg.Options)
   754  		}
   755  		return nil
   756  	})
   757  }
   758  
   759  type ptrMsg[T any] interface {
   760  	*T
   761  	proto.Message
   762  }
   763  
   764  type fieldValue struct {
   765  	fd  protoreflect.FieldDescriptor
   766  	val protoreflect.Value
   767  }
   768  
   769  func removeDynamicExtensionsFromOptions[O ptrMsg[T], T any](opts O) O {
   770  	if opts == nil {
   771  		return nil
   772  	}
   773  	var dynamicExtensions []fieldValue
   774  	opts.ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool {
   775  		if fd.IsExtension() {
   776  			dynamicExtensions = append(dynamicExtensions, fieldValue{fd: fd, val: val})
   777  		}
   778  		return true
   779  	})
   780  
   781  	// serialize only these custom options
   782  	optsWithOnlyDyn := opts.ProtoReflect().Type().New()
   783  	for _, fv := range dynamicExtensions {
   784  		optsWithOnlyDyn.Set(fv.fd, fv.val)
   785  	}
   786  	data, err := proto.MarshalOptions{AllowPartial: true}.Marshal(optsWithOnlyDyn.Interface())
   787  	if err != nil {
   788  		// oh, well... can't fix this one
   789  		return opts
   790  	}
   791  
   792  	// and then replace values by clearing these custom options and deserializing
   793  	optsClone := proto.Clone(opts).ProtoReflect()
   794  	for _, fv := range dynamicExtensions {
   795  		optsClone.Clear(fv.fd)
   796  	}
   797  	err = proto.UnmarshalOptions{AllowPartial: true, Merge: true}.Unmarshal(data, optsClone.Interface())
   798  	if err != nil {
   799  		// bummer, can't fix this one
   800  		return opts
   801  	}
   802  
   803  	return optsClone.Interface().(O)
   804  }