github.com/jhump/protoreflect@v1.16.0/desc/protoparse/parser.go (about) 1 package protoparse 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "path/filepath" 10 "sort" 11 "strings" 12 13 "github.com/bufbuild/protocompile" 14 ast2 "github.com/bufbuild/protocompile/ast" 15 "github.com/bufbuild/protocompile/linker" 16 "github.com/bufbuild/protocompile/options" 17 "github.com/bufbuild/protocompile/parser" 18 "github.com/bufbuild/protocompile/protoutil" 19 "github.com/bufbuild/protocompile/reporter" 20 "github.com/bufbuild/protocompile/sourceinfo" 21 "github.com/bufbuild/protocompile/walk" 22 "google.golang.org/protobuf/proto" 23 "google.golang.org/protobuf/reflect/protoreflect" 24 "google.golang.org/protobuf/types/descriptorpb" 25 26 "github.com/jhump/protoreflect/desc" 27 "github.com/jhump/protoreflect/desc/internal" 28 "github.com/jhump/protoreflect/desc/protoparse/ast" 29 ) 30 31 // FileAccessor is an abstraction for opening proto source files. It takes the 32 // name of the file to open and returns either the input reader or an error. 33 type FileAccessor func(filename string) (io.ReadCloser, error) 34 35 // FileContentsFromMap returns a FileAccessor that uses the given map of file 36 // contents. This allows proto source files to be constructed in memory and 37 // easily supplied to a parser. The map keys are the paths to the proto source 38 // files, and the values are the actual proto source contents. 39 func FileContentsFromMap(files map[string]string) FileAccessor { 40 return func(filename string) (io.ReadCloser, error) { 41 contents, ok := files[filename] 42 if !ok { 43 // Try changing path separators since user-provided 44 // map may use different separators. 45 contents, ok = files[filepath.ToSlash(filename)] 46 if !ok { 47 return nil, os.ErrNotExist 48 } 49 } 50 return ioutil.NopCloser(strings.NewReader(contents)), nil 51 } 52 } 53 54 // Parser parses proto source into descriptors. 55 type Parser struct { 56 // The paths used to search for dependencies that are referenced in import 57 // statements in proto source files. If no import paths are provided then 58 // "." (current directory) is assumed to be the only import path. 59 // 60 // This setting is only used during ParseFiles operations. Since calls to 61 // ParseFilesButDoNotLink do not link, there is no need to load and parse 62 // dependencies. 63 ImportPaths []string 64 65 // If true, the supplied file names/paths need not necessarily match how the 66 // files are referenced in import statements. The parser will attempt to 67 // match import statements to supplied paths, "guessing" the import paths 68 // for the files. Note that this inference is not perfect and link errors 69 // could result. It works best when all proto files are organized such that 70 // a single import path can be inferred (e.g. all files under a single tree 71 // with import statements all being relative to the root of this tree). 72 InferImportPaths bool 73 74 // LookupImport is a function that accepts a filename and 75 // returns a file descriptor, which will be consulted when resolving imports. 76 // This allows a compiled Go proto in another Go module to be referenced 77 // in the proto(s) being parsed. 78 // 79 // In the event of a filename collision, Accessor is consulted first, 80 // then LookupImport is consulted, and finally the well-known protos 81 // are used. 82 // 83 // For example, in order to automatically look up compiled Go protos that 84 // have been imported and be able to use them as imports, set this to 85 // desc.LoadFileDescriptor. 86 LookupImport func(string) (*desc.FileDescriptor, error) 87 88 // LookupImportProto has the same functionality as LookupImport, however it returns 89 // a FileDescriptorProto instead of a FileDescriptor. 90 LookupImportProto func(string) (*descriptorpb.FileDescriptorProto, error) 91 92 // Used to create a reader for a given filename, when loading proto source 93 // file contents. If unset, os.Open is used. If ImportPaths is also empty 94 // then relative paths are will be relative to the process's current working 95 // directory. 96 Accessor FileAccessor 97 98 // If true, the resulting file descriptors will retain source code info, 99 // that maps elements to their location in the source files as well as 100 // includes comments found during parsing (and attributed to elements of 101 // the source file). 102 IncludeSourceCodeInfo bool 103 104 // If true, the results from ParseFilesButDoNotLink will be passed through 105 // some additional validations. But only constraints that do not require 106 // linking can be checked. These include proto2 vs. proto3 language features, 107 // looking for incorrect usage of reserved names or tags, and ensuring that 108 // fields have unique tags and that enum values have unique numbers (unless 109 // the enum allows aliases). 110 ValidateUnlinkedFiles bool 111 112 // If true, the results from ParseFilesButDoNotLink will have options 113 // interpreted. Any uninterpretable options (including any custom options or 114 // options that refer to message and enum types, which can only be 115 // interpreted after linking) will be left in uninterpreted_options. Also, 116 // the "default" pseudo-option for fields can only be interpreted for scalar 117 // fields, excluding enums. (Interpreting default values for enum fields 118 // requires resolving enum names, which requires linking.) 119 InterpretOptionsInUnlinkedFiles bool 120 121 // A custom reporter of syntax and link errors. If not specified, the 122 // default reporter just returns the reported error, which causes parsing 123 // to abort after encountering a single error. 124 // 125 // The reporter is not invoked for system or I/O errors, only for syntax and 126 // link errors. 127 ErrorReporter ErrorReporter 128 129 // A custom reporter of warnings. If not specified, warning messages are ignored. 130 WarningReporter WarningReporter 131 } 132 133 // ParseFiles parses the named files into descriptors. The returned slice has 134 // the same number of entries as the give filenames, in the same order. So the 135 // first returned descriptor corresponds to the first given name, and so on. 136 // 137 // All dependencies for all specified files (including transitive dependencies) 138 // must be accessible via the parser's Accessor or a link error will occur. The 139 // exception to this rule is that files can import standard Google-provided 140 // files -- e.g. google/protobuf/*.proto -- without needing to supply sources 141 // for these files. Like protoc, this parser has a built-in version of these 142 // files it can use if they aren't explicitly supplied. 143 // 144 // If the Parser has no ErrorReporter set and a syntax or link error occurs, 145 // parsing will abort with the first such error encountered. If there is an 146 // ErrorReporter configured and it returns non-nil, parsing will abort with the 147 // error it returns. If syntax or link errors are encountered but the configured 148 // ErrorReporter always returns nil, the parse fails with ErrInvalidSource. 149 func (p Parser) ParseFiles(filenames ...string) ([]*desc.FileDescriptor, error) { 150 srcInfoMode := protocompile.SourceInfoNone 151 if p.IncludeSourceCodeInfo { 152 srcInfoMode = protocompile.SourceInfoExtraComments 153 } 154 rep := newReporter(p.ErrorReporter, p.WarningReporter) 155 res, srcSpanAddr := p.getResolver(filenames) 156 157 if p.InferImportPaths { 158 // we must first compile everything to protos 159 results, err := parseToProtosRecursive(res, filenames, reporter.NewHandler(rep), srcSpanAddr) 160 if err != nil { 161 return nil, err 162 } 163 // then we can infer import paths 164 var rewritten map[string]string 165 results, rewritten = fixupFilenames(results) 166 if len(rewritten) > 0 { 167 for i := range filenames { 168 if replace, ok := rewritten[filenames[i]]; ok { 169 filenames[i] = replace 170 } 171 } 172 } 173 resolverFromResults := protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 174 res, ok := results[path] 175 if !ok { 176 return protocompile.SearchResult{}, os.ErrNotExist 177 } 178 return protocompile.SearchResult{ParseResult: noCloneParseResult{res}}, nil 179 }) 180 res = protocompile.CompositeResolver{resolverFromResults, res} 181 } 182 183 c := protocompile.Compiler{ 184 Resolver: res, 185 MaxParallelism: 1, 186 SourceInfoMode: srcInfoMode, 187 Reporter: rep, 188 } 189 results, err := c.Compile(context.Background(), filenames...) 190 if err != nil { 191 return nil, err 192 } 193 194 fds := make([]protoreflect.FileDescriptor, len(results)) 195 alreadySeen := make(map[string]struct{}, len(results)) 196 for i, res := range results { 197 removeDynamicExtensions(res, alreadySeen) 198 fds[i] = res 199 } 200 return desc.WrapFiles(fds) 201 } 202 203 type noCloneParseResult struct { 204 parser.Result 205 } 206 207 func (r noCloneParseResult) Clone() parser.Result { 208 // protocompile will clone parser.Result to make sure it can't be shared 209 // with other compilation operations (which would not be thread-safe). 210 // However, this parse result cannot be shared with another compile 211 // operation. That means the clone is unnecessary; so we skip it, to avoid 212 // the associated performance costs. 213 return r.Result 214 } 215 216 // ParseFilesButDoNotLink parses the named files into descriptor protos. The 217 // results are just protos, not fully-linked descriptors. It is possible that 218 // descriptors are invalid and still be returned in parsed form without error 219 // due to the fact that the linking step is skipped (and thus many validation 220 // steps omitted). 221 // 222 // There are a few side effects to not linking the descriptors: 223 // 1. No options will be interpreted. Options can refer to extensions or have 224 // message and enum types. Without linking, these extension and type 225 // references are not resolved, so the options may not be interpretable. 226 // So all options will appear in UninterpretedOption fields of the various 227 // descriptor options messages. 228 // 2. Type references will not be resolved. This means that the actual type 229 // names in the descriptors may be unqualified and even relative to the 230 // scope in which the type reference appears. This goes for fields that 231 // have message and enum types. It also applies to methods and their 232 // references to request and response message types. 233 // 3. Type references are not known. For non-scalar fields, until the type 234 // name is resolved (during linking), it is not known whether the type 235 // refers to a message or an enum. So all fields with such type references 236 // will not have their Type set, only the TypeName. 237 // 238 // This method will still validate the syntax of parsed files. If the parser's 239 // ValidateUnlinkedFiles field is true, additional checks, beyond syntax will 240 // also be performed. 241 // 242 // If the Parser has no ErrorReporter set and a syntax error occurs, parsing 243 // will abort with the first such error encountered. If there is an 244 // ErrorReporter configured and it returns non-nil, parsing will abort with the 245 // error it returns. If syntax errors are encountered but the configured 246 // ErrorReporter always returns nil, the parse fails with ErrInvalidSource. 247 func (p Parser) ParseFilesButDoNotLink(filenames ...string) ([]*descriptorpb.FileDescriptorProto, error) { 248 rep := newReporter(p.ErrorReporter, p.WarningReporter) 249 p.ImportPaths = nil // not used for this "do not link" operation. 250 res, _ := p.getResolver(filenames) 251 results, err := parseToProtos(res, filenames, reporter.NewHandler(rep), p.ValidateUnlinkedFiles) 252 if err != nil { 253 return nil, err 254 } 255 256 if p.InferImportPaths { 257 resultsMap := make(map[string]parser.Result, len(results)) 258 for _, res := range results { 259 resultsMap[res.FileDescriptorProto().GetName()] = res 260 } 261 var rewritten map[string]string 262 resultsMap, rewritten = fixupFilenames(resultsMap) 263 if len(rewritten) > 0 { 264 for i := range filenames { 265 if replace, ok := rewritten[filenames[i]]; ok { 266 filenames[i] = replace 267 } 268 } 269 } 270 for i := range filenames { 271 results[i] = resultsMap[filenames[i]] 272 } 273 } 274 275 protos := make([]*descriptorpb.FileDescriptorProto, len(results)) 276 for i, res := range results { 277 protos[i] = res.FileDescriptorProto() 278 var optsIndex sourceinfo.OptionIndex 279 if p.InterpretOptionsInUnlinkedFiles { 280 var err error 281 optsIndex, err = options.InterpretUnlinkedOptions(res) 282 if err != nil { 283 return nil, err 284 } 285 removeDynamicExtensionsFromProto(protos[i]) 286 } 287 if p.IncludeSourceCodeInfo { 288 protos[i].SourceCodeInfo = sourceinfo.GenerateSourceInfo(res.AST(), optsIndex, sourceinfo.WithExtraComments()) 289 } 290 } 291 292 return protos, nil 293 } 294 295 // ParseToAST parses the named files into ASTs, or Abstract Syntax Trees. This 296 // is for consumers of proto files that don't care about compiling the files to 297 // descriptors, but care deeply about a non-lossy structured representation of 298 // the source (since descriptors are lossy). This includes formatting tools and 299 // possibly linters, too. 300 // 301 // If the requested filenames include standard imports (such as 302 // "google/protobuf/empty.proto") and no source is provided, the corresponding 303 // AST in the returned slice will be nil. These standard imports are only 304 // available for use as descriptors; no source is available unless it is 305 // provided by the configured Accessor. 306 // 307 // If the Parser has no ErrorReporter set and a syntax error occurs, parsing 308 // will abort with the first such error encountered. If there is an 309 // ErrorReporter configured and it returns non-nil, parsing will abort with the 310 // error it returns. If syntax errors are encountered but the configured 311 // ErrorReporter always returns nil, the parse fails with ErrInvalidSource. 312 func (p Parser) ParseToAST(filenames ...string) ([]*ast.FileNode, error) { 313 rep := newReporter(p.ErrorReporter, p.WarningReporter) 314 res, _ := p.getResolver(filenames) 315 asts, _, err := parseToASTs(res, filenames, reporter.NewHandler(rep)) 316 if err != nil { 317 return nil, err 318 } 319 results := make([]*ast.FileNode, len(asts)) 320 for i := range asts { 321 if asts[i] == nil { 322 // should not be possible but... 323 return nil, fmt.Errorf("resolver did not produce source for %v", filenames[i]) 324 } 325 results[i] = convertAST(asts[i]) 326 } 327 return results, nil 328 } 329 330 func parseToAST(res protocompile.Resolver, filename string, rep *reporter.Handler) (*ast2.FileNode, parser.Result, error) { 331 searchResult, err := res.FindFileByPath(filename) 332 if err != nil { 333 _ = rep.HandleError(err) 334 return nil, nil, rep.Error() 335 } 336 switch { 337 case searchResult.ParseResult != nil: 338 return nil, searchResult.ParseResult, nil 339 case searchResult.Proto != nil: 340 return nil, parser.ResultWithoutAST(searchResult.Proto), nil 341 case searchResult.Desc != nil: 342 return nil, parser.ResultWithoutAST(protoutil.ProtoFromFileDescriptor(searchResult.Desc)), nil 343 case searchResult.AST != nil: 344 return searchResult.AST, nil, nil 345 case searchResult.Source != nil: 346 astRoot, err := parser.Parse(filename, searchResult.Source, rep) 347 return astRoot, nil, err 348 default: 349 _ = rep.HandleError(fmt.Errorf("resolver did not produce a result for %v", filename)) 350 return nil, nil, rep.Error() 351 } 352 } 353 354 func parseToASTs(res protocompile.Resolver, filenames []string, rep *reporter.Handler) ([]*ast2.FileNode, []parser.Result, error) { 355 asts := make([]*ast2.FileNode, len(filenames)) 356 results := make([]parser.Result, len(filenames)) 357 for i := range filenames { 358 asts[i], results[i], _ = parseToAST(res, filenames[i], rep) 359 if rep.ReporterError() != nil { 360 break 361 } 362 } 363 return asts, results, rep.Error() 364 } 365 366 func parseToProtos(res protocompile.Resolver, filenames []string, rep *reporter.Handler, validate bool) ([]parser.Result, error) { 367 asts, results, err := parseToASTs(res, filenames, rep) 368 if err != nil { 369 return nil, err 370 } 371 for i := range results { 372 if results[i] != nil { 373 continue 374 } 375 var err error 376 results[i], err = parser.ResultFromAST(asts[i], validate, rep) 377 if err != nil { 378 return nil, err 379 } 380 } 381 return results, nil 382 } 383 384 func parseToProtosRecursive(res protocompile.Resolver, filenames []string, rep *reporter.Handler, srcSpanAddr *ast2.SourceSpan) (map[string]parser.Result, error) { 385 results := make(map[string]parser.Result, len(filenames)) 386 for _, filename := range filenames { 387 if err := parseToProtoRecursive(res, filename, rep, srcSpanAddr, results); err != nil { 388 return results, err 389 } 390 } 391 return results, rep.Error() 392 } 393 394 func parseToProtoRecursive(res protocompile.Resolver, filename string, rep *reporter.Handler, srcSpanAddr *ast2.SourceSpan, results map[string]parser.Result) error { 395 if _, ok := results[filename]; ok { 396 // already processed this one 397 return nil 398 } 399 results[filename] = nil // placeholder entry 400 401 astRoot, parseResult, err := parseToAST(res, filename, rep) 402 if err != nil { 403 return err 404 } 405 if parseResult == nil { 406 parseResult, err = parser.ResultFromAST(astRoot, true, rep) 407 if err != nil { 408 return err 409 } 410 } 411 results[filename] = parseResult 412 413 if astRoot != nil { 414 // We have an AST, so we use it to recursively examine imports. 415 for _, decl := range astRoot.Decls { 416 imp, ok := decl.(*ast2.ImportNode) 417 if !ok { 418 continue 419 } 420 err := func() error { 421 orig := *srcSpanAddr 422 *srcSpanAddr = astRoot.NodeInfo(imp.Name) 423 defer func() { 424 *srcSpanAddr = orig 425 }() 426 427 return parseToProtoRecursive(res, imp.Name.AsString(), rep, srcSpanAddr, results) 428 }() 429 if err != nil { 430 return err 431 } 432 } 433 return nil 434 } 435 436 // Without an AST, we must recursively examine the proto. This makes it harder 437 // (but not necessarily impossible) to get the source location of the import. 438 fd := parseResult.FileDescriptorProto() 439 for i, dep := range fd.Dependency { 440 path := []int32{internal.File_dependencyTag, int32(i)} 441 err := func() error { 442 orig := *srcSpanAddr 443 found := false 444 for _, loc := range fd.GetSourceCodeInfo().GetLocation() { 445 if pathsEqual(loc.Path, path) { 446 start := SourcePos{ 447 Filename: dep, 448 Line: int(loc.Span[0]), 449 Col: int(loc.Span[1]), 450 } 451 var end SourcePos 452 if len(loc.Span) > 3 { 453 end = SourcePos{ 454 Filename: dep, 455 Line: int(loc.Span[2]), 456 Col: int(loc.Span[3]), 457 } 458 } else { 459 end = SourcePos{ 460 Filename: dep, 461 Line: int(loc.Span[0]), 462 Col: int(loc.Span[2]), 463 } 464 } 465 *srcSpanAddr = ast2.NewSourceSpan(start, end) 466 found = true 467 break 468 } 469 } 470 if !found { 471 *srcSpanAddr = ast2.UnknownSpan(dep) 472 } 473 defer func() { 474 *srcSpanAddr = orig 475 }() 476 477 return parseToProtoRecursive(res, dep, rep, srcSpanAddr, results) 478 }() 479 if err != nil { 480 return err 481 } 482 } 483 return nil 484 } 485 486 func pathsEqual(a, b []int32) bool { 487 if len(a) != len(b) { 488 return false 489 } 490 for i := range a { 491 if a[i] != b[i] { 492 return false 493 } 494 } 495 return true 496 } 497 498 func newReporter(errRep ErrorReporter, warnRep WarningReporter) reporter.Reporter { 499 if errRep != nil { 500 delegate := errRep 501 errRep = func(err ErrorWithPos) error { 502 if _, ok := err.(ErrorWithSourcePos); !ok { 503 err = toErrorWithSourcePos(err) 504 } 505 return delegate(err) 506 } 507 } 508 if warnRep != nil { 509 delegate := warnRep 510 warnRep = func(err ErrorWithPos) { 511 if _, ok := err.(ErrorWithSourcePos); !ok { 512 err = toErrorWithSourcePos(err) 513 } 514 delegate(err) 515 } 516 } 517 return reporter.NewReporter(errRep, warnRep) 518 } 519 520 func (p Parser) getResolver(filenames []string) (protocompile.Resolver, *ast2.SourceSpan) { 521 var srcSpan ast2.SourceSpan 522 accessor := p.Accessor 523 if accessor == nil { 524 accessor = func(name string) (io.ReadCloser, error) { 525 return os.Open(name) 526 } 527 } 528 sourceResolver := &protocompile.SourceResolver{ 529 Accessor: func(filename string) (io.ReadCloser, error) { 530 in, err := accessor(filename) 531 if err != nil { 532 if !strings.Contains(err.Error(), filename) { 533 // errors that don't include the filename that failed are no bueno 534 err = errorWithFilename{filename: filename, underlying: err} 535 } 536 if srcSpan != nil { 537 err = reporter.Error(srcSpan, err) 538 } 539 } 540 return in, err 541 }, 542 ImportPaths: p.ImportPaths, 543 } 544 var importResolver protocompile.CompositeResolver 545 if p.LookupImport != nil { 546 importResolver = append(importResolver, protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 547 fd, err := p.LookupImport(path) 548 if err != nil { 549 return protocompile.SearchResult{}, err 550 } 551 return protocompile.SearchResult{Desc: fd.UnwrapFile()}, nil 552 })) 553 } 554 if p.LookupImportProto != nil { 555 importResolver = append(importResolver, protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 556 fd, err := p.LookupImportProto(path) 557 if err != nil { 558 return protocompile.SearchResult{}, err 559 } 560 return protocompile.SearchResult{Proto: fd}, nil 561 })) 562 } 563 backupResolver := protocompile.WithStandardImports(importResolver) 564 return protocompile.CompositeResolver{ 565 sourceResolver, 566 protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 567 return backupResolver.FindFileByPath(path) 568 }), 569 }, &srcSpan 570 } 571 572 func fixupFilenames(protos map[string]parser.Result) (revisedProtos map[string]parser.Result, rewrittenPaths map[string]string) { 573 // In the event that the given filenames (keys in the supplied map) do not 574 // match the actual paths used in 'import' statements in the files, we try 575 // to revise names in the protos so that they will match and be linkable. 576 revisedProtos = make(map[string]parser.Result, len(protos)) 577 rewrittenPaths = make(map[string]string, len(protos)) 578 579 protoPaths := map[string]struct{}{} 580 // TODO: this is O(n^2) but could likely be O(n) with a clever data structure (prefix tree that is indexed backwards?) 581 importCandidates := map[string]map[string]struct{}{} 582 candidatesAvailable := map[string]struct{}{} 583 for name := range protos { 584 candidatesAvailable[name] = struct{}{} 585 for _, f := range protos { 586 for _, imp := range f.FileDescriptorProto().Dependency { 587 if strings.HasSuffix(name, imp) || strings.HasSuffix(imp, name) { 588 candidates := importCandidates[imp] 589 if candidates == nil { 590 candidates = map[string]struct{}{} 591 importCandidates[imp] = candidates 592 } 593 candidates[name] = struct{}{} 594 } 595 } 596 } 597 } 598 for imp, candidates := range importCandidates { 599 // if we found multiple possible candidates, use the one that is an exact match 600 // if it exists, and otherwise, guess that it's the shortest path (fewest elements) 601 var best string 602 for c := range candidates { 603 if _, ok := candidatesAvailable[c]; !ok { 604 // already used this candidate and re-written its filename accordingly 605 continue 606 } 607 if c == imp { 608 // exact match! 609 best = c 610 break 611 } 612 if best == "" { 613 best = c 614 } else { 615 // NB: We can't actually tell which file is supposed to match 616 // this import. So we prefer the longest name. On a tie, we 617 // choose the lexically earliest match. 618 minLen := strings.Count(best, string(filepath.Separator)) 619 cLen := strings.Count(c, string(filepath.Separator)) 620 if cLen > minLen || (cLen == minLen && c < best) { 621 best = c 622 } 623 } 624 } 625 if best != "" { 626 if len(best) > len(imp) { 627 prefix := best[:len(best)-len(imp)] 628 protoPaths[prefix] = struct{}{} 629 } 630 f := protos[best] 631 f.FileDescriptorProto().Name = proto.String(imp) 632 revisedProtos[imp] = f 633 rewrittenPaths[best] = imp 634 delete(candidatesAvailable, best) 635 636 // If other candidates are actually references to the same file, remove them. 637 for c := range candidates { 638 if _, ok := candidatesAvailable[c]; !ok { 639 // already used this candidate and re-written its filename accordingly 640 continue 641 } 642 possibleDup := protos[c] 643 prevName := possibleDup.FileDescriptorProto().Name 644 possibleDup.FileDescriptorProto().Name = proto.String(imp) 645 if !proto.Equal(f.FileDescriptorProto(), protos[c].FileDescriptorProto()) { 646 // not equal: restore name and look at next one 647 possibleDup.FileDescriptorProto().Name = prevName 648 continue 649 } 650 // This file used a different name but was actually the same file. So 651 // we prune it from the set. 652 rewrittenPaths[c] = imp 653 delete(candidatesAvailable, c) 654 if len(c) > len(imp) { 655 prefix := c[:len(c)-len(imp)] 656 protoPaths[prefix] = struct{}{} 657 } 658 } 659 } 660 } 661 662 if len(candidatesAvailable) == 0 { 663 return revisedProtos, rewrittenPaths 664 } 665 666 if len(protoPaths) == 0 { 667 for c := range candidatesAvailable { 668 revisedProtos[c] = protos[c] 669 } 670 return revisedProtos, rewrittenPaths 671 } 672 673 // Any remaining candidates are entry-points (not imported by others), so 674 // the best bet to "fixing" their file name is to see if they're in one of 675 // the proto paths we found, and if so strip that prefix. 676 protoPathStrs := make([]string, len(protoPaths)) 677 i := 0 678 for p := range protoPaths { 679 protoPathStrs[i] = p 680 i++ 681 } 682 sort.Strings(protoPathStrs) 683 // we look at paths in reverse order, so we'll use a longer proto path if 684 // there is more than one match 685 for c := range candidatesAvailable { 686 var imp string 687 for i := len(protoPathStrs) - 1; i >= 0; i-- { 688 p := protoPathStrs[i] 689 if strings.HasPrefix(c, p) { 690 imp = c[len(p):] 691 break 692 } 693 } 694 if imp != "" { 695 f := protos[c] 696 f.FileDescriptorProto().Name = proto.String(imp) 697 f.FileNode() 698 revisedProtos[imp] = f 699 rewrittenPaths[c] = imp 700 } else { 701 revisedProtos[c] = protos[c] 702 } 703 } 704 705 return revisedProtos, rewrittenPaths 706 } 707 708 func removeDynamicExtensions(fd protoreflect.FileDescriptor, alreadySeen map[string]struct{}) { 709 if _, ok := alreadySeen[fd.Path()]; ok { 710 // already processed 711 return 712 } 713 alreadySeen[fd.Path()] = struct{}{} 714 res, ok := fd.(linker.Result) 715 if ok { 716 removeDynamicExtensionsFromProto(res.FileDescriptorProto()) 717 } 718 // also remove extensions from dependencies 719 for i, length := 0, fd.Imports().Len(); i < length; i++ { 720 removeDynamicExtensions(fd.Imports().Get(i).FileDescriptor, alreadySeen) 721 } 722 } 723 724 func removeDynamicExtensionsFromProto(fd *descriptorpb.FileDescriptorProto) { 725 // protocompile returns descriptors with dynamic extension fields for custom options. 726 // But protoparse only used known custom options and everything else defined in the 727 // sources would be stored as unrecognized fields. So to bridge the difference in 728 // behavior, we need to remove custom options from the given file and add them back 729 // via serializing-then-de-serializing them back into the options messages. That way, 730 // statically known options will be properly typed and others will be unrecognized. 731 // 732 // This is best effort. So if an error occurs, we'll still return a result, but it 733 // may include a dynamic extension. 734 fd.Options = removeDynamicExtensionsFromOptions(fd.Options) 735 _ = walk.DescriptorProtos(fd, func(_ protoreflect.FullName, msg proto.Message) error { 736 switch msg := msg.(type) { 737 case *descriptorpb.DescriptorProto: 738 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 739 for _, extr := range msg.ExtensionRange { 740 extr.Options = removeDynamicExtensionsFromOptions(extr.Options) 741 } 742 case *descriptorpb.FieldDescriptorProto: 743 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 744 case *descriptorpb.OneofDescriptorProto: 745 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 746 case *descriptorpb.EnumDescriptorProto: 747 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 748 case *descriptorpb.EnumValueDescriptorProto: 749 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 750 case *descriptorpb.ServiceDescriptorProto: 751 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 752 case *descriptorpb.MethodDescriptorProto: 753 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 754 } 755 return nil 756 }) 757 } 758 759 type ptrMsg[T any] interface { 760 *T 761 proto.Message 762 } 763 764 type fieldValue struct { 765 fd protoreflect.FieldDescriptor 766 val protoreflect.Value 767 } 768 769 func removeDynamicExtensionsFromOptions[O ptrMsg[T], T any](opts O) O { 770 if opts == nil { 771 return nil 772 } 773 var dynamicExtensions []fieldValue 774 opts.ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { 775 if fd.IsExtension() { 776 dynamicExtensions = append(dynamicExtensions, fieldValue{fd: fd, val: val}) 777 } 778 return true 779 }) 780 781 // serialize only these custom options 782 optsWithOnlyDyn := opts.ProtoReflect().Type().New() 783 for _, fv := range dynamicExtensions { 784 optsWithOnlyDyn.Set(fv.fd, fv.val) 785 } 786 data, err := proto.MarshalOptions{AllowPartial: true}.Marshal(optsWithOnlyDyn.Interface()) 787 if err != nil { 788 // oh, well... can't fix this one 789 return opts 790 } 791 792 // and then replace values by clearing these custom options and deserializing 793 optsClone := proto.Clone(opts).ProtoReflect() 794 for _, fv := range dynamicExtensions { 795 optsClone.Clear(fv.fd) 796 } 797 err = proto.UnmarshalOptions{AllowPartial: true, Merge: true}.Unmarshal(data, optsClone.Interface()) 798 if err != nil { 799 // bummer, can't fix this one 800 return opts 801 } 802 803 return optsClone.Interface().(O) 804 }