github.com/Big-big-orange/protoreflect@v0.0.0-20240408141420-285cedfdf6a4/desc/protoparse/parser.go (about) 1 package protoparse 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "path/filepath" 10 "sort" 11 "strings" 12 13 "github.com/bufbuild/protocompile" 14 ast2 "github.com/bufbuild/protocompile/ast" 15 "github.com/bufbuild/protocompile/linker" 16 "github.com/bufbuild/protocompile/options" 17 "github.com/bufbuild/protocompile/parser" 18 "github.com/bufbuild/protocompile/protoutil" 19 "github.com/bufbuild/protocompile/reporter" 20 "github.com/bufbuild/protocompile/sourceinfo" 21 "github.com/bufbuild/protocompile/walk" 22 "google.golang.org/protobuf/proto" 23 "google.golang.org/protobuf/reflect/protoreflect" 24 "google.golang.org/protobuf/types/descriptorpb" 25 26 "github.com/Big-big-orange/protoreflect/desc" 27 "github.com/Big-big-orange/protoreflect/desc/internal" 28 "github.com/Big-big-orange/protoreflect/desc/protoparse/ast" 29 ) 30 31 // FileAccessor is an abstraction for opening proto source files. It takes the 32 // name of the file to open and returns either the input reader or an error. 33 type FileAccessor func(filename string) (io.ReadCloser, error) 34 35 // FileContentsFromMap returns a FileAccessor that uses the given map of file 36 // contents. This allows proto source files to be constructed in memory and 37 // easily supplied to a parser. The map keys are the paths to the proto source 38 // files, and the values are the actual proto source contents. 39 func FileContentsFromMap(files map[string]string) FileAccessor { 40 return func(filename string) (io.ReadCloser, error) { 41 contents, ok := files[filename] 42 if !ok { 43 return nil, os.ErrNotExist 44 } 45 return ioutil.NopCloser(strings.NewReader(contents)), nil 46 } 47 } 48 49 // Parser parses proto source into descriptors. 50 type Parser struct { 51 // The paths used to search for dependencies that are referenced in import 52 // statements in proto source files. If no import paths are provided then 53 // "." (current directory) is assumed to be the only import path. 54 // 55 // This setting is only used during ParseFiles operations. Since calls to 56 // ParseFilesButDoNotLink do not link, there is no need to load and parse 57 // dependencies. 58 ImportPaths []string 59 60 // If true, the supplied file names/paths need not necessarily match how the 61 // files are referenced in import statements. The parser will attempt to 62 // match import statements to supplied paths, "guessing" the import paths 63 // for the files. Note that this inference is not perfect and link errors 64 // could result. It works best when all proto files are organized such that 65 // a single import path can be inferred (e.g. all files under a single tree 66 // with import statements all being relative to the root of this tree). 67 InferImportPaths bool 68 69 // LookupImport is a function that accepts a filename and 70 // returns a file descriptor, which will be consulted when resolving imports. 71 // This allows a compiled Go proto in another Go module to be referenced 72 // in the proto(s) being parsed. 73 // 74 // In the event of a filename collision, Accessor is consulted first, 75 // then LookupImport is consulted, and finally the well-known protos 76 // are used. 77 // 78 // For example, in order to automatically look up compiled Go protos that 79 // have been imported and be able to use them as imports, set this to 80 // desc.LoadFileDescriptor. 81 LookupImport func(string) (*desc.FileDescriptor, error) 82 83 // LookupImportProto has the same functionality as LookupImport, however it returns 84 // a FileDescriptorProto instead of a FileDescriptor. 85 LookupImportProto func(string) (*descriptorpb.FileDescriptorProto, error) 86 87 // Used to create a reader for a given filename, when loading proto source 88 // file contents. If unset, os.Open is used. If ImportPaths is also empty 89 // then relative paths are will be relative to the process's current working 90 // directory. 91 Accessor FileAccessor 92 93 // If true, the resulting file descriptors will retain source code info, 94 // that maps elements to their location in the source files as well as 95 // includes comments found during parsing (and attributed to elements of 96 // the source file). 97 IncludeSourceCodeInfo bool 98 99 // If true, the results from ParseFilesButDoNotLink will be passed through 100 // some additional validations. But only constraints that do not require 101 // linking can be checked. These include proto2 vs. proto3 language features, 102 // looking for incorrect usage of reserved names or tags, and ensuring that 103 // fields have unique tags and that enum values have unique numbers (unless 104 // the enum allows aliases). 105 ValidateUnlinkedFiles bool 106 107 // If true, the results from ParseFilesButDoNotLink will have options 108 // interpreted. Any uninterpretable options (including any custom options or 109 // options that refer to message and enum types, which can only be 110 // interpreted after linking) will be left in uninterpreted_options. Also, 111 // the "default" pseudo-option for fields can only be interpreted for scalar 112 // fields, excluding enums. (Interpreting default values for enum fields 113 // requires resolving enum names, which requires linking.) 114 InterpretOptionsInUnlinkedFiles bool 115 116 // A custom reporter of syntax and link errors. If not specified, the 117 // default reporter just returns the reported error, which causes parsing 118 // to abort after encountering a single error. 119 // 120 // The reporter is not invoked for system or I/O errors, only for syntax and 121 // link errors. 122 ErrorReporter ErrorReporter 123 124 // A custom reporter of warnings. If not specified, warning messages are ignored. 125 WarningReporter WarningReporter 126 } 127 128 // ParseFiles parses the named files into descriptors. The returned slice has 129 // the same number of entries as the give filenames, in the same order. So the 130 // first returned descriptor corresponds to the first given name, and so on. 131 // 132 // All dependencies for all specified files (including transitive dependencies) 133 // must be accessible via the parser's Accessor or a link error will occur. The 134 // exception to this rule is that files can import standard Google-provided 135 // files -- e.g. google/protobuf/*.proto -- without needing to supply sources 136 // for these files. Like protoc, this parser has a built-in version of these 137 // files it can use if they aren't explicitly supplied. 138 // 139 // If the Parser has no ErrorReporter set and a syntax or link error occurs, 140 // parsing will abort with the first such error encountered. If there is an 141 // ErrorReporter configured and it returns non-nil, parsing will abort with the 142 // error it returns. If syntax or link errors are encountered but the configured 143 // ErrorReporter always returns nil, the parse fails with ErrInvalidSource. 144 func (p Parser) ParseFiles(filenames ...string) ([]*desc.FileDescriptor, error) { 145 srcInfoMode := protocompile.SourceInfoNone 146 if p.IncludeSourceCodeInfo { 147 srcInfoMode = protocompile.SourceInfoExtraComments 148 } 149 rep := newReporter(p.ErrorReporter, p.WarningReporter) 150 res, srcSpanAddr := p.getResolver(filenames) 151 152 if p.InferImportPaths { 153 // we must first compile everything to protos 154 results, err := parseToProtosRecursive(res, filenames, reporter.NewHandler(rep), srcSpanAddr) 155 if err != nil { 156 return nil, err 157 } 158 // then we can infer import paths 159 var rewritten map[string]string 160 results, rewritten = fixupFilenames(results) 161 if len(rewritten) > 0 { 162 for i := range filenames { 163 if replace, ok := rewritten[filenames[i]]; ok { 164 filenames[i] = replace 165 } 166 } 167 } 168 resolverFromResults := protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 169 res, ok := results[path] 170 if !ok { 171 return protocompile.SearchResult{}, os.ErrNotExist 172 } 173 return protocompile.SearchResult{ParseResult: noCloneParseResult{res}}, nil 174 }) 175 res = protocompile.CompositeResolver{resolverFromResults, res} 176 } 177 178 c := protocompile.Compiler{ 179 Resolver: res, 180 MaxParallelism: 1, 181 SourceInfoMode: srcInfoMode, 182 Reporter: rep, 183 } 184 results, err := c.Compile(context.Background(), filenames...) 185 if err != nil { 186 return nil, err 187 } 188 189 fds := make([]protoreflect.FileDescriptor, len(results)) 190 alreadySeen := make(map[string]struct{}, len(results)) 191 for i, res := range results { 192 removeDynamicExtensions(res, alreadySeen) 193 fds[i] = res 194 } 195 return desc.WrapFiles(fds) 196 } 197 198 type noCloneParseResult struct { 199 parser.Result 200 } 201 202 func (r noCloneParseResult) Clone() parser.Result { 203 // protocompile will clone parser.Result to make sure it can't be shared 204 // with other compilation operations (which would not be thread-safe). 205 // However, this parse result cannot be shared with another compile 206 // operation. That means the clone is unnecessary; so we skip it, to avoid 207 // the associated performance costs. 208 return r.Result 209 } 210 211 // ParseFilesButDoNotLink parses the named files into descriptor protos. The 212 // results are just protos, not fully-linked descriptors. It is possible that 213 // descriptors are invalid and still be returned in parsed form without error 214 // due to the fact that the linking step is skipped (and thus many validation 215 // steps omitted). 216 // 217 // There are a few side effects to not linking the descriptors: 218 // 1. No options will be interpreted. Options can refer to extensions or have 219 // message and enum types. Without linking, these extension and type 220 // references are not resolved, so the options may not be interpretable. 221 // So all options will appear in UninterpretedOption fields of the various 222 // descriptor options messages. 223 // 2. Type references will not be resolved. This means that the actual type 224 // names in the descriptors may be unqualified and even relative to the 225 // scope in which the type reference appears. This goes for fields that 226 // have message and enum types. It also applies to methods and their 227 // references to request and response message types. 228 // 3. Type references are not known. For non-scalar fields, until the type 229 // name is resolved (during linking), it is not known whether the type 230 // refers to a message or an enum. So all fields with such type references 231 // will not have their Type set, only the TypeName. 232 // 233 // This method will still validate the syntax of parsed files. If the parser's 234 // ValidateUnlinkedFiles field is true, additional checks, beyond syntax will 235 // also be performed. 236 // 237 // If the Parser has no ErrorReporter set and a syntax error occurs, parsing 238 // will abort with the first such error encountered. If there is an 239 // ErrorReporter configured and it returns non-nil, parsing will abort with the 240 // error it returns. If syntax errors are encountered but the configured 241 // ErrorReporter always returns nil, the parse fails with ErrInvalidSource. 242 func (p Parser) ParseFilesButDoNotLink(filenames ...string) ([]*descriptorpb.FileDescriptorProto, error) { 243 rep := newReporter(p.ErrorReporter, p.WarningReporter) 244 p.ImportPaths = nil // not used for this "do not link" operation. 245 res, _ := p.getResolver(filenames) 246 results, err := parseToProtos(res, filenames, reporter.NewHandler(rep), p.ValidateUnlinkedFiles) 247 if err != nil { 248 return nil, err 249 } 250 251 if p.InferImportPaths { 252 resultsMap := make(map[string]parser.Result, len(results)) 253 for _, res := range results { 254 resultsMap[res.FileDescriptorProto().GetName()] = res 255 } 256 var rewritten map[string]string 257 resultsMap, rewritten = fixupFilenames(resultsMap) 258 if len(rewritten) > 0 { 259 for i := range filenames { 260 if replace, ok := rewritten[filenames[i]]; ok { 261 filenames[i] = replace 262 } 263 } 264 } 265 for i := range filenames { 266 results[i] = resultsMap[filenames[i]] 267 } 268 } 269 270 protos := make([]*descriptorpb.FileDescriptorProto, len(results)) 271 for i, res := range results { 272 protos[i] = res.FileDescriptorProto() 273 var optsIndex sourceinfo.OptionIndex 274 if p.InterpretOptionsInUnlinkedFiles { 275 var err error 276 optsIndex, err = options.InterpretUnlinkedOptions(res) 277 if err != nil { 278 return nil, err 279 } 280 removeDynamicExtensionsFromProto(protos[i]) 281 } 282 if p.IncludeSourceCodeInfo { 283 protos[i].SourceCodeInfo = sourceinfo.GenerateSourceInfo(res.AST(), optsIndex, sourceinfo.WithExtraComments()) 284 } 285 } 286 287 return protos, nil 288 } 289 290 // ParseToAST parses the named files into ASTs, or Abstract Syntax Trees. This 291 // is for consumers of proto files that don't care about compiling the files to 292 // descriptors, but care deeply about a non-lossy structured representation of 293 // the source (since descriptors are lossy). This includes formatting tools and 294 // possibly linters, too. 295 // 296 // If the requested filenames include standard imports (such as 297 // "google/protobuf/empty.proto") and no source is provided, the corresponding 298 // AST in the returned slice will be nil. These standard imports are only 299 // available for use as descriptors; no source is available unless it is 300 // provided by the configured Accessor. 301 // 302 // If the Parser has no ErrorReporter set and a syntax error occurs, parsing 303 // will abort with the first such error encountered. If there is an 304 // ErrorReporter configured and it returns non-nil, parsing will abort with the 305 // error it returns. If syntax errors are encountered but the configured 306 // ErrorReporter always returns nil, the parse fails with ErrInvalidSource. 307 func (p Parser) ParseToAST(filenames ...string) ([]*ast.FileNode, error) { 308 rep := newReporter(p.ErrorReporter, p.WarningReporter) 309 res, _ := p.getResolver(filenames) 310 asts, _, err := parseToASTs(res, filenames, reporter.NewHandler(rep)) 311 if err != nil { 312 return nil, err 313 } 314 results := make([]*ast.FileNode, len(asts)) 315 for i := range asts { 316 if asts[i] == nil { 317 // should not be possible but... 318 return nil, fmt.Errorf("resolver did not produce source for %v", filenames[i]) 319 } 320 results[i] = convertAST(asts[i]) 321 } 322 return results, nil 323 } 324 325 func parseToAST(res protocompile.Resolver, filename string, rep *reporter.Handler) (*ast2.FileNode, parser.Result, error) { 326 searchResult, err := res.FindFileByPath(filename) 327 if err != nil { 328 _ = rep.HandleError(err) 329 return nil, nil, rep.Error() 330 } 331 switch { 332 case searchResult.ParseResult != nil: 333 return nil, searchResult.ParseResult, nil 334 case searchResult.Proto != nil: 335 return nil, parser.ResultWithoutAST(searchResult.Proto), nil 336 case searchResult.Desc != nil: 337 return nil, parser.ResultWithoutAST(protoutil.ProtoFromFileDescriptor(searchResult.Desc)), nil 338 case searchResult.AST != nil: 339 return searchResult.AST, nil, nil 340 case searchResult.Source != nil: 341 astRoot, err := parser.Parse(filename, searchResult.Source, rep) 342 return astRoot, nil, err 343 default: 344 _ = rep.HandleError(fmt.Errorf("resolver did not produce a result for %v", filename)) 345 return nil, nil, rep.Error() 346 } 347 } 348 349 func parseToASTs(res protocompile.Resolver, filenames []string, rep *reporter.Handler) ([]*ast2.FileNode, []parser.Result, error) { 350 asts := make([]*ast2.FileNode, len(filenames)) 351 results := make([]parser.Result, len(filenames)) 352 for i := range filenames { 353 asts[i], results[i], _ = parseToAST(res, filenames[i], rep) 354 if rep.ReporterError() != nil { 355 break 356 } 357 } 358 return asts, results, rep.Error() 359 } 360 361 func parseToProtos(res protocompile.Resolver, filenames []string, rep *reporter.Handler, validate bool) ([]parser.Result, error) { 362 asts, results, err := parseToASTs(res, filenames, rep) 363 if err != nil { 364 return nil, err 365 } 366 for i := range results { 367 if results[i] != nil { 368 continue 369 } 370 var err error 371 results[i], err = parser.ResultFromAST(asts[i], validate, rep) 372 if err != nil { 373 return nil, err 374 } 375 } 376 return results, nil 377 } 378 379 func parseToProtosRecursive(res protocompile.Resolver, filenames []string, rep *reporter.Handler, srcSpanAddr *ast2.SourceSpan) (map[string]parser.Result, error) { 380 results := make(map[string]parser.Result, len(filenames)) 381 for _, filename := range filenames { 382 if err := parseToProtoRecursive(res, filename, rep, srcSpanAddr, results); err != nil { 383 return results, err 384 } 385 } 386 return results, rep.Error() 387 } 388 389 func parseToProtoRecursive(res protocompile.Resolver, filename string, rep *reporter.Handler, srcSpanAddr *ast2.SourceSpan, results map[string]parser.Result) error { 390 if _, ok := results[filename]; ok { 391 // already processed this one 392 return nil 393 } 394 results[filename] = nil // placeholder entry 395 396 astRoot, parseResult, err := parseToAST(res, filename, rep) 397 if err != nil { 398 return err 399 } 400 if parseResult == nil { 401 parseResult, err = parser.ResultFromAST(astRoot, true, rep) 402 if err != nil { 403 return err 404 } 405 } 406 results[filename] = parseResult 407 408 if astRoot != nil { 409 // We have an AST, so we use it to recursively examine imports. 410 for _, decl := range astRoot.Decls { 411 imp, ok := decl.(*ast2.ImportNode) 412 if !ok { 413 continue 414 } 415 err := func() error { 416 orig := *srcSpanAddr 417 *srcSpanAddr = astRoot.NodeInfo(imp.Name) 418 defer func() { 419 *srcSpanAddr = orig 420 }() 421 422 return parseToProtoRecursive(res, imp.Name.AsString(), rep, srcSpanAddr, results) 423 }() 424 if err != nil { 425 return err 426 } 427 } 428 return nil 429 } 430 431 // Without an AST, we must recursively examine the proto. This makes it harder 432 // (but not necessarily impossible) to get the source location of the import. 433 fd := parseResult.FileDescriptorProto() 434 for i, dep := range fd.Dependency { 435 path := []int32{internal.File_dependencyTag, int32(i)} 436 err := func() error { 437 orig := *srcSpanAddr 438 found := false 439 for _, loc := range fd.GetSourceCodeInfo().GetLocation() { 440 if pathsEqual(loc.Path, path) { 441 start := SourcePos{ 442 Filename: dep, 443 Line: int(loc.Span[0]), 444 Col: int(loc.Span[1]), 445 } 446 var end SourcePos 447 if len(loc.Span) > 3 { 448 end = SourcePos{ 449 Filename: dep, 450 Line: int(loc.Span[2]), 451 Col: int(loc.Span[3]), 452 } 453 } else { 454 end = SourcePos{ 455 Filename: dep, 456 Line: int(loc.Span[0]), 457 Col: int(loc.Span[2]), 458 } 459 } 460 *srcSpanAddr = ast2.NewSourceSpan(start, end) 461 found = true 462 break 463 } 464 } 465 if !found { 466 *srcSpanAddr = ast2.UnknownSpan(dep) 467 } 468 defer func() { 469 *srcSpanAddr = orig 470 }() 471 472 return parseToProtoRecursive(res, dep, rep, srcSpanAddr, results) 473 }() 474 if err != nil { 475 return err 476 } 477 } 478 return nil 479 } 480 481 func pathsEqual(a, b []int32) bool { 482 if len(a) != len(b) { 483 return false 484 } 485 for i := range a { 486 if a[i] != b[i] { 487 return false 488 } 489 } 490 return true 491 } 492 493 func newReporter(errRep ErrorReporter, warnRep WarningReporter) reporter.Reporter { 494 if errRep != nil { 495 delegate := errRep 496 errRep = func(err ErrorWithPos) error { 497 if _, ok := err.(ErrorWithSourcePos); !ok { 498 err = toErrorWithSourcePos(err) 499 } 500 return delegate(err) 501 } 502 } 503 if warnRep != nil { 504 delegate := warnRep 505 warnRep = func(err ErrorWithPos) { 506 if _, ok := err.(ErrorWithSourcePos); !ok { 507 err = toErrorWithSourcePos(err) 508 } 509 delegate(err) 510 } 511 } 512 return reporter.NewReporter(errRep, warnRep) 513 } 514 515 func (p Parser) getResolver(filenames []string) (protocompile.Resolver, *ast2.SourceSpan) { 516 var srcSpan ast2.SourceSpan 517 accessor := p.Accessor 518 if accessor == nil { 519 accessor = func(name string) (io.ReadCloser, error) { 520 return os.Open(name) 521 } 522 } 523 sourceResolver := &protocompile.SourceResolver{ 524 Accessor: func(filename string) (io.ReadCloser, error) { 525 in, err := accessor(filename) 526 if err != nil { 527 if !strings.Contains(err.Error(), filename) { 528 // errors that don't include the filename that failed are no bueno 529 err = errorWithFilename{filename: filename, underlying: err} 530 } 531 if srcSpan != nil { 532 err = reporter.Error(srcSpan, err) 533 } 534 } 535 return in, err 536 }, 537 ImportPaths: p.ImportPaths, 538 } 539 var importResolver protocompile.CompositeResolver 540 if p.LookupImport != nil { 541 importResolver = append(importResolver, protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 542 fd, err := p.LookupImport(path) 543 if err != nil { 544 return protocompile.SearchResult{}, err 545 } 546 return protocompile.SearchResult{Desc: fd.UnwrapFile()}, nil 547 })) 548 } 549 if p.LookupImportProto != nil { 550 importResolver = append(importResolver, protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 551 fd, err := p.LookupImportProto(path) 552 if err != nil { 553 return protocompile.SearchResult{}, err 554 } 555 return protocompile.SearchResult{Proto: fd}, nil 556 })) 557 } 558 backupResolver := protocompile.WithStandardImports(importResolver) 559 return protocompile.CompositeResolver{ 560 sourceResolver, 561 protocompile.ResolverFunc(func(path string) (protocompile.SearchResult, error) { 562 return backupResolver.FindFileByPath(path) 563 }), 564 }, &srcSpan 565 } 566 567 func fixupFilenames(protos map[string]parser.Result) (revisedProtos map[string]parser.Result, rewrittenPaths map[string]string) { 568 // In the event that the given filenames (keys in the supplied map) do not 569 // match the actual paths used in 'import' statements in the files, we try 570 // to revise names in the protos so that they will match and be linkable. 571 revisedProtos = make(map[string]parser.Result, len(protos)) 572 rewrittenPaths = make(map[string]string, len(protos)) 573 574 protoPaths := map[string]struct{}{} 575 // TODO: this is O(n^2) but could likely be O(n) with a clever data structure (prefix tree that is indexed backwards?) 576 importCandidates := map[string]map[string]struct{}{} 577 candidatesAvailable := map[string]struct{}{} 578 for name := range protos { 579 candidatesAvailable[name] = struct{}{} 580 for _, f := range protos { 581 for _, imp := range f.FileDescriptorProto().Dependency { 582 if strings.HasSuffix(name, imp) || strings.HasSuffix(imp, name) { 583 candidates := importCandidates[imp] 584 if candidates == nil { 585 candidates = map[string]struct{}{} 586 importCandidates[imp] = candidates 587 } 588 candidates[name] = struct{}{} 589 } 590 } 591 } 592 } 593 for imp, candidates := range importCandidates { 594 // if we found multiple possible candidates, use the one that is an exact match 595 // if it exists, and otherwise, guess that it's the shortest path (fewest elements) 596 var best string 597 for c := range candidates { 598 if _, ok := candidatesAvailable[c]; !ok { 599 // already used this candidate and re-written its filename accordingly 600 continue 601 } 602 if c == imp { 603 // exact match! 604 best = c 605 break 606 } 607 if best == "" { 608 best = c 609 } else { 610 // NB: We can't actually tell which file is supposed to match 611 // this import. So we prefer the longest name. On a tie, we 612 // choose the lexically earliest match. 613 minLen := strings.Count(best, string(filepath.Separator)) 614 cLen := strings.Count(c, string(filepath.Separator)) 615 if cLen > minLen || (cLen == minLen && c < best) { 616 best = c 617 } 618 } 619 } 620 if best != "" { 621 if len(best) > len(imp) { 622 prefix := best[:len(best)-len(imp)] 623 protoPaths[prefix] = struct{}{} 624 } 625 f := protos[best] 626 f.FileDescriptorProto().Name = proto.String(imp) 627 revisedProtos[imp] = f 628 rewrittenPaths[best] = imp 629 delete(candidatesAvailable, best) 630 631 // If other candidates are actually references to the same file, remove them. 632 for c := range candidates { 633 if _, ok := candidatesAvailable[c]; !ok { 634 // already used this candidate and re-written its filename accordingly 635 continue 636 } 637 possibleDup := protos[c] 638 prevName := possibleDup.FileDescriptorProto().Name 639 possibleDup.FileDescriptorProto().Name = proto.String(imp) 640 if !proto.Equal(f.FileDescriptorProto(), protos[c].FileDescriptorProto()) { 641 // not equal: restore name and look at next one 642 possibleDup.FileDescriptorProto().Name = prevName 643 continue 644 } 645 // This file used a different name but was actually the same file. So 646 // we prune it from the set. 647 rewrittenPaths[c] = imp 648 delete(candidatesAvailable, c) 649 if len(c) > len(imp) { 650 prefix := c[:len(c)-len(imp)] 651 protoPaths[prefix] = struct{}{} 652 } 653 } 654 } 655 } 656 657 if len(candidatesAvailable) == 0 { 658 return revisedProtos, rewrittenPaths 659 } 660 661 if len(protoPaths) == 0 { 662 for c := range candidatesAvailable { 663 revisedProtos[c] = protos[c] 664 } 665 return revisedProtos, rewrittenPaths 666 } 667 668 // Any remaining candidates are entry-points (not imported by others), so 669 // the best bet to "fixing" their file name is to see if they're in one of 670 // the proto paths we found, and if so strip that prefix. 671 protoPathStrs := make([]string, len(protoPaths)) 672 i := 0 673 for p := range protoPaths { 674 protoPathStrs[i] = p 675 i++ 676 } 677 sort.Strings(protoPathStrs) 678 // we look at paths in reverse order, so we'll use a longer proto path if 679 // there is more than one match 680 for c := range candidatesAvailable { 681 var imp string 682 for i := len(protoPathStrs) - 1; i >= 0; i-- { 683 p := protoPathStrs[i] 684 if strings.HasPrefix(c, p) { 685 imp = c[len(p):] 686 break 687 } 688 } 689 if imp != "" { 690 f := protos[c] 691 f.FileDescriptorProto().Name = proto.String(imp) 692 f.FileNode() 693 revisedProtos[imp] = f 694 rewrittenPaths[c] = imp 695 } else { 696 revisedProtos[c] = protos[c] 697 } 698 } 699 700 return revisedProtos, rewrittenPaths 701 } 702 703 func removeDynamicExtensions(fd protoreflect.FileDescriptor, alreadySeen map[string]struct{}) { 704 if _, ok := alreadySeen[fd.Path()]; ok { 705 // already processed 706 return 707 } 708 alreadySeen[fd.Path()] = struct{}{} 709 res, ok := fd.(linker.Result) 710 if ok { 711 removeDynamicExtensionsFromProto(res.FileDescriptorProto()) 712 } 713 // also remove extensions from dependencies 714 for i, length := 0, fd.Imports().Len(); i < length; i++ { 715 removeDynamicExtensions(fd.Imports().Get(i).FileDescriptor, alreadySeen) 716 } 717 } 718 719 func removeDynamicExtensionsFromProto(fd *descriptorpb.FileDescriptorProto) { 720 // protocompile returns descriptors with dynamic extension fields for custom options. 721 // But protoparse only used known custom options and everything else defined in the 722 // sources would be stored as unrecognized fields. So to bridge the difference in 723 // behavior, we need to remove custom options from the given file and add them back 724 // via serializing-then-de-serializing them back into the options messages. That way, 725 // statically known options will be properly typed and others will be unrecognized. 726 // 727 // This is best effort. So if an error occurs, we'll still return a result, but it 728 // may include a dynamic extension. 729 fd.Options = removeDynamicExtensionsFromOptions(fd.Options) 730 _ = walk.DescriptorProtos(fd, func(_ protoreflect.FullName, msg proto.Message) error { 731 switch msg := msg.(type) { 732 case *descriptorpb.DescriptorProto: 733 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 734 for _, extr := range msg.ExtensionRange { 735 extr.Options = removeDynamicExtensionsFromOptions(extr.Options) 736 } 737 case *descriptorpb.FieldDescriptorProto: 738 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 739 case *descriptorpb.OneofDescriptorProto: 740 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 741 case *descriptorpb.EnumDescriptorProto: 742 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 743 case *descriptorpb.EnumValueDescriptorProto: 744 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 745 case *descriptorpb.ServiceDescriptorProto: 746 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 747 case *descriptorpb.MethodDescriptorProto: 748 msg.Options = removeDynamicExtensionsFromOptions(msg.Options) 749 } 750 return nil 751 }) 752 } 753 754 type ptrMsg[T any] interface { 755 *T 756 proto.Message 757 } 758 759 type fieldValue struct { 760 fd protoreflect.FieldDescriptor 761 val protoreflect.Value 762 } 763 764 func removeDynamicExtensionsFromOptions[O ptrMsg[T], T any](opts O) O { 765 if opts == nil { 766 return nil 767 } 768 var dynamicExtensions []fieldValue 769 opts.ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { 770 if fd.IsExtension() { 771 dynamicExtensions = append(dynamicExtensions, fieldValue{fd: fd, val: val}) 772 } 773 return true 774 }) 775 776 // serialize only these custom options 777 optsWithOnlyDyn := opts.ProtoReflect().Type().New() 778 for _, fv := range dynamicExtensions { 779 optsWithOnlyDyn.Set(fv.fd, fv.val) 780 } 781 data, err := proto.MarshalOptions{AllowPartial: true}.Marshal(optsWithOnlyDyn.Interface()) 782 if err != nil { 783 // oh, well... can't fix this one 784 return opts 785 } 786 787 // and then replace values by clearing these custom options and deserializing 788 optsClone := proto.Clone(opts).ProtoReflect() 789 for _, fv := range dynamicExtensions { 790 optsClone.Clear(fv.fd) 791 } 792 err = proto.UnmarshalOptions{AllowPartial: true, Merge: true}.Unmarshal(data, optsClone.Interface()) 793 if err != nil { 794 // bummer, can't fix this one 795 return opts 796 } 797 798 return optsClone.Interface().(O) 799 }