github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/encoding/protobuf/protobuf.go (about)

     1  // Copyright 2019 CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package protobuf defines functionality for parsing protocol buffer
    16  // definitions and instances.
    17  //
    18  // Proto definition mapping follows the guidelines of mapping Proto to JSON as
    19  // discussed in https://developers.google.com/protocol-buffers/docs/proto3, and
    20  // carries some of the mapping further when possible with CUE.
    21  //
    22  //
    23  // Package Paths
    24  //
    25  // If a .proto file contains a go_package directive, it will be used as the
    26  // destination package fo the generated .cue files. A common use case is to
    27  // generate the CUE in the same directory as the .proto definition. If a
    28  // destination package is not within the current CUE module, it will be written
    29  // relative to the pkg directory.
    30  //
    31  // If a .proto file does not specify go_package, it will convert a proto package
    32  // "google.parent.sub" to the import path "googleapis.com/google/parent/sub".
    33  // It is safe to mix package with and without a go_package within the same
    34  // project.
    35  //
    36  // Type Mappings
    37  //
    38  // The following type mappings of definitions apply:
    39  //
    40  //   Proto type     CUE type/def     Comments
    41  //   message        struct           Message fields become CUE fields, whereby
    42  //                                   names are mapped to lowerCamelCase.
    43  //   enum           e1 | e2 | ...    Where ex are strings. A separate mapping is
    44  //                                   generated to obtain the numeric values.
    45  //   map<K, V>      { <>: V }        All keys are converted to strings.
    46  //   repeated V     [...V]           null is accepted as the empty list [].
    47  //   bool           bool
    48  //   string         string
    49  //   bytes          bytes            A base64-encoded string when converted to JSON.
    50  //   int32, fixed32 int32            An integer with bounds as defined by int32.
    51  //   uint32         uint32           An integer with bounds as defined by uint32.
    52  //   int64, fixed64 int64            An integer with bounds as defined by int64.
    53  //   uint64         uint64           An integer with bounds as defined by uint64.
    54  //   float          float32          A number with bounds as defined by float32.
    55  //   double         float64          A number with bounds as defined by float64.
    56  //   Struct         struct           See struct.proto.
    57  //   Value          _                See struct.proto.
    58  //   ListValue      [...]            See struct.proto.
    59  //   NullValue      null             See struct.proto.
    60  //   BoolValue      bool             See struct.proto.
    61  //   StringValue    string           See struct.proto.
    62  //   NumberValue    number           See struct.proto.
    63  //   StringValue    string           See struct.proto.
    64  //   Empty          close({})
    65  //   Timestamp      time.Time        See struct.proto.
    66  //   Duration       time.Duration    See struct.proto.
    67  //
    68  // Protobuf definitions can be annotated with CUE constraints that are included
    69  // in the generated CUE:
    70  //    (cue.val)     string        CUE expression defining a constraint for this
    71  //                                field. The string may refer to other fields
    72  //                                in a message definition using their JSON name.
    73  //
    74  //    (cue.opt)     FieldOptions
    75  //       required   bool          Defines the field is required. Use with
    76  //                                caution.
    77  //
    78  package protobuf
    79  
    80  // TODO mappings:
    81  //
    82  // Wrapper types	various types	2, "2", "foo", true, "true", null, 0, …	Wrappers use the same representation in JSON as the wrapped primitive type, except that null is allowed and preserved during data conversion and transfer.
    83  // FieldMask	string	"f.fooBar,h"	See field_mask.proto.
    84  //   Any            {"@type":"url",  See struct.proto.
    85  //                   f1: value,
    86  //                   ...}
    87  
    88  import (
    89  	"os"
    90  	"path/filepath"
    91  	"sort"
    92  	"strings"
    93  
    94  	"github.com/mpvl/unique"
    95  
    96  	"github.com/joomcode/cue/cue/ast"
    97  	"github.com/joomcode/cue/cue/build"
    98  	"github.com/joomcode/cue/cue/errors"
    99  	"github.com/joomcode/cue/cue/format"
   100  	"github.com/joomcode/cue/cue/parser"
   101  	"github.com/joomcode/cue/cue/token"
   102  	"github.com/joomcode/cue/internal"
   103  
   104  	// Generated protobuf CUE may use builtins. Ensure that these can always be
   105  	// found, even if the user does not use cue/load or another package that
   106  	// triggers its loading.
   107  	//
   108  	// TODO: consider whether just linking in the necessary packages suffices.
   109  	// It probably does, but this may reorder some of the imports, which may,
   110  	// in turn, change the numbering, which can be confusing while debugging.
   111  	_ "github.com/joomcode/cue/pkg"
   112  )
   113  
   114  // Config specifies the environment into which to parse a proto definition file.
   115  type Config struct {
   116  	// Root specifies the root of the CUE project, which typically coincides
   117  	// with, for example, a version control repository root or the Go module.
   118  	// Any imports of proto files within the directory tree of this of this root
   119  	// are considered to be "project files" and are generated at the
   120  	// corresponding location with this hierarchy. Any other imports are
   121  	// considered to be external. Files for such imports are rooted under the
   122  	// $Root/pkg/, using the Go package path specified in the .proto file.
   123  	Root string
   124  
   125  	// Module is the Go package import path of the module root. It is the value
   126  	// as after "module" in a cue.mod/modules.cue file, if a module file is
   127  	// present.
   128  	Module string // TODO: determine automatically if unspecified.
   129  
   130  	// Paths defines the include directory in which to search for imports.
   131  	Paths []string
   132  
   133  	// PkgName specifies the package name for a generated CUE file. A value
   134  	// will be derived from the Go package name if undefined.
   135  	PkgName string
   136  
   137  	// EnumMode defines whether enums should be set as integer values, instead
   138  	// of strings.
   139  	//
   140  	//    json    value is a string, corresponding to the standard JSON mapping
   141  	//            of Protobuf. The value is associated with a #enumValue
   142  	//            to allow the json+pb interpretation to interpret integers
   143  	//            as well.
   144  	//
   145  	//    int     value is an integer associated with an #enumValue definition
   146  	//            The json+pb interpreter uses the definition names in the
   147  	//            disjunction of the enum to interpret strings.
   148  	//
   149  	EnumMode string
   150  }
   151  
   152  // An Extractor converts a collection of proto files, typically belonging to one
   153  // repo or module, to CUE. It thereby observes the CUE package layout.
   154  //
   155  // CUE observes the same package layout as Go and requires .proto files to have
   156  // the go_package directive. Generated CUE files are put in the same directory
   157  // as their corresponding .proto files if the .proto files are located in the
   158  // specified Root (or current working directory if none is specified).
   159  // All other imported files are assigned to the CUE pkg dir ($Root/pkg)
   160  // according to their Go package import path.
   161  //
   162  type Extractor struct {
   163  	root     string
   164  	cwd      string
   165  	module   string
   166  	paths    []string
   167  	pkgName  string
   168  	enumMode string
   169  
   170  	fileCache map[string]result
   171  	imports   map[string]*build.Instance
   172  
   173  	errs errors.Error
   174  	done bool
   175  }
   176  
   177  type result struct {
   178  	p   *protoConverter
   179  	err error
   180  }
   181  
   182  // NewExtractor creates an Extractor. If the configuration contained any errors
   183  // it will be observable by the Err method fo the Extractor. It is safe,
   184  // however, to only check errors after building the output.
   185  func NewExtractor(c *Config) *Extractor {
   186  	cwd, _ := os.Getwd()
   187  	b := &Extractor{
   188  		root:      c.Root,
   189  		cwd:       cwd,
   190  		paths:     c.Paths,
   191  		pkgName:   c.PkgName,
   192  		module:    c.Module,
   193  		enumMode:  c.EnumMode,
   194  		fileCache: map[string]result{},
   195  		imports:   map[string]*build.Instance{},
   196  	}
   197  
   198  	if b.root == "" {
   199  		b.root = b.cwd
   200  	}
   201  
   202  	return b
   203  }
   204  
   205  // Err returns the errors accumulated during testing. The returned error may be
   206  // of type github.com/joomcode/cue/cue/errors.List.
   207  func (b *Extractor) Err() error {
   208  	return b.errs
   209  }
   210  
   211  func (b *Extractor) addErr(err error) {
   212  	b.errs = errors.Append(b.errs, errors.Promote(err, "unknown error"))
   213  }
   214  
   215  // AddFile adds a proto definition file to be converted into CUE by the builder.
   216  // Relatives paths are always taken relative to the Root with which the b is
   217  // configured.
   218  //
   219  // AddFile assumes that the proto file compiles with protoc and may not report
   220  // an error if it does not. Imports are resolved using the paths defined in
   221  // Config.
   222  //
   223  func (b *Extractor) AddFile(filename string, src interface{}) error {
   224  	if b.done {
   225  		err := errors.Newf(token.NoPos,
   226  			"protobuf: cannot call AddFile: Instances was already called")
   227  		b.errs = errors.Append(b.errs, err)
   228  		return err
   229  	}
   230  	if b.root != b.cwd && !filepath.IsAbs(filename) {
   231  		filename = filepath.Join(b.root, filename)
   232  	}
   233  	_, err := b.parse(filename, src)
   234  	return err
   235  }
   236  
   237  // TODO: some way of (recursively) adding multiple proto files with filter.
   238  
   239  // Files returns a File for each proto file that was added or imported,
   240  // recursively.
   241  func (b *Extractor) Files() (files []*ast.File, err error) {
   242  	defer func() { err = b.Err() }()
   243  	b.done = true
   244  
   245  	instances, err := b.Instances()
   246  	if err != nil {
   247  		return nil, err
   248  	}
   249  
   250  	for _, p := range instances {
   251  		for _, f := range p.Files {
   252  			files = append(files, f)
   253  		}
   254  	}
   255  	return files, nil
   256  }
   257  
   258  // Instances creates a build.Instances for every package for which a proto file
   259  // was added to the builder. This includes transitive dependencies. It does not
   260  // write the generated files to disk.
   261  //
   262  // The returned instances can be passed to cue.Build to generated the
   263  // corresponding CUE instances.
   264  //
   265  // All import paths are located within the specified Root, where external
   266  // packages are located under $Root/pkg. Instances for builtin (like time)
   267  // packages may be omitted, and if not will have no associated files.
   268  func (b *Extractor) Instances() (instances []*build.Instance, err error) {
   269  	defer func() { err = b.Err() }()
   270  	b.done = true
   271  
   272  	for _, r := range b.fileCache {
   273  		if r.err != nil {
   274  			b.addErr(r.err)
   275  			continue
   276  		}
   277  		inst := b.getInst(r.p)
   278  		if inst == nil {
   279  			continue
   280  		}
   281  
   282  		// Set canonical CUE path for generated file.
   283  		f := r.p.file
   284  		base := filepath.Base(f.Filename)
   285  		base = base[:len(base)-len(".proto")] + "_proto_gen.cue"
   286  		f.Filename = filepath.Join(inst.Dir, base)
   287  		buf, err := format.Node(f)
   288  		if err != nil {
   289  			b.addErr(err)
   290  			// return nil, err
   291  			continue
   292  		}
   293  		f, err = parser.ParseFile(f.Filename, buf, parser.ParseComments)
   294  		if err != nil {
   295  			b.addErr(err)
   296  			continue
   297  		}
   298  
   299  		inst.Files = append(inst.Files, f)
   300  
   301  		for pkg := range r.p.imported {
   302  			inst.ImportPaths = append(inst.ImportPaths, pkg)
   303  		}
   304  	}
   305  
   306  	for _, p := range b.imports {
   307  		instances = append(instances, p)
   308  		sort.Strings(p.ImportPaths)
   309  		unique.Strings(&p.ImportPaths)
   310  		for _, i := range p.ImportPaths {
   311  			if imp := b.imports[i]; imp != nil {
   312  				p.Imports = append(p.Imports, imp)
   313  			}
   314  		}
   315  
   316  		sort.Slice(p.Files, func(i, j int) bool {
   317  			return p.Files[i].Filename < p.Files[j].Filename
   318  		})
   319  	}
   320  	sort.Slice(instances, func(i, j int) bool {
   321  		return instances[i].ImportPath < instances[j].ImportPath
   322  	})
   323  
   324  	if err != nil {
   325  		return instances, err
   326  	}
   327  	return instances, nil
   328  }
   329  
   330  func (b *Extractor) getInst(p *protoConverter) *build.Instance {
   331  	if b.errs != nil {
   332  		return nil
   333  	}
   334  	importPath := p.qualifiedImportPath()
   335  	if importPath == "" {
   336  		err := errors.Newf(token.NoPos,
   337  			"no package clause for proto package %q in file %s", p.id, p.file.Filename)
   338  		b.errs = errors.Append(b.errs, err)
   339  		// TODO: find an alternative. Is proto package good enough?
   340  		return nil
   341  	}
   342  
   343  	dir := b.root
   344  	path := p.importPath()
   345  	file := p.file.Filename
   346  	if !filepath.IsAbs(file) {
   347  		file = filepath.Join(b.root, p.file.Filename)
   348  	}
   349  	// Determine whether the generated file should be included in place, or
   350  	// within cue.mod.
   351  	inPlace := strings.HasPrefix(file, b.root)
   352  	if !strings.HasPrefix(path, b.module) {
   353  		// b.module is either "", in which case we assume the setting for
   354  		// inPlace, or not, in which case the module in the protobuf must
   355  		// correspond with that of the proto package.
   356  		inPlace = false
   357  	}
   358  	if !inPlace {
   359  		dir = filepath.Join(internal.GenPath(dir), path)
   360  	} else {
   361  		dir = filepath.Dir(p.file.Filename)
   362  	}
   363  
   364  	// TODO: verify module name from go_package option against that of actual
   365  	// CUE module. Maybe keep this old code for some strict mode?
   366  	// want := filepath.Dir(p.file.Filename)
   367  	// dir = filepath.Join(dir, path[len(b.module)+1:])
   368  	// if !filepath.IsAbs(want) {
   369  	// 	want = filepath.Join(b.root, want)
   370  	// }
   371  	// if dir != want {
   372  	// 	err := errors.Newf(token.NoPos,
   373  	// 		"file %s mapped to inconsistent path %s; module name %q may be inconsistent with root dir %s",
   374  	// 		want, dir, b.module, b.root,
   375  	// 	)
   376  	// 	b.errs = errors.Append(b.errs, err)
   377  	// }
   378  
   379  	inst := b.imports[importPath]
   380  	if inst == nil {
   381  		inst = &build.Instance{
   382  			Root:        b.root,
   383  			Dir:         dir,
   384  			ImportPath:  importPath,
   385  			PkgName:     p.shortPkgName,
   386  			DisplayPath: p.protoPkg,
   387  		}
   388  		b.imports[importPath] = inst
   389  	}
   390  	return inst
   391  }
   392  
   393  // Extract parses a single proto file and returns its contents translated to a CUE
   394  // file. If src is not nil, it will use this as the contents of the file. It may
   395  // be a string, []byte or io.Reader. Otherwise Extract will open the given file
   396  // name at the fully qualified path.
   397  //
   398  // Extract assumes the proto file compiles with protoc and may not report an error
   399  // if it does not. Imports are resolved using the paths defined in Config.
   400  //
   401  func Extract(filename string, src interface{}, c *Config) (f *ast.File, err error) {
   402  	if c == nil {
   403  		c = &Config{}
   404  	}
   405  	b := NewExtractor(c)
   406  
   407  	p, err := b.parse(filename, src)
   408  	if err != nil {
   409  		return nil, err
   410  	}
   411  	p.file.Filename = filename[:len(filename)-len(".proto")] + "_gen.cue"
   412  	return p.file, b.Err()
   413  }
   414  
   415  // TODO
   416  // func GenDefinition
   417  
   418  // func MarshalText(cue.Value) (string, error) {
   419  // 	return "", nil
   420  // }
   421  
   422  // func MarshalBytes(cue.Value) ([]byte, error) {
   423  // 	return nil, nil
   424  // }
   425  
   426  // func UnmarshalText(descriptor cue.Value, b string) (ast.Expr, error) {
   427  // 	return nil, nil
   428  // }
   429  
   430  // func UnmarshalBytes(descriptor cue.Value, b []byte) (ast.Expr, error) {
   431  // 	return nil, nil
   432  // }