github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/encoding/protobuf/protobuf.go (about) 1 // Copyright 2019 CUE Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package protobuf defines functionality for parsing protocol buffer 16 // definitions and instances. 17 // 18 // Proto definition mapping follows the guidelines of mapping Proto to JSON as 19 // discussed in https://developers.google.com/protocol-buffers/docs/proto3, and 20 // carries some of the mapping further when possible with CUE. 21 // 22 // 23 // Package Paths 24 // 25 // If a .proto file contains a go_package directive, it will be used as the 26 // destination package fo the generated .cue files. A common use case is to 27 // generate the CUE in the same directory as the .proto definition. If a 28 // destination package is not within the current CUE module, it will be written 29 // relative to the pkg directory. 30 // 31 // If a .proto file does not specify go_package, it will convert a proto package 32 // "google.parent.sub" to the import path "googleapis.com/google/parent/sub". 33 // It is safe to mix package with and without a go_package within the same 34 // project. 35 // 36 // Type Mappings 37 // 38 // The following type mappings of definitions apply: 39 // 40 // Proto type CUE type/def Comments 41 // message struct Message fields become CUE fields, whereby 42 // names are mapped to lowerCamelCase. 43 // enum e1 | e2 | ... Where ex are strings. A separate mapping is 44 // generated to obtain the numeric values. 45 // map<K, V> { <>: V } All keys are converted to strings. 46 // repeated V [...V] null is accepted as the empty list []. 47 // bool bool 48 // string string 49 // bytes bytes A base64-encoded string when converted to JSON. 50 // int32, fixed32 int32 An integer with bounds as defined by int32. 51 // uint32 uint32 An integer with bounds as defined by uint32. 52 // int64, fixed64 int64 An integer with bounds as defined by int64. 53 // uint64 uint64 An integer with bounds as defined by uint64. 54 // float float32 A number with bounds as defined by float32. 55 // double float64 A number with bounds as defined by float64. 56 // Struct struct See struct.proto. 57 // Value _ See struct.proto. 58 // ListValue [...] See struct.proto. 59 // NullValue null See struct.proto. 60 // BoolValue bool See struct.proto. 61 // StringValue string See struct.proto. 62 // NumberValue number See struct.proto. 63 // StringValue string See struct.proto. 64 // Empty close({}) 65 // Timestamp time.Time See struct.proto. 66 // Duration time.Duration See struct.proto. 67 // 68 // Protobuf definitions can be annotated with CUE constraints that are included 69 // in the generated CUE: 70 // (cue.val) string CUE expression defining a constraint for this 71 // field. The string may refer to other fields 72 // in a message definition using their JSON name. 73 // 74 // (cue.opt) FieldOptions 75 // required bool Defines the field is required. Use with 76 // caution. 77 // 78 package protobuf 79 80 // TODO mappings: 81 // 82 // Wrapper types various types 2, "2", "foo", true, "true", null, 0, … Wrappers use the same representation in JSON as the wrapped primitive type, except that null is allowed and preserved during data conversion and transfer. 83 // FieldMask string "f.fooBar,h" See field_mask.proto. 84 // Any {"@type":"url", See struct.proto. 85 // f1: value, 86 // ...} 87 88 import ( 89 "os" 90 "path/filepath" 91 "sort" 92 "strings" 93 94 "github.com/mpvl/unique" 95 96 "github.com/joomcode/cue/cue/ast" 97 "github.com/joomcode/cue/cue/build" 98 "github.com/joomcode/cue/cue/errors" 99 "github.com/joomcode/cue/cue/format" 100 "github.com/joomcode/cue/cue/parser" 101 "github.com/joomcode/cue/cue/token" 102 "github.com/joomcode/cue/internal" 103 104 // Generated protobuf CUE may use builtins. Ensure that these can always be 105 // found, even if the user does not use cue/load or another package that 106 // triggers its loading. 107 // 108 // TODO: consider whether just linking in the necessary packages suffices. 109 // It probably does, but this may reorder some of the imports, which may, 110 // in turn, change the numbering, which can be confusing while debugging. 111 _ "github.com/joomcode/cue/pkg" 112 ) 113 114 // Config specifies the environment into which to parse a proto definition file. 115 type Config struct { 116 // Root specifies the root of the CUE project, which typically coincides 117 // with, for example, a version control repository root or the Go module. 118 // Any imports of proto files within the directory tree of this of this root 119 // are considered to be "project files" and are generated at the 120 // corresponding location with this hierarchy. Any other imports are 121 // considered to be external. Files for such imports are rooted under the 122 // $Root/pkg/, using the Go package path specified in the .proto file. 123 Root string 124 125 // Module is the Go package import path of the module root. It is the value 126 // as after "module" in a cue.mod/modules.cue file, if a module file is 127 // present. 128 Module string // TODO: determine automatically if unspecified. 129 130 // Paths defines the include directory in which to search for imports. 131 Paths []string 132 133 // PkgName specifies the package name for a generated CUE file. A value 134 // will be derived from the Go package name if undefined. 135 PkgName string 136 137 // EnumMode defines whether enums should be set as integer values, instead 138 // of strings. 139 // 140 // json value is a string, corresponding to the standard JSON mapping 141 // of Protobuf. The value is associated with a #enumValue 142 // to allow the json+pb interpretation to interpret integers 143 // as well. 144 // 145 // int value is an integer associated with an #enumValue definition 146 // The json+pb interpreter uses the definition names in the 147 // disjunction of the enum to interpret strings. 148 // 149 EnumMode string 150 } 151 152 // An Extractor converts a collection of proto files, typically belonging to one 153 // repo or module, to CUE. It thereby observes the CUE package layout. 154 // 155 // CUE observes the same package layout as Go and requires .proto files to have 156 // the go_package directive. Generated CUE files are put in the same directory 157 // as their corresponding .proto files if the .proto files are located in the 158 // specified Root (or current working directory if none is specified). 159 // All other imported files are assigned to the CUE pkg dir ($Root/pkg) 160 // according to their Go package import path. 161 // 162 type Extractor struct { 163 root string 164 cwd string 165 module string 166 paths []string 167 pkgName string 168 enumMode string 169 170 fileCache map[string]result 171 imports map[string]*build.Instance 172 173 errs errors.Error 174 done bool 175 } 176 177 type result struct { 178 p *protoConverter 179 err error 180 } 181 182 // NewExtractor creates an Extractor. If the configuration contained any errors 183 // it will be observable by the Err method fo the Extractor. It is safe, 184 // however, to only check errors after building the output. 185 func NewExtractor(c *Config) *Extractor { 186 cwd, _ := os.Getwd() 187 b := &Extractor{ 188 root: c.Root, 189 cwd: cwd, 190 paths: c.Paths, 191 pkgName: c.PkgName, 192 module: c.Module, 193 enumMode: c.EnumMode, 194 fileCache: map[string]result{}, 195 imports: map[string]*build.Instance{}, 196 } 197 198 if b.root == "" { 199 b.root = b.cwd 200 } 201 202 return b 203 } 204 205 // Err returns the errors accumulated during testing. The returned error may be 206 // of type github.com/joomcode/cue/cue/errors.List. 207 func (b *Extractor) Err() error { 208 return b.errs 209 } 210 211 func (b *Extractor) addErr(err error) { 212 b.errs = errors.Append(b.errs, errors.Promote(err, "unknown error")) 213 } 214 215 // AddFile adds a proto definition file to be converted into CUE by the builder. 216 // Relatives paths are always taken relative to the Root with which the b is 217 // configured. 218 // 219 // AddFile assumes that the proto file compiles with protoc and may not report 220 // an error if it does not. Imports are resolved using the paths defined in 221 // Config. 222 // 223 func (b *Extractor) AddFile(filename string, src interface{}) error { 224 if b.done { 225 err := errors.Newf(token.NoPos, 226 "protobuf: cannot call AddFile: Instances was already called") 227 b.errs = errors.Append(b.errs, err) 228 return err 229 } 230 if b.root != b.cwd && !filepath.IsAbs(filename) { 231 filename = filepath.Join(b.root, filename) 232 } 233 _, err := b.parse(filename, src) 234 return err 235 } 236 237 // TODO: some way of (recursively) adding multiple proto files with filter. 238 239 // Files returns a File for each proto file that was added or imported, 240 // recursively. 241 func (b *Extractor) Files() (files []*ast.File, err error) { 242 defer func() { err = b.Err() }() 243 b.done = true 244 245 instances, err := b.Instances() 246 if err != nil { 247 return nil, err 248 } 249 250 for _, p := range instances { 251 for _, f := range p.Files { 252 files = append(files, f) 253 } 254 } 255 return files, nil 256 } 257 258 // Instances creates a build.Instances for every package for which a proto file 259 // was added to the builder. This includes transitive dependencies. It does not 260 // write the generated files to disk. 261 // 262 // The returned instances can be passed to cue.Build to generated the 263 // corresponding CUE instances. 264 // 265 // All import paths are located within the specified Root, where external 266 // packages are located under $Root/pkg. Instances for builtin (like time) 267 // packages may be omitted, and if not will have no associated files. 268 func (b *Extractor) Instances() (instances []*build.Instance, err error) { 269 defer func() { err = b.Err() }() 270 b.done = true 271 272 for _, r := range b.fileCache { 273 if r.err != nil { 274 b.addErr(r.err) 275 continue 276 } 277 inst := b.getInst(r.p) 278 if inst == nil { 279 continue 280 } 281 282 // Set canonical CUE path for generated file. 283 f := r.p.file 284 base := filepath.Base(f.Filename) 285 base = base[:len(base)-len(".proto")] + "_proto_gen.cue" 286 f.Filename = filepath.Join(inst.Dir, base) 287 buf, err := format.Node(f) 288 if err != nil { 289 b.addErr(err) 290 // return nil, err 291 continue 292 } 293 f, err = parser.ParseFile(f.Filename, buf, parser.ParseComments) 294 if err != nil { 295 b.addErr(err) 296 continue 297 } 298 299 inst.Files = append(inst.Files, f) 300 301 for pkg := range r.p.imported { 302 inst.ImportPaths = append(inst.ImportPaths, pkg) 303 } 304 } 305 306 for _, p := range b.imports { 307 instances = append(instances, p) 308 sort.Strings(p.ImportPaths) 309 unique.Strings(&p.ImportPaths) 310 for _, i := range p.ImportPaths { 311 if imp := b.imports[i]; imp != nil { 312 p.Imports = append(p.Imports, imp) 313 } 314 } 315 316 sort.Slice(p.Files, func(i, j int) bool { 317 return p.Files[i].Filename < p.Files[j].Filename 318 }) 319 } 320 sort.Slice(instances, func(i, j int) bool { 321 return instances[i].ImportPath < instances[j].ImportPath 322 }) 323 324 if err != nil { 325 return instances, err 326 } 327 return instances, nil 328 } 329 330 func (b *Extractor) getInst(p *protoConverter) *build.Instance { 331 if b.errs != nil { 332 return nil 333 } 334 importPath := p.qualifiedImportPath() 335 if importPath == "" { 336 err := errors.Newf(token.NoPos, 337 "no package clause for proto package %q in file %s", p.id, p.file.Filename) 338 b.errs = errors.Append(b.errs, err) 339 // TODO: find an alternative. Is proto package good enough? 340 return nil 341 } 342 343 dir := b.root 344 path := p.importPath() 345 file := p.file.Filename 346 if !filepath.IsAbs(file) { 347 file = filepath.Join(b.root, p.file.Filename) 348 } 349 // Determine whether the generated file should be included in place, or 350 // within cue.mod. 351 inPlace := strings.HasPrefix(file, b.root) 352 if !strings.HasPrefix(path, b.module) { 353 // b.module is either "", in which case we assume the setting for 354 // inPlace, or not, in which case the module in the protobuf must 355 // correspond with that of the proto package. 356 inPlace = false 357 } 358 if !inPlace { 359 dir = filepath.Join(internal.GenPath(dir), path) 360 } else { 361 dir = filepath.Dir(p.file.Filename) 362 } 363 364 // TODO: verify module name from go_package option against that of actual 365 // CUE module. Maybe keep this old code for some strict mode? 366 // want := filepath.Dir(p.file.Filename) 367 // dir = filepath.Join(dir, path[len(b.module)+1:]) 368 // if !filepath.IsAbs(want) { 369 // want = filepath.Join(b.root, want) 370 // } 371 // if dir != want { 372 // err := errors.Newf(token.NoPos, 373 // "file %s mapped to inconsistent path %s; module name %q may be inconsistent with root dir %s", 374 // want, dir, b.module, b.root, 375 // ) 376 // b.errs = errors.Append(b.errs, err) 377 // } 378 379 inst := b.imports[importPath] 380 if inst == nil { 381 inst = &build.Instance{ 382 Root: b.root, 383 Dir: dir, 384 ImportPath: importPath, 385 PkgName: p.shortPkgName, 386 DisplayPath: p.protoPkg, 387 } 388 b.imports[importPath] = inst 389 } 390 return inst 391 } 392 393 // Extract parses a single proto file and returns its contents translated to a CUE 394 // file. If src is not nil, it will use this as the contents of the file. It may 395 // be a string, []byte or io.Reader. Otherwise Extract will open the given file 396 // name at the fully qualified path. 397 // 398 // Extract assumes the proto file compiles with protoc and may not report an error 399 // if it does not. Imports are resolved using the paths defined in Config. 400 // 401 func Extract(filename string, src interface{}, c *Config) (f *ast.File, err error) { 402 if c == nil { 403 c = &Config{} 404 } 405 b := NewExtractor(c) 406 407 p, err := b.parse(filename, src) 408 if err != nil { 409 return nil, err 410 } 411 p.file.Filename = filename[:len(filename)-len(".proto")] + "_gen.cue" 412 return p.file, b.Err() 413 } 414 415 // TODO 416 // func GenDefinition 417 418 // func MarshalText(cue.Value) (string, error) { 419 // return "", nil 420 // } 421 422 // func MarshalBytes(cue.Value) ([]byte, error) { 423 // return nil, nil 424 // } 425 426 // func UnmarshalText(descriptor cue.Value, b string) (ast.Expr, error) { 427 // return nil, nil 428 // } 429 430 // func UnmarshalBytes(descriptor cue.Value, b []byte) (ast.Expr, error) { 431 // return nil, nil 432 // }