github.com/matm/etcd@v0.3.1-0.20140328024009-5b4a473f1453/third_party/code.google.com/p/gogoprotobuf/proto/properties.go (about) 1 // Extensions for Protocol Buffers to create more go like structures. 2 // 3 // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 // http://code.google.com/p/gogoprotobuf/gogoproto 5 // 6 // Go support for Protocol Buffers - Google's data interchange format 7 // 8 // Copyright 2010 The Go Authors. All rights reserved. 9 // http://code.google.com/p/goprotobuf/ 10 // 11 // Redistribution and use in source and binary forms, with or without 12 // modification, are permitted provided that the following conditions are 13 // met: 14 // 15 // * Redistributions of source code must retain the above copyright 16 // notice, this list of conditions and the following disclaimer. 17 // * Redistributions in binary form must reproduce the above 18 // copyright notice, this list of conditions and the following disclaimer 19 // in the documentation and/or other materials provided with the 20 // distribution. 21 // * Neither the name of Google Inc. nor the names of its 22 // contributors may be used to endorse or promote products derived from 23 // this software without specific prior written permission. 24 // 25 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 37 package proto 38 39 /* 40 * Routines for encoding data into the wire format for protocol buffers. 41 */ 42 43 import ( 44 "fmt" 45 "os" 46 "reflect" 47 "sort" 48 "strconv" 49 "strings" 50 "sync" 51 ) 52 53 const debug bool = false 54 55 // Constants that identify the encoding of a value on the wire. 56 const ( 57 WireVarint = 0 58 WireFixed64 = 1 59 WireBytes = 2 60 WireStartGroup = 3 61 WireEndGroup = 4 62 WireFixed32 = 5 63 ) 64 65 const startSize = 10 // initial slice/string sizes 66 67 // Encoders are defined in encode.go 68 // An encoder outputs the full representation of a field, including its 69 // tag and encoder type. 70 type encoder func(p *Buffer, prop *Properties, base structPointer) error 71 72 // A valueEncoder encodes a single integer in a particular encoding. 73 type valueEncoder func(o *Buffer, x uint64) error 74 75 // Sizers are defined in encode.go 76 // A sizer returns the encoded size of a field, including its tag and encoder 77 // type. 78 type sizer func(prop *Properties, base structPointer) int 79 80 // A valueSizer returns the encoded size of a single integer in a particular 81 // encoding. 82 type valueSizer func(x uint64) int 83 84 // Decoders are defined in decode.go 85 // A decoder creates a value from its wire representation. 86 // Unrecognized subelements are saved in unrec. 87 type decoder func(p *Buffer, prop *Properties, base structPointer) error 88 89 // A valueDecoder decodes a single integer in a particular encoding. 90 type valueDecoder func(o *Buffer) (x uint64, err error) 91 92 // tagMap is an optimization over map[int]int for typical protocol buffer 93 // use-cases. Encoded protocol buffers are often in tag order with small tag 94 // numbers. 95 type tagMap struct { 96 fastTags []int 97 slowTags map[int]int 98 } 99 100 // tagMapFastLimit is the upper bound on the tag number that will be stored in 101 // the tagMap slice rather than its map. 102 const tagMapFastLimit = 1024 103 104 func (p *tagMap) get(t int) (int, bool) { 105 if t > 0 && t < tagMapFastLimit { 106 if t >= len(p.fastTags) { 107 return 0, false 108 } 109 fi := p.fastTags[t] 110 return fi, fi >= 0 111 } 112 fi, ok := p.slowTags[t] 113 return fi, ok 114 } 115 116 func (p *tagMap) put(t int, fi int) { 117 if t > 0 && t < tagMapFastLimit { 118 for len(p.fastTags) < t+1 { 119 p.fastTags = append(p.fastTags, -1) 120 } 121 p.fastTags[t] = fi 122 return 123 } 124 if p.slowTags == nil { 125 p.slowTags = make(map[int]int) 126 } 127 p.slowTags[t] = fi 128 } 129 130 // StructProperties represents properties for all the fields of a struct. 131 // decoderTags and decoderOrigNames should only be used by the decoder. 132 type StructProperties struct { 133 Prop []*Properties // properties for each field 134 reqCount int // required count 135 decoderTags tagMap // map from proto tag to struct field number 136 decoderOrigNames map[string]int // map from original name to struct field number 137 order []int // list of struct field numbers in tag order 138 unrecField field // field id of the XXX_unrecognized []byte field 139 extendable bool // is this an extendable proto 140 } 141 142 // Implement the sorting interface so we can sort the fields in tag order, as recommended by the spec. 143 // See encode.go, (*Buffer).enc_struct. 144 145 func (sp *StructProperties) Len() int { return len(sp.order) } 146 func (sp *StructProperties) Less(i, j int) bool { 147 return sp.Prop[sp.order[i]].Tag < sp.Prop[sp.order[j]].Tag 148 } 149 func (sp *StructProperties) Swap(i, j int) { sp.order[i], sp.order[j] = sp.order[j], sp.order[i] } 150 151 // Properties represents the protocol-specific behavior of a single struct field. 152 type Properties struct { 153 Name string // name of the field, for error messages 154 OrigName string // original name before protocol compiler (always set) 155 Wire string 156 WireType int 157 Tag int 158 Required bool 159 Optional bool 160 Repeated bool 161 Packed bool // relevant for repeated primitives only 162 Enum string // set for enum types only 163 Default string // default value 164 CustomType string 165 def_uint64 uint64 166 167 enc encoder 168 valEnc valueEncoder // set for bool and numeric types only 169 field field 170 tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType) 171 tagbuf [8]byte 172 stype reflect.Type // set for struct types only 173 sstype reflect.Type // set for slices of structs types only 174 ctype reflect.Type // set for custom types only 175 sprop *StructProperties // set for struct types only 176 isMarshaler bool 177 isUnmarshaler bool 178 179 size sizer 180 valSize valueSizer // set for bool and numeric types only 181 182 dec decoder 183 valDec valueDecoder // set for bool and numeric types only 184 185 // If this is a packable field, this will be the decoder for the packed version of the field. 186 packedDec decoder 187 } 188 189 // String formats the properties in the protobuf struct field tag style. 190 func (p *Properties) String() string { 191 s := p.Wire 192 s = "," 193 s += strconv.Itoa(p.Tag) 194 if p.Required { 195 s += ",req" 196 } 197 if p.Optional { 198 s += ",opt" 199 } 200 if p.Repeated { 201 s += ",rep" 202 } 203 if p.Packed { 204 s += ",packed" 205 } 206 if p.OrigName != p.Name { 207 s += ",name=" + p.OrigName 208 } 209 if len(p.Enum) > 0 { 210 s += ",enum=" + p.Enum 211 } 212 if len(p.Default) > 0 { 213 s += ",def=" + p.Default 214 } 215 return s 216 } 217 218 // Parse populates p by parsing a string in the protobuf struct field tag style. 219 func (p *Properties) Parse(s string) { 220 // "bytes,49,opt,name=foo,def=hello!" 221 fields := strings.Split(s, ",") // breaks def=, but handled below. 222 if len(fields) < 2 { 223 fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s) 224 return 225 } 226 227 p.Wire = fields[0] 228 switch p.Wire { 229 case "varint": 230 p.WireType = WireVarint 231 p.valEnc = (*Buffer).EncodeVarint 232 p.valDec = (*Buffer).DecodeVarint 233 p.valSize = sizeVarint 234 case "fixed32": 235 p.WireType = WireFixed32 236 p.valEnc = (*Buffer).EncodeFixed32 237 p.valDec = (*Buffer).DecodeFixed32 238 p.valSize = sizeFixed32 239 case "fixed64": 240 p.WireType = WireFixed64 241 p.valEnc = (*Buffer).EncodeFixed64 242 p.valDec = (*Buffer).DecodeFixed64 243 p.valSize = sizeFixed64 244 case "zigzag32": 245 p.WireType = WireVarint 246 p.valEnc = (*Buffer).EncodeZigzag32 247 p.valDec = (*Buffer).DecodeZigzag32 248 p.valSize = sizeZigzag32 249 case "zigzag64": 250 p.WireType = WireVarint 251 p.valEnc = (*Buffer).EncodeZigzag64 252 p.valDec = (*Buffer).DecodeZigzag64 253 p.valSize = sizeZigzag64 254 case "bytes", "group": 255 p.WireType = WireBytes 256 // no numeric converter for non-numeric types 257 default: 258 fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s) 259 return 260 } 261 262 var err error 263 p.Tag, err = strconv.Atoi(fields[1]) 264 if err != nil { 265 return 266 } 267 268 for i := 2; i < len(fields); i++ { 269 f := fields[i] 270 switch { 271 case f == "req": 272 p.Required = true 273 case f == "opt": 274 p.Optional = true 275 case f == "rep": 276 p.Repeated = true 277 case f == "packed": 278 p.Packed = true 279 case strings.HasPrefix(f, "name="): 280 p.OrigName = f[5:] 281 case strings.HasPrefix(f, "enum="): 282 p.Enum = f[5:] 283 case strings.HasPrefix(f, "def="): 284 p.Default = f[4:] // rest of string 285 if i+1 < len(fields) { 286 // Commas aren't escaped, and def is always last. 287 p.Default += "," + strings.Join(fields[i+1:], ",") 288 break 289 } 290 case strings.HasPrefix(f, "embedded="): 291 p.OrigName = strings.Split(f, "=")[1] 292 case strings.HasPrefix(f, "customtype="): 293 p.CustomType = strings.Split(f, "=")[1] 294 } 295 } 296 } 297 298 func logNoSliceEnc(t1, t2 reflect.Type) { 299 fmt.Fprintf(os.Stderr, "proto: no slice oenc for %T = []%T\n", t1, t2) 300 } 301 302 var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() 303 304 // Initialize the fields for encoding and decoding. 305 func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { 306 p.enc = nil 307 p.dec = nil 308 p.size = nil 309 if len(p.CustomType) > 0 { 310 p.setCustomEncAndDec(typ) 311 p.setTag(lockGetProp) 312 return 313 } 314 switch t1 := typ; t1.Kind() { 315 default: 316 if !p.setNonNullableEncAndDec(t1) { 317 fmt.Fprintf(os.Stderr, "proto: no coders for %T\n", t1) 318 } 319 case reflect.Ptr: 320 switch t2 := t1.Elem(); t2.Kind() { 321 default: 322 fmt.Fprintf(os.Stderr, "proto: no encoder function for %T -> %T\n", t1, t2) 323 break 324 case reflect.Bool: 325 p.enc = (*Buffer).enc_bool 326 p.dec = (*Buffer).dec_bool 327 p.size = size_bool 328 case reflect.Int32, reflect.Uint32: 329 p.enc = (*Buffer).enc_int32 330 p.dec = (*Buffer).dec_int32 331 p.size = size_int32 332 case reflect.Int64, reflect.Uint64: 333 p.enc = (*Buffer).enc_int64 334 p.dec = (*Buffer).dec_int64 335 p.size = size_int64 336 case reflect.Float32: 337 p.enc = (*Buffer).enc_int32 // can just treat them as bits 338 p.dec = (*Buffer).dec_int32 339 p.size = size_int32 340 case reflect.Float64: 341 p.enc = (*Buffer).enc_int64 // can just treat them as bits 342 p.dec = (*Buffer).dec_int64 343 p.size = size_int64 344 case reflect.String: 345 p.enc = (*Buffer).enc_string 346 p.dec = (*Buffer).dec_string 347 p.size = size_string 348 case reflect.Struct: 349 p.stype = t1.Elem() 350 p.isMarshaler = isMarshaler(t1) 351 p.isUnmarshaler = isUnmarshaler(t1) 352 if p.Wire == "bytes" { 353 p.enc = (*Buffer).enc_struct_message 354 p.dec = (*Buffer).dec_struct_message 355 p.size = size_struct_message 356 } else { 357 p.enc = (*Buffer).enc_struct_group 358 p.dec = (*Buffer).dec_struct_group 359 p.size = size_struct_group 360 } 361 } 362 363 case reflect.Slice: 364 switch t2 := t1.Elem(); t2.Kind() { 365 default: 366 logNoSliceEnc(t1, t2) 367 break 368 case reflect.Bool: 369 if p.Packed { 370 p.enc = (*Buffer).enc_slice_packed_bool 371 p.size = size_slice_packed_bool 372 } else { 373 p.enc = (*Buffer).enc_slice_bool 374 p.size = size_slice_bool 375 } 376 p.dec = (*Buffer).dec_slice_bool 377 p.packedDec = (*Buffer).dec_slice_packed_bool 378 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: 379 switch t2.Bits() { 380 case 32: 381 if p.Packed { 382 p.enc = (*Buffer).enc_slice_packed_int32 383 p.size = size_slice_packed_int32 384 } else { 385 p.enc = (*Buffer).enc_slice_int32 386 p.size = size_slice_int32 387 } 388 p.dec = (*Buffer).dec_slice_int32 389 p.packedDec = (*Buffer).dec_slice_packed_int32 390 case 64: 391 if p.Packed { 392 p.enc = (*Buffer).enc_slice_packed_int64 393 p.size = size_slice_packed_int64 394 } else { 395 p.enc = (*Buffer).enc_slice_int64 396 p.size = size_slice_int64 397 } 398 p.dec = (*Buffer).dec_slice_int64 399 p.packedDec = (*Buffer).dec_slice_packed_int64 400 case 8: 401 if t2.Kind() == reflect.Uint8 { 402 p.enc = (*Buffer).enc_slice_byte 403 p.dec = (*Buffer).dec_slice_byte 404 p.size = size_slice_byte 405 } 406 default: 407 logNoSliceEnc(t1, t2) 408 break 409 } 410 case reflect.Float32, reflect.Float64: 411 switch t2.Bits() { 412 case 32: 413 // can just treat them as bits 414 if p.Packed { 415 p.enc = (*Buffer).enc_slice_packed_int32 416 p.size = size_slice_packed_int32 417 } else { 418 p.enc = (*Buffer).enc_slice_int32 419 p.size = size_slice_int32 420 } 421 p.dec = (*Buffer).dec_slice_int32 422 p.packedDec = (*Buffer).dec_slice_packed_int32 423 case 64: 424 // can just treat them as bits 425 if p.Packed { 426 p.enc = (*Buffer).enc_slice_packed_int64 427 p.size = size_slice_packed_int64 428 } else { 429 p.enc = (*Buffer).enc_slice_int64 430 p.size = size_slice_int64 431 } 432 p.dec = (*Buffer).dec_slice_int64 433 p.packedDec = (*Buffer).dec_slice_packed_int64 434 default: 435 logNoSliceEnc(t1, t2) 436 break 437 } 438 case reflect.String: 439 p.enc = (*Buffer).enc_slice_string 440 p.dec = (*Buffer).dec_slice_string 441 p.size = size_slice_string 442 case reflect.Ptr: 443 switch t3 := t2.Elem(); t3.Kind() { 444 default: 445 fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T -> %T\n", t1, t2, t3) 446 break 447 case reflect.Struct: 448 p.stype = t2.Elem() 449 p.isMarshaler = isMarshaler(t2) 450 p.isUnmarshaler = isUnmarshaler(t2) 451 if p.Wire == "bytes" { 452 p.enc = (*Buffer).enc_slice_struct_message 453 p.dec = (*Buffer).dec_slice_struct_message 454 p.size = size_slice_struct_message 455 } else { 456 p.enc = (*Buffer).enc_slice_struct_group 457 p.dec = (*Buffer).dec_slice_struct_group 458 p.size = size_slice_struct_group 459 } 460 } 461 case reflect.Slice: 462 switch t2.Elem().Kind() { 463 default: 464 fmt.Fprintf(os.Stderr, "proto: no slice elem oenc for %T -> %T -> %T\n", t1, t2, t2.Elem()) 465 break 466 case reflect.Uint8: 467 p.enc = (*Buffer).enc_slice_slice_byte 468 p.dec = (*Buffer).dec_slice_slice_byte 469 p.size = size_slice_slice_byte 470 } 471 case reflect.Struct: 472 p.setSliceOfNonPointerStructs(t1) 473 } 474 } 475 p.setTag(lockGetProp) 476 } 477 478 func (p *Properties) setTag(lockGetProp bool) { 479 // precalculate tag code 480 wire := p.WireType 481 if p.Packed { 482 wire = WireBytes 483 } 484 x := uint32(p.Tag)<<3 | uint32(wire) 485 i := 0 486 for i = 0; x > 127; i++ { 487 p.tagbuf[i] = 0x80 | uint8(x&0x7F) 488 x >>= 7 489 } 490 p.tagbuf[i] = uint8(x) 491 p.tagcode = p.tagbuf[0 : i+1] 492 493 if p.stype != nil { 494 if lockGetProp { 495 p.sprop = GetProperties(p.stype) 496 } else { 497 p.sprop = getPropertiesLocked(p.stype) 498 } 499 } 500 } 501 502 var ( 503 marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() 504 unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() 505 ) 506 507 // isMarshaler reports whether type t implements Marshaler. 508 func isMarshaler(t reflect.Type) bool { 509 return t.Implements(marshalerType) 510 } 511 512 // isUnmarshaler reports whether type t implements Unmarshaler. 513 func isUnmarshaler(t reflect.Type) bool { 514 return t.Implements(unmarshalerType) 515 } 516 517 // Init populates the properties from a protocol buffer struct tag. 518 func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { 519 p.init(typ, name, tag, f, true) 520 } 521 522 func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructField, lockGetProp bool) { 523 // "bytes,49,opt,def=hello!" 524 p.Name = name 525 p.OrigName = name 526 if f != nil { 527 p.field = toField(f) 528 } 529 if tag == "" { 530 return 531 } 532 p.Parse(tag) 533 p.setEncAndDec(typ, lockGetProp) 534 } 535 536 var ( 537 mutex sync.Mutex 538 propertiesMap = make(map[reflect.Type]*StructProperties) 539 ) 540 541 // GetProperties returns the list of properties for the type represented by t. 542 func GetProperties(t reflect.Type) *StructProperties { 543 mutex.Lock() 544 sprop := getPropertiesLocked(t) 545 mutex.Unlock() 546 return sprop 547 } 548 549 // getPropertiesLocked requires that mutex is held. 550 func getPropertiesLocked(t reflect.Type) *StructProperties { 551 if prop, ok := propertiesMap[t]; ok { 552 if collectStats { 553 stats.Chit++ 554 } 555 return prop 556 } 557 if collectStats { 558 stats.Cmiss++ 559 } 560 561 prop := new(StructProperties) 562 // in case of recursive protos, fill this in now. 563 propertiesMap[t] = prop 564 565 // build properties 566 prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) 567 prop.unrecField = invalidField 568 prop.Prop = make([]*Properties, t.NumField()) 569 prop.order = make([]int, t.NumField()) 570 571 for i := 0; i < t.NumField(); i++ { 572 f := t.Field(i) 573 p := new(Properties) 574 name := f.Name 575 p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) 576 577 if f.Name == "XXX_extensions" { // special case 578 p.enc = (*Buffer).enc_map 579 p.dec = nil // not needed 580 p.size = size_map 581 } 582 if f.Name == "XXX_unrecognized" { // special case 583 prop.unrecField = toField(&f) 584 } 585 prop.Prop[i] = p 586 prop.order[i] = i 587 if debug { 588 print(i, " ", f.Name, " ", t.String(), " ") 589 if p.Tag > 0 { 590 print(p.String()) 591 } 592 print("\n") 593 } 594 if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") { 595 fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]") 596 } 597 } 598 599 // Re-order prop.order. 600 sort.Sort(prop) 601 602 // build required counts 603 // build tags 604 reqCount := 0 605 prop.decoderOrigNames = make(map[string]int) 606 for i, p := range prop.Prop { 607 if strings.HasPrefix(p.Name, "XXX_") { 608 // Internal fields should not appear in tags/origNames maps. 609 // They are handled specially when encoding and decoding. 610 continue 611 } 612 if p.Required { 613 reqCount++ 614 } 615 prop.decoderTags.put(p.Tag, i) 616 prop.decoderOrigNames[p.OrigName] = i 617 } 618 prop.reqCount = reqCount 619 620 return prop 621 } 622 623 // Return the Properties object for the x[0]'th field of the structure. 624 func propByIndex(t reflect.Type, x []int) *Properties { 625 if len(x) != 1 { 626 fmt.Fprintf(os.Stderr, "proto: field index dimension %d (not 1) for type %s\n", len(x), t) 627 return nil 628 } 629 prop := GetProperties(t) 630 return prop.Prop[x[0]] 631 } 632 633 // Get the address and type of a pointer to a struct from an interface. 634 func getbase(pb Message) (t reflect.Type, b structPointer, err error) { 635 if pb == nil { 636 err = ErrNil 637 return 638 } 639 // get the reflect type of the pointer to the struct. 640 t = reflect.TypeOf(pb) 641 // get the address of the struct. 642 value := reflect.ValueOf(pb) 643 b = toStructPointer(value) 644 return 645 } 646 647 // A global registry of enum types. 648 // The generated code will register the generated maps by calling RegisterEnum. 649 650 var enumValueMaps = make(map[string]map[string]int32) 651 var enumStringMaps = make(map[string]map[int32]string) 652 653 // RegisterEnum is called from the generated code to install the enum descriptor 654 // maps into the global table to aid parsing text format protocol buffers. 655 func RegisterEnum(typeName string, unusedNameMap map[int32]string, valueMap map[string]int32) { 656 if _, ok := enumValueMaps[typeName]; ok { 657 panic("proto: duplicate enum registered: " + typeName) 658 } 659 enumValueMaps[typeName] = valueMap 660 if _, ok := enumStringMaps[typeName]; ok { 661 panic("proto: duplicate enum registered: " + typeName) 662 } 663 enumStringMaps[typeName] = unusedNameMap 664 }