github.com/turingchain2020/turingchain@v1.1.21/types/jsonpb/jsonpb.go (about) 1 // Copyright Turing Corp. 2018 All Rights Reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Go support for Protocol Buffers - Google's data interchange format 6 // 7 // Copyright 2015 The Go Authors. All rights reserved. 8 // https://github.com/golang/protobuf 9 // 10 // Redistribution and use in source and binary forms, with or without 11 // modification, are permitted provided that the following conditions are 12 // met: 13 // 14 // * Redistributions of source code must retain the above copyright 15 // notice, this list of conditions and the following disclaimer. 16 // * Redistributions in binary form must reproduce the above 17 // copyright notice, this list of conditions and the following disclaimer 18 // in the documentation and/or other materials provided with the 19 // distribution. 20 // * Neither the name of Google Inc. nor the names of its 21 // contributors may be used to endorse or promote products derived from 22 // this software without specific prior written permission. 23 // 24 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 36 /* 37 Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON. 38 It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json. 39 40 This package produces a different output than the standard "encoding/json" package, 41 which does not operate correctly on protocol buffers. 42 */ 43 package jsonpb 44 45 import ( 46 "bytes" 47 "encoding/json" 48 "errors" 49 "fmt" 50 "io" 51 "math" 52 "reflect" 53 "sort" 54 "strconv" 55 "strings" 56 "time" 57 "unicode/utf8" 58 59 "github.com/turingchain2020/turingchain/common" 60 "github.com/golang/protobuf/proto" 61 62 stpb "github.com/golang/protobuf/ptypes/struct" 63 ) 64 65 const secondInNanos = int64(time.Second / time.Nanosecond) 66 67 // Marshaler is a configurable object for converting between 68 // protocol buffer objects and a JSON representation for them. 69 type Marshaler struct { 70 // Whether to render enum values as integers, as opposed to string values. 71 EnumsAsInts bool 72 73 // Whether to render fields with zero values. 74 EmitDefaults bool 75 76 //Enable utf8 bytes to string 77 EnableUTF8BytesToString bool 78 79 // A string to indent each level by. The presence of this field will 80 // also cause a space to appear between the field separator and 81 // value, and for newlines to be appear between fields and array 82 // elements. 83 Indent string 84 85 // Whether to use the original (.proto) name for fields. 86 OrigName bool 87 88 // A custom URL resolver to use when marshaling Any messages to JSON. 89 // If unset, the default resolution strategy is to extract the 90 // fully-qualified type name from the type URL and pass that to 91 // proto.MessageType(string). 92 AnyResolver AnyResolver 93 } 94 95 // AnyResolver takes a type URL, present in an Any message, and resolves it into 96 // an instance of the associated message. 97 type AnyResolver interface { 98 Resolve(typeURL string) (proto.Message, error) 99 } 100 101 func defaultResolveAny(typeURL string) (proto.Message, error) { 102 // Only the part of typeUrl after the last slash is relevant. 103 mname := typeURL 104 if slash := strings.LastIndex(mname, "/"); slash >= 0 { 105 mname = mname[slash+1:] 106 } 107 mt := proto.MessageType(mname) 108 if mt == nil { 109 return nil, fmt.Errorf("unknown message type %q", mname) 110 } 111 return reflect.New(mt.Elem()).Interface().(proto.Message), nil 112 } 113 114 // JSONPBmarshaler is implemented by protobuf messages that customize the 115 // way they are marshaled to JSON. Messages that implement this should 116 // also implement JSONPBUnmarshaler so that the custom format can be 117 // parsed. 118 // 119 // The JSON marshaling must follow the proto to JSON specification: 120 // https://developers.google.com/protocol-buffers/docs/proto3#json 121 type JSONPBmarshaler interface { 122 MarshalJSONPB(*Marshaler) ([]byte, error) 123 } 124 125 // JSONPBunmarshaler is implemented by protobuf messages that customize 126 // the way they are unmarshaled from JSON. Messages that implement this 127 // should also implement JSONPBMarshaler so that the custom format can be 128 // produced. 129 // 130 // The JSON unmarshaling must follow the JSON to proto specification: 131 // https://developers.google.com/protocol-buffers/docs/proto3#json 132 type JSONPBunmarshaler interface { 133 UnmarshalJSONPB(*Unmarshaler, []byte) error 134 } 135 136 // Marshal marshals a protocol buffer into JSON. 137 func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error { 138 v := reflect.ValueOf(pb) 139 if pb == nil || (v.Kind() == reflect.Ptr && v.IsNil()) { 140 return errors.New("Marshal called with nil") 141 } 142 // Check for unset required fields first. 143 if err := checkRequiredFields(pb); err != nil { 144 return err 145 } 146 writer := &errWriter{writer: out} 147 return m.marshalObject(writer, pb, "", "") 148 } 149 150 // MarshalToString converts a protocol buffer object to JSON string. 151 func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) { 152 var buf bytes.Buffer 153 if err := m.Marshal(&buf, pb); err != nil { 154 return "", err 155 } 156 return buf.String(), nil 157 } 158 159 type int32Slice []int32 160 161 var nonFinite = map[string]float64{ 162 `"NaN"`: math.NaN(), 163 `"Infinity"`: math.Inf(1), 164 `"-Infinity"`: math.Inf(-1), 165 } 166 167 // For sorting extensions ids to ensure stable output. 168 func (s int32Slice) Len() int { return len(s) } 169 func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } 170 func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 171 172 type wkt interface { 173 XXX_WellKnownType() string 174 } 175 176 // marshalObject writes a struct to the Writer. 177 func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error { 178 if jsm, ok := v.(JSONPBmarshaler); ok { 179 b, err := jsm.MarshalJSONPB(m) 180 if err != nil { 181 return err 182 } 183 if typeURL != "" { 184 // we are marshaling this object to an Any type 185 var js map[string]*json.RawMessage 186 if err = json.Unmarshal(b, &js); err != nil { 187 return fmt.Errorf("type %T produced invalid JSON: %v", v, err) 188 } 189 turl, err := json.Marshal(typeURL) 190 if err != nil { 191 return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err) 192 } 193 js["@type"] = (*json.RawMessage)(&turl) 194 if b, err = json.Marshal(js); err != nil { 195 return err 196 } 197 } 198 199 out.write(string(b)) 200 return out.err 201 } 202 203 s := reflect.ValueOf(v).Elem() 204 205 // Handle well-known types. 206 if wkt, ok := v.(wkt); ok { 207 switch wkt.XXX_WellKnownType() { 208 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 209 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 210 // "Wrappers use the same representation in JSON 211 // as the wrapped primitive type, ..." 212 sprop := proto.GetProperties(s.Type()) 213 return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent) 214 case "Any": 215 // Any is a bit more involved. 216 return m.marshalAny(out, v, indent) 217 case "Duration": 218 // "Generated output always contains 0, 3, 6, or 9 fractional digits, 219 // depending on required precision." 220 s, ns := s.Field(0).Int(), s.Field(1).Int() 221 if ns <= -secondInNanos || ns >= secondInNanos { 222 return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos) 223 } 224 if (s > 0 && ns < 0) || (s < 0 && ns > 0) { 225 return errors.New("signs of seconds and nanos do not match") 226 } 227 if s < 0 { 228 ns = -ns 229 } 230 x := fmt.Sprintf("%d.%09d", s, ns) 231 x = strings.TrimSuffix(x, "000") 232 x = strings.TrimSuffix(x, "000") 233 x = strings.TrimSuffix(x, ".000") 234 out.write(`"`) 235 out.write(x) 236 out.write(`s"`) 237 return out.err 238 case "Struct", "ListValue": 239 // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice. 240 // TODO: pass the correct Properties if needed. 241 return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent) 242 case "Timestamp": 243 // "RFC 3339, where generated output will always be Z-normalized 244 // and uses 0, 3, 6 or 9 fractional digits." 245 s, ns := s.Field(0).Int(), s.Field(1).Int() 246 if ns < 0 || ns >= secondInNanos { 247 return fmt.Errorf("ns out of range [0, %v)", secondInNanos) 248 } 249 t := time.Unix(s, ns).UTC() 250 // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits). 251 x := t.Format("2006-01-02T15:04:05.000000000") 252 x = strings.TrimSuffix(x, "000") 253 x = strings.TrimSuffix(x, "000") 254 x = strings.TrimSuffix(x, ".000") 255 out.write(`"`) 256 out.write(x) 257 out.write(`Z"`) 258 return out.err 259 case "Value": 260 // Value has a single oneof. 261 kind := s.Field(0) 262 if kind.IsNil() { 263 // "absence of any variant indicates an error" 264 return errors.New("nil Value") 265 } 266 // oneof -> *T -> T -> T.F 267 x := kind.Elem().Elem().Field(0) 268 // TODO: pass the correct Properties if needed. 269 return m.marshalValue(out, &proto.Properties{}, x, indent) 270 } 271 } 272 273 out.write("{") 274 if m.Indent != "" { 275 out.write("\n") 276 } 277 278 firstField := true 279 280 if typeURL != "" { 281 if err := m.marshalTypeURL(out, indent, typeURL); err != nil { 282 return err 283 } 284 firstField = false 285 } 286 287 for i := 0; i < s.NumField(); i++ { 288 value := s.Field(i) 289 valueField := s.Type().Field(i) 290 if strings.HasPrefix(valueField.Name, "XXX_") { 291 continue 292 } 293 294 // IsNil will panic on most value kinds. 295 switch value.Kind() { 296 case reflect.Chan, reflect.Func, reflect.Interface: 297 if value.IsNil() { 298 continue 299 } 300 } 301 302 if !m.EmitDefaults { 303 switch value.Kind() { 304 case reflect.Bool: 305 if !value.Bool() { 306 continue 307 } 308 case reflect.Int32, reflect.Int64: 309 if value.Int() == 0 { 310 continue 311 } 312 case reflect.Uint32, reflect.Uint64: 313 if value.Uint() == 0 { 314 continue 315 } 316 case reflect.Float32, reflect.Float64: 317 if value.Float() == 0 { 318 continue 319 } 320 case reflect.String: 321 if value.Len() == 0 { 322 continue 323 } 324 case reflect.Map, reflect.Ptr, reflect.Slice: 325 if value.IsNil() { 326 continue 327 } 328 } 329 } 330 331 // Oneof fields need special handling. 332 if valueField.Tag.Get("protobuf_oneof") != "" { 333 // value is an interface containing &T{real_value}. 334 sv := value.Elem().Elem() // interface -> *T -> T 335 value = sv.Field(0) 336 valueField = sv.Type().Field(0) 337 } 338 prop := jsonProperties(valueField, m.OrigName) 339 if !firstField { 340 m.writeSep(out) 341 } 342 if err := m.marshalField(out, prop, value, indent); err != nil { 343 return err 344 } 345 firstField = false 346 } 347 348 // Handle proto2 extensions. 349 if ep, ok := v.(proto.Message); ok { 350 extensions := proto.RegisteredExtensions(v) 351 // Sort extensions for stable output. 352 ids := make([]int32, 0, len(extensions)) 353 for id, desc := range extensions { 354 if !proto.HasExtension(ep, desc) { 355 continue 356 } 357 ids = append(ids, id) 358 } 359 sort.Sort(int32Slice(ids)) 360 for _, id := range ids { 361 desc := extensions[id] 362 if desc == nil { 363 // unknown extension 364 continue 365 } 366 ext, extErr := proto.GetExtension(ep, desc) 367 if extErr != nil { 368 return extErr 369 } 370 value := reflect.ValueOf(ext) 371 var prop proto.Properties 372 prop.Parse(desc.Tag) 373 prop.JSONName = fmt.Sprintf("[%s]", desc.Name) 374 if !firstField { 375 m.writeSep(out) 376 } 377 if err := m.marshalField(out, &prop, value, indent); err != nil { 378 return err 379 } 380 firstField = false 381 } 382 383 } 384 385 if m.Indent != "" { 386 out.write("\n") 387 out.write(indent) 388 } 389 out.write("}") 390 return out.err 391 } 392 393 func (m *Marshaler) writeSep(out *errWriter) { 394 if m.Indent != "" { 395 out.write(",\n") 396 } else { 397 out.write(",") 398 } 399 } 400 401 func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error { 402 // "If the Any contains a value that has a special JSON mapping, 403 // it will be converted as follows: {"@type": xxx, "value": yyy}. 404 // Otherwise, the value will be converted into a JSON object, 405 // and the "@type" field will be inserted to indicate the actual data type." 406 v := reflect.ValueOf(any).Elem() 407 turl := v.Field(0).String() 408 val := v.Field(1).Bytes() 409 410 var msg proto.Message 411 var err error 412 if m.AnyResolver != nil { 413 msg, err = m.AnyResolver.Resolve(turl) 414 } else { 415 msg, err = defaultResolveAny(turl) 416 } 417 if err != nil { 418 return err 419 } 420 421 if err := proto.Unmarshal(val, msg); err != nil { 422 return err 423 } 424 425 if _, ok := msg.(wkt); ok { 426 out.write("{") 427 if m.Indent != "" { 428 out.write("\n") 429 } 430 if err := m.marshalTypeURL(out, indent, turl); err != nil { 431 return err 432 } 433 m.writeSep(out) 434 if m.Indent != "" { 435 out.write(indent) 436 out.write(m.Indent) 437 out.write(`"value": `) 438 } else { 439 out.write(`"value":`) 440 } 441 if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil { 442 return err 443 } 444 if m.Indent != "" { 445 out.write("\n") 446 out.write(indent) 447 } 448 out.write("}") 449 return out.err 450 } 451 452 return m.marshalObject(out, msg, indent, turl) 453 } 454 455 func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error { 456 if m.Indent != "" { 457 out.write(indent) 458 out.write(m.Indent) 459 } 460 out.write(`"@type":`) 461 if m.Indent != "" { 462 out.write(" ") 463 } 464 b, err := json.Marshal(typeURL) 465 if err != nil { 466 return err 467 } 468 out.write(string(b)) 469 return out.err 470 } 471 472 // marshalField writes field description and value to the Writer. 473 func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 474 if m.Indent != "" { 475 out.write(indent) 476 out.write(m.Indent) 477 } 478 out.write(`"`) 479 out.write(prop.JSONName) 480 out.write(`":`) 481 if m.Indent != "" { 482 out.write(" ") 483 } 484 if err := m.marshalValue(out, prop, v, indent); err != nil { 485 return err 486 } 487 return nil 488 } 489 490 // marshalValue writes the value to the Writer. 491 func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 492 var err error 493 v = reflect.Indirect(v) 494 495 // Handle nil pointer 496 if v.Kind() == reflect.Invalid { 497 out.write("null") 498 return out.err 499 } 500 501 // Handle repeated elements. 502 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { 503 out.write("[") 504 comma := "" 505 for i := 0; i < v.Len(); i++ { 506 sliceVal := v.Index(i) 507 out.write(comma) 508 if m.Indent != "" { 509 out.write("\n") 510 out.write(indent) 511 out.write(m.Indent) 512 out.write(m.Indent) 513 } 514 if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil { 515 return err 516 } 517 comma = "," 518 } 519 if m.Indent != "" { 520 out.write("\n") 521 out.write(indent) 522 out.write(m.Indent) 523 } 524 out.write("]") 525 return out.err 526 } 527 528 //[]byte 写bytes 的情况,默认情况下,转化成 hex 529 //为什么不用base64: 530 //1. 我们的数据都经过压缩(base64带来的字节数的减少有限) 531 //2. hex 是一种最容易解析的格式 532 //3. 我们的hash 默认是 bytes,而且转化成hex 533 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() == reflect.Uint8 { 534 if v.IsNil() { 535 out.write("null") 536 return out.err 537 } 538 data := v.Interface().([]byte) 539 //开启这个选项后,会把utf8的字符串转化成string,而不会弄成hex 540 if m.EnableUTF8BytesToString && utf8.Valid(data) { 541 s := string(data) 542 b, err := json.Marshal(s) 543 if err != nil { 544 return err 545 } 546 out.write(string(b)) 547 } else { 548 out.write(`"`) 549 out.write(common.ToHex(data)) 550 out.write(`"`) 551 } 552 return out.err 553 } 554 555 // Handle well-known types. 556 // Most are handled up in marshalObject (because 99% are messages). 557 if wkt, ok := v.Interface().(wkt); ok { 558 switch wkt.XXX_WellKnownType() { 559 case "NullValue": 560 out.write("null") 561 return out.err 562 } 563 } 564 565 // Handle enumerations. 566 if !m.EnumsAsInts && prop.Enum != "" { 567 // Unknown enum values will are stringified by the proto library as their 568 // value. Such values should _not_ be quoted or they will be interpreted 569 // as an enum string instead of their value. 570 enumStr := v.Interface().(fmt.Stringer).String() 571 var valStr string 572 if v.Kind() == reflect.Ptr { 573 valStr = strconv.Itoa(int(v.Elem().Int())) 574 } else { 575 valStr = strconv.Itoa(int(v.Int())) 576 } 577 isKnownEnum := enumStr != valStr 578 if isKnownEnum { 579 out.write(`"`) 580 } 581 out.write(enumStr) 582 if isKnownEnum { 583 out.write(`"`) 584 } 585 return out.err 586 } 587 588 // Handle nested messages. 589 if v.Kind() == reflect.Struct { 590 return m.marshalObject(out, v.Addr().Interface().(proto.Message), indent+m.Indent, "") 591 } 592 593 // Handle maps. 594 // Since Go randomizes map iteration, we sort keys for stable output. 595 if v.Kind() == reflect.Map { 596 out.write(`{`) 597 keys := v.MapKeys() 598 sort.Sort(mapKeys(keys)) 599 for i, k := range keys { 600 if i > 0 { 601 out.write(`,`) 602 } 603 if m.Indent != "" { 604 out.write("\n") 605 out.write(indent) 606 out.write(m.Indent) 607 out.write(m.Indent) 608 } 609 610 // TODO handle map key prop properly 611 b, err := json.Marshal(k.Interface()) 612 if err != nil { 613 return err 614 } 615 s := string(b) 616 617 // If the JSON is not a string value, encode it again to make it one. 618 if !strings.HasPrefix(s, `"`) { 619 b, err := json.Marshal(s) 620 if err != nil { 621 return err 622 } 623 s = string(b) 624 } 625 626 out.write(s) 627 out.write(`:`) 628 if m.Indent != "" { 629 out.write(` `) 630 } 631 632 vprop := prop 633 if prop != nil && prop.MapValProp != nil { 634 vprop = prop.MapValProp 635 } 636 if err := m.marshalValue(out, vprop, v.MapIndex(k), indent+m.Indent); err != nil { 637 return err 638 } 639 } 640 if m.Indent != "" { 641 out.write("\n") 642 out.write(indent) 643 out.write(m.Indent) 644 } 645 out.write(`}`) 646 return out.err 647 } 648 649 // Handle non-finite floats, e.g. NaN, Infinity and -Infinity. 650 if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 { 651 f := v.Float() 652 var sval string 653 switch { 654 case math.IsInf(f, 1): 655 sval = `"Infinity"` 656 case math.IsInf(f, -1): 657 sval = `"-Infinity"` 658 case math.IsNaN(f): 659 sval = `"NaN"` 660 } 661 if sval != "" { 662 out.write(sval) 663 return out.err 664 } 665 } 666 667 // Default handling defers to the encoding/json library. 668 b, err := json.Marshal(v.Interface()) 669 if err != nil { 670 return err 671 } 672 needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64) 673 if needToQuote { 674 out.write(`"`) 675 } 676 out.write(string(b)) 677 if needToQuote { 678 out.write(`"`) 679 } 680 return out.err 681 } 682 683 // Unmarshaler is a configurable object for converting from a JSON 684 // representation to a protocol buffer object. 685 type Unmarshaler struct { 686 // Whether to allow messages to contain unknown fields, as opposed to 687 // failing to unmarshal. 688 AllowUnknownFields bool 689 690 //Enable utf8 bytes to string 691 EnableUTF8BytesToString bool 692 693 // A custom URL resolver to use when unmarshaling Any messages from JSON. 694 // If unset, the default resolution strategy is to extract the 695 // fully-qualified type name from the type URL and pass that to 696 // proto.MessageType(string). 697 AnyResolver AnyResolver 698 } 699 700 // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 701 // This function is lenient and will decode any options permutations of the 702 // related Marshaler. 703 func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 704 inputValue := json.RawMessage{} 705 if err := dec.Decode(&inputValue); err != nil { 706 return err 707 } 708 if err := u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil); err != nil { 709 return err 710 } 711 return checkRequiredFields(pb) 712 } 713 714 // Unmarshal unmarshals a JSON object stream into a protocol 715 // buffer. This function is lenient and will decode any options 716 // permutations of the related Marshaler. 717 func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error { 718 dec := json.NewDecoder(r) 719 return u.UnmarshalNext(dec, pb) 720 } 721 722 // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 723 // This function is lenient and will decode any options permutations of the 724 // related Marshaler. 725 func UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 726 return new(Unmarshaler).UnmarshalNext(dec, pb) 727 } 728 729 // Unmarshal unmarshals a JSON object stream into a protocol 730 // buffer. This function is lenient and will decode any options 731 // permutations of the related Marshaler. 732 func Unmarshal(r io.Reader, pb proto.Message) error { 733 return new(Unmarshaler).Unmarshal(r, pb) 734 } 735 736 // UnmarshalString will populate the fields of a protocol buffer based 737 // on a JSON string. This function is lenient and will decode any options 738 // permutations of the related Marshaler. 739 func UnmarshalString(str string, pb proto.Message) error { 740 return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb) 741 } 742 743 // unmarshalValue converts/copies a value into the target. 744 // prop may be nil. 745 func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error { 746 targetType := target.Type() 747 748 // Allocate memory for pointer fields. 749 if targetType.Kind() == reflect.Ptr { 750 // If input value is "null" and target is a pointer type, then the field should be treated as not set 751 // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue. 752 _, isJSONPBUnmarshaler := target.Interface().(JSONPBunmarshaler) 753 if string(inputValue) == "null" && targetType != reflect.TypeOf(&stpb.Value{}) && !isJSONPBUnmarshaler { 754 return nil 755 } 756 target.Set(reflect.New(targetType.Elem())) 757 758 return u.unmarshalValue(target.Elem(), inputValue, prop) 759 } 760 761 if jsu, ok := target.Addr().Interface().(JSONPBunmarshaler); ok { 762 return jsu.UnmarshalJSONPB(u, []byte(inputValue)) 763 } 764 765 // Handle well-known types that are not pointers. 766 if w, ok := target.Addr().Interface().(wkt); ok { 767 switch w.XXX_WellKnownType() { 768 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 769 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 770 return u.unmarshalValue(target.Field(0), inputValue, prop) 771 case "Any": 772 // Use json.RawMessage pointer type instead of value to support pre-1.8 version. 773 // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see 774 // https://github.com/golang/go/issues/14493 775 var jsonFields map[string]*json.RawMessage 776 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 777 return err 778 } 779 780 val, ok := jsonFields["@type"] 781 if !ok || val == nil { 782 return errors.New("Any JSON doesn't have '@type'") 783 } 784 785 var turl string 786 if err := json.Unmarshal([]byte(*val), &turl); err != nil { 787 return fmt.Errorf("can't unmarshal Any's '@type': %q", *val) 788 } 789 target.Field(0).SetString(turl) 790 791 var m proto.Message 792 var err error 793 if u.AnyResolver != nil { 794 m, err = u.AnyResolver.Resolve(turl) 795 } else { 796 m, err = defaultResolveAny(turl) 797 } 798 if err != nil { 799 return err 800 } 801 802 if _, ok := m.(wkt); ok { 803 val, ok := jsonFields["value"] 804 if !ok { 805 return errors.New("Any JSON doesn't have 'value'") 806 } 807 808 if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil { 809 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 810 } 811 } else { 812 delete(jsonFields, "@type") 813 nestedProto, err := json.Marshal(jsonFields) 814 if err != nil { 815 return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err) 816 } 817 818 if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil { 819 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 820 } 821 } 822 823 b, err := proto.Marshal(m) 824 if err != nil { 825 return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err) 826 } 827 target.Field(1).SetBytes(b) 828 829 return nil 830 case "Duration": 831 unq, err := unquote(string(inputValue)) 832 if err != nil { 833 return err 834 } 835 836 d, err := time.ParseDuration(unq) 837 if err != nil { 838 return fmt.Errorf("bad Duration: %v", err) 839 } 840 841 ns := d.Nanoseconds() 842 s := ns / 1e9 843 ns %= 1e9 844 target.Field(0).SetInt(s) 845 target.Field(1).SetInt(ns) 846 return nil 847 case "Timestamp": 848 unq, err := unquote(string(inputValue)) 849 if err != nil { 850 return err 851 } 852 853 t, err := time.Parse(time.RFC3339Nano, unq) 854 if err != nil { 855 return fmt.Errorf("bad Timestamp: %v", err) 856 } 857 858 target.Field(0).SetInt(t.Unix()) 859 target.Field(1).SetInt(int64(t.Nanosecond())) 860 return nil 861 case "Struct": 862 var m map[string]json.RawMessage 863 if err := json.Unmarshal(inputValue, &m); err != nil { 864 return fmt.Errorf("bad StructValue: %v", err) 865 } 866 867 target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{})) 868 for k, jv := range m { 869 pv := &stpb.Value{} 870 if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil { 871 return fmt.Errorf("bad value in StructValue for key %q: %v", k, err) 872 } 873 target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv)) 874 } 875 return nil 876 case "ListValue": 877 var s []json.RawMessage 878 if err := json.Unmarshal(inputValue, &s); err != nil { 879 return fmt.Errorf("bad ListValue: %v", err) 880 } 881 882 target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s)))) 883 for i, sv := range s { 884 if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil { 885 return err 886 } 887 } 888 return nil 889 case "Value": 890 ivStr := string(inputValue) 891 if ivStr == "null" { 892 target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{})) 893 } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil { 894 target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{NumberValue: v})) 895 } else if v, err := unquote(ivStr); err == nil { 896 target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{StringValue: v})) 897 } else if v, err := strconv.ParseBool(ivStr); err == nil { 898 target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{BoolValue: v})) 899 } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil { 900 lv := &stpb.ListValue{} 901 target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{ListValue: lv})) 902 return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop) 903 } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil { 904 sv := &stpb.Struct{} 905 target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{StructValue: sv})) 906 return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop) 907 } else { 908 return fmt.Errorf("unrecognized type for Value %q", ivStr) 909 } 910 return nil 911 } 912 } 913 914 // Handle enums, which have an underlying type of int32, 915 // and may appear as strings. 916 // The case of an enum appearing as a number is handled 917 // at the bottom of this function. 918 if inputValue[0] == '"' && prop != nil && prop.Enum != "" { 919 vmap := proto.EnumValueMap(prop.Enum) 920 // Don't need to do unquoting; valid enum names 921 // are from a limited character set. 922 s := inputValue[1 : len(inputValue)-1] 923 n, ok := vmap[string(s)] 924 if !ok { 925 return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum) 926 } 927 if target.Kind() == reflect.Ptr { // proto2 928 target.Set(reflect.New(targetType.Elem())) 929 target = target.Elem() 930 } 931 if targetType.Kind() != reflect.Int32 { 932 return fmt.Errorf("invalid target %q for enum %s", targetType.Kind(), prop.Enum) 933 } 934 target.SetInt(int64(n)) 935 return nil 936 } 937 938 // Handle nested messages. 939 if targetType.Kind() == reflect.Struct { 940 var jsonFields map[string]json.RawMessage 941 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 942 return err 943 } 944 945 consumeField := func(prop *proto.Properties) (json.RawMessage, bool) { 946 // Be liberal in what names we accept; both orig_name and camelName are okay. 947 fieldNames := acceptedJSONFieldNames(prop) 948 949 vOrig, okOrig := jsonFields[fieldNames.orig] 950 vCamel, okCamel := jsonFields[fieldNames.camel] 951 if !okOrig && !okCamel { 952 return nil, false 953 } 954 // If, for some reason, both are present in the data, favour the camelName. 955 var raw json.RawMessage 956 if okOrig { 957 raw = vOrig 958 delete(jsonFields, fieldNames.orig) 959 } 960 if okCamel { 961 raw = vCamel 962 delete(jsonFields, fieldNames.camel) 963 } 964 return raw, true 965 } 966 967 sprops := proto.GetProperties(targetType) 968 for i := 0; i < target.NumField(); i++ { 969 ft := target.Type().Field(i) 970 if strings.HasPrefix(ft.Name, "XXX_") { 971 continue 972 } 973 974 valueForField, ok := consumeField(sprops.Prop[i]) 975 if !ok { 976 continue 977 } 978 979 if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil { 980 return err 981 } 982 } 983 // Check for any oneof fields. 984 if len(jsonFields) > 0 { 985 for _, oop := range sprops.OneofTypes { 986 raw, ok := consumeField(oop.Prop) 987 if !ok { 988 continue 989 } 990 nv := reflect.New(oop.Type.Elem()) 991 target.Field(oop.Field).Set(nv) 992 if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil { 993 return err 994 } 995 } 996 } 997 // Handle proto2 extensions. 998 if len(jsonFields) > 0 { 999 if ep, ok := target.Addr().Interface().(proto.Message); ok { 1000 for _, ext := range proto.RegisteredExtensions(ep) { 1001 name := fmt.Sprintf("[%s]", ext.Name) 1002 raw, ok := jsonFields[name] 1003 if !ok { 1004 continue 1005 } 1006 delete(jsonFields, name) 1007 nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem()) 1008 if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil { 1009 return err 1010 } 1011 if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil { 1012 return err 1013 } 1014 } 1015 } 1016 } 1017 if !u.AllowUnknownFields && len(jsonFields) > 0 { 1018 // Pick any field to be the scapegoat. 1019 var f string 1020 for fname := range jsonFields { 1021 f = fname 1022 break 1023 } 1024 return fmt.Errorf("unknown field %q in %v", f, targetType) 1025 } 1026 return nil 1027 } 1028 1029 // Handle arrays (which aren't encoded bytes) 1030 if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() != reflect.Uint8 { 1031 var slc []json.RawMessage 1032 if err := json.Unmarshal(inputValue, &slc); err != nil { 1033 return err 1034 } 1035 if slc != nil { 1036 l := len(slc) 1037 target.Set(reflect.MakeSlice(targetType, l, l)) 1038 for i := 0; i < l; i++ { 1039 if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil { 1040 return err 1041 } 1042 } 1043 } 1044 return nil 1045 } 1046 1047 //decode bytes 1048 if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() == reflect.Uint8 { 1049 if string(inputValue) == "null" { 1050 target.SetBytes(nil) 1051 return nil 1052 } 1053 if string(inputValue) == "[]" { 1054 target.SetBytes([]byte{}) 1055 return nil 1056 } 1057 var hexstr string 1058 err := json.Unmarshal(inputValue, &hexstr) 1059 if err != nil { 1060 return err 1061 } 1062 b, err := parseBytes(hexstr, u.EnableUTF8BytesToString) 1063 if err != nil { 1064 return err 1065 } 1066 target.SetBytes(b) 1067 return nil 1068 } 1069 1070 // Handle maps (whose keys are always strings) 1071 if targetType.Kind() == reflect.Map { 1072 var mp map[string]json.RawMessage 1073 if err := json.Unmarshal(inputValue, &mp); err != nil { 1074 return err 1075 } 1076 if mp != nil { 1077 target.Set(reflect.MakeMap(targetType)) 1078 for ks, raw := range mp { 1079 // Unmarshal map key. The core json library already decoded the key into a 1080 // string, so we handle that specially. Other types were quoted post-serialization. 1081 var k reflect.Value 1082 if targetType.Key().Kind() == reflect.String { 1083 k = reflect.ValueOf(ks) 1084 } else { 1085 k = reflect.New(targetType.Key()).Elem() 1086 var kprop *proto.Properties 1087 if prop != nil && prop.MapKeyProp != nil { 1088 kprop = prop.MapKeyProp 1089 } 1090 if err := u.unmarshalValue(k, json.RawMessage(ks), kprop); err != nil { 1091 return err 1092 } 1093 } 1094 1095 // Unmarshal map value. 1096 v := reflect.New(targetType.Elem()).Elem() 1097 var vprop *proto.Properties 1098 if prop != nil && prop.MapValProp != nil { 1099 vprop = prop.MapValProp 1100 } 1101 if err := u.unmarshalValue(v, raw, vprop); err != nil { 1102 return err 1103 } 1104 target.SetMapIndex(k, v) 1105 } 1106 } 1107 return nil 1108 } 1109 1110 // Non-finite numbers can be encoded as strings. 1111 isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1112 if isFloat { 1113 if num, ok := nonFinite[string(inputValue)]; ok { 1114 target.SetFloat(num) 1115 return nil 1116 } 1117 } 1118 1119 // integers & floats can be encoded as strings. In this case we drop 1120 // the quotes and proceed as normal. 1121 isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64 || 1122 targetType.Kind() == reflect.Int32 || targetType.Kind() == reflect.Uint32 || 1123 targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1124 if isNum && strings.HasPrefix(string(inputValue), `"`) { 1125 inputValue = inputValue[1 : len(inputValue)-1] 1126 } 1127 1128 // Use the encoding/json for parsing other value types. 1129 return json.Unmarshal(inputValue, target.Addr().Interface()) 1130 } 1131 1132 func unquote(s string) (string, error) { 1133 var ret string 1134 err := json.Unmarshal([]byte(s), &ret) 1135 return ret, err 1136 } 1137 1138 // jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute. 1139 func jsonProperties(f reflect.StructField, origName bool) *proto.Properties { 1140 var prop proto.Properties 1141 prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f) 1142 if origName || prop.JSONName == "" { 1143 prop.JSONName = prop.OrigName 1144 } 1145 return &prop 1146 } 1147 1148 type fieldNames struct { 1149 orig, camel string 1150 } 1151 1152 func acceptedJSONFieldNames(prop *proto.Properties) fieldNames { 1153 opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName} 1154 if prop.JSONName != "" { 1155 opts.camel = prop.JSONName 1156 } 1157 return opts 1158 } 1159 1160 // Writer wrapper inspired by https://blog.golang.org/errors-are-values 1161 type errWriter struct { 1162 writer io.Writer 1163 err error 1164 } 1165 1166 func (w *errWriter) write(str string) { 1167 if w.err != nil { 1168 return 1169 } 1170 _, w.err = w.writer.Write([]byte(str)) 1171 } 1172 1173 // Map fields may have key types of non-float scalars, strings and enums. 1174 // The easiest way to sort them in some deterministic order is to use fmt. 1175 // If this turns out to be inefficient we can always consider other options, 1176 // such as doing a Schwartzian transform. 1177 // 1178 // Numeric keys are sorted in numeric order per 1179 // https://developers.google.com/protocol-buffers/docs/proto#maps. 1180 type mapKeys []reflect.Value 1181 1182 func (s mapKeys) Len() int { return len(s) } 1183 func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 1184 func (s mapKeys) Less(i, j int) bool { 1185 if k := s[i].Kind(); k == s[j].Kind() { 1186 switch k { 1187 case reflect.String: 1188 return s[i].String() < s[j].String() 1189 case reflect.Int32, reflect.Int64: 1190 return s[i].Int() < s[j].Int() 1191 case reflect.Uint32, reflect.Uint64: 1192 return s[i].Uint() < s[j].Uint() 1193 } 1194 } 1195 return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface()) 1196 } 1197 1198 // checkRequiredFields returns an error if any required field in the given proto message is not set. 1199 // This function is used by both Marshal and Unmarshal. While required fields only exist in a 1200 // proto2 message, a proto3 message can contain proto2 message(s). 1201 func checkRequiredFields(pb proto.Message) error { 1202 // Most well-known type messages do not contain required fields. The "Any" type may contain 1203 // a message that has required fields. 1204 // 1205 // When an Any message is being marshaled, the code will invoked proto.Unmarshal on Any.Value 1206 // field in order to transform that into JSON, and that should have returned an error if a 1207 // required field is not set in the embedded message. 1208 // 1209 // When an Any message is being unmarshaled, the code will have invoked proto.Marshal on the 1210 // embedded message to store the serialized message in Any.Value field, and that should have 1211 // returned an error if a required field is not set. 1212 if _, ok := pb.(wkt); ok { 1213 return nil 1214 } 1215 1216 v := reflect.ValueOf(pb) 1217 // Skip message if it is not a struct pointer. 1218 if v.Kind() != reflect.Ptr { 1219 return nil 1220 } 1221 v = v.Elem() 1222 if v.Kind() != reflect.Struct { 1223 return nil 1224 } 1225 1226 for i := 0; i < v.NumField(); i++ { 1227 field := v.Field(i) 1228 sfield := v.Type().Field(i) 1229 1230 if sfield.PkgPath != "" { 1231 // blank PkgPath means the field is exported; skip if not exported 1232 continue 1233 } 1234 1235 if strings.HasPrefix(sfield.Name, "XXX_") { 1236 continue 1237 } 1238 1239 // Oneof field is an interface implemented by wrapper structs containing the actual oneof 1240 // field, i.e. an interface containing &T{real_value}. 1241 if sfield.Tag.Get("protobuf_oneof") != "" { 1242 if field.Kind() != reflect.Interface { 1243 continue 1244 } 1245 v := field.Elem() 1246 if v.Kind() != reflect.Ptr || v.IsNil() { 1247 continue 1248 } 1249 v = v.Elem() 1250 if v.Kind() != reflect.Struct || v.NumField() < 1 { 1251 continue 1252 } 1253 field = v.Field(0) 1254 sfield = v.Type().Field(0) 1255 } 1256 1257 protoTag := sfield.Tag.Get("protobuf") 1258 if protoTag == "" { 1259 continue 1260 } 1261 var prop proto.Properties 1262 prop.Init(sfield.Type, sfield.Name, protoTag, &sfield) 1263 1264 switch field.Kind() { 1265 case reflect.Map: 1266 if field.IsNil() { 1267 continue 1268 } 1269 // Check each map value. 1270 keys := field.MapKeys() 1271 for _, k := range keys { 1272 v := field.MapIndex(k) 1273 if err := checkRequiredFieldsInValue(v); err != nil { 1274 return err 1275 } 1276 } 1277 case reflect.Slice: 1278 // Handle non-repeated type, e.g. bytes. 1279 if !prop.Repeated { 1280 if prop.Required && field.IsNil() { 1281 return fmt.Errorf("required field %q is not set", prop.Name) 1282 } 1283 continue 1284 } 1285 1286 // Handle repeated type. 1287 if field.IsNil() { 1288 continue 1289 } 1290 // Check each slice item. 1291 for i := 0; i < field.Len(); i++ { 1292 v := field.Index(i) 1293 if err := checkRequiredFieldsInValue(v); err != nil { 1294 return err 1295 } 1296 } 1297 case reflect.Ptr: 1298 if field.IsNil() { 1299 if prop.Required { 1300 return fmt.Errorf("required field %q is not set", prop.Name) 1301 } 1302 continue 1303 } 1304 if err := checkRequiredFieldsInValue(field); err != nil { 1305 return err 1306 } 1307 } 1308 } 1309 1310 // Handle proto2 extensions. 1311 for _, ext := range proto.RegisteredExtensions(pb) { 1312 if !proto.HasExtension(pb, ext) { 1313 continue 1314 } 1315 ep, err := proto.GetExtension(pb, ext) 1316 if err != nil { 1317 return err 1318 } 1319 err = checkRequiredFieldsInValue(reflect.ValueOf(ep)) 1320 if err != nil { 1321 return err 1322 } 1323 } 1324 1325 return nil 1326 } 1327 1328 func checkRequiredFieldsInValue(v reflect.Value) error { 1329 if pm, ok := v.Interface().(proto.Message); ok { 1330 return checkRequiredFields(pm) 1331 } 1332 return nil 1333 } 1334 1335 //ErrBytesFormat 错误的bytes 类型 1336 var ErrBytesFormat = errors.New("ErrBytesFormat") 1337 1338 func parseBytes(jsonstr string, enableUTF8BytesToString bool) ([]byte, error) { 1339 if jsonstr == "" { 1340 return []byte{}, nil 1341 } 1342 if strings.HasPrefix(jsonstr, "str://") { 1343 return []byte(jsonstr[len("str://"):]), nil 1344 } 1345 if strings.HasPrefix(jsonstr, "0x") || strings.HasPrefix(jsonstr, "0X") { 1346 return common.FromHex(jsonstr) 1347 } 1348 //字符串不是 hex 格式, 也不是 str:// 格式,但是是一个普通的utf8 字符串 1349 //那么强制转化为bytes, 注意这个选项默认不开启. 1350 if utf8.ValidString(jsonstr) && enableUTF8BytesToString { 1351 return []byte(jsonstr), nil 1352 } 1353 return nil, ErrBytesFormat 1354 }