github.com/hamba/avro@v1.8.0/schema.go (about) 1 package avro 2 3 import ( 4 "crypto/md5" 5 "crypto/sha256" 6 "errors" 7 "fmt" 8 "hash" 9 "strconv" 10 "strings" 11 "sync/atomic" 12 13 "github.com/hamba/avro/pkg/crc64" 14 jsoniter "github.com/json-iterator/go" 15 "github.com/modern-go/concurrent" 16 ) 17 18 var nullDefault = struct{}{} 19 20 // Type is a schema type. 21 type Type string 22 23 // Schema type constants. 24 const ( 25 Record Type = "record" 26 Error Type = "error" 27 Ref Type = "<ref>" 28 Enum Type = "enum" 29 Array Type = "array" 30 Map Type = "map" 31 Union Type = "union" 32 Fixed Type = "fixed" 33 String Type = "string" 34 Bytes Type = "bytes" 35 Int Type = "int" 36 Long Type = "long" 37 Float Type = "float" 38 Double Type = "double" 39 Boolean Type = "boolean" 40 Null Type = "null" 41 ) 42 43 // LogicalType is a schema logical type. 44 type LogicalType string 45 46 // Schema logical type constants. 47 const ( 48 Decimal LogicalType = "decimal" 49 UUID LogicalType = "uuid" 50 Date LogicalType = "date" 51 TimeMillis LogicalType = "time-millis" 52 TimeMicros LogicalType = "time-micros" 53 TimestampMillis LogicalType = "timestamp-millis" 54 TimestampMicros LogicalType = "timestamp-micros" 55 Duration LogicalType = "duration" 56 ) 57 58 // FingerprintType is a fingerprinting algorithm. 59 type FingerprintType string 60 61 // Fingerprint type constants. 62 const ( 63 CRC64Avro FingerprintType = "CRC64-AVRO" 64 MD5 FingerprintType = "MD5" 65 SHA256 FingerprintType = "SHA256" 66 ) 67 68 var fingerprinters = map[FingerprintType]hash.Hash{ 69 CRC64Avro: crc64.New(), 70 MD5: md5.New(), 71 SHA256: sha256.New(), 72 } 73 74 // SchemaCache is a cache of schemas. 75 type SchemaCache struct { 76 cache concurrent.Map // map[string]Schema 77 } 78 79 // Add adds a schema to the cache with the given name. 80 func (c *SchemaCache) Add(name string, schema Schema) { 81 c.cache.Store(name, schema) 82 } 83 84 // Get returns the Schema if it exists. 85 func (c *SchemaCache) Get(name string) Schema { 86 if v, ok := c.cache.Load(name); ok { 87 return v.(Schema) 88 } 89 90 return nil 91 } 92 93 // Schemas is a slice of Schemas. 94 type Schemas []Schema 95 96 // Get gets a schema and position by type or name if it is a named schema. 97 func (s Schemas) Get(name string) (Schema, int) { 98 for i, schema := range s { 99 if schemaTypeName(schema) == name { 100 return schema, i 101 } 102 } 103 104 return nil, -1 105 } 106 107 // Schema represents an Avro schema. 108 type Schema interface { 109 // Type returns the type of the schema. 110 Type() Type 111 112 // String returns the canonical form of the schema. 113 String() string 114 115 // Fingerprint returns the SHA256 fingerprint of the schema. 116 Fingerprint() [32]byte 117 118 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 119 FingerprintUsing(FingerprintType) ([]byte, error) 120 } 121 122 // LogicalSchema represents an Avro schema with a logical type. 123 type LogicalSchema interface { 124 // Type returns the type of the logical schema. 125 Type() LogicalType 126 127 // String returns the canonical form of the logical schema. 128 String() string 129 } 130 131 // PropertySchema represents a schema with properties. 132 type PropertySchema interface { 133 // AddProp adds a property to the schema. 134 // 135 // AddProp will not overwrite existing properties. 136 AddProp(name string, value interface{}) 137 138 // Prop gets a property from the schema. 139 Prop(string) interface{} 140 } 141 142 // NamedSchema represents a schema with a name. 143 type NamedSchema interface { 144 Schema 145 PropertySchema 146 147 // Name returns the name of the schema. 148 Name() string 149 150 // Namespace returns the namespace of a schema. 151 Namespace() string 152 153 // FullName returns the full qualified name of a schema. 154 FullName() string 155 } 156 157 // LogicalTypeSchema represents a schema that can contain a logical type. 158 type LogicalTypeSchema interface { 159 // Logical returns the logical schema or nil. 160 Logical() LogicalSchema 161 } 162 163 type name struct { 164 name string 165 space string 166 full string 167 } 168 169 func newName(n, s string) (name, error) { 170 if idx := strings.LastIndexByte(n, '.'); idx > -1 { 171 s = n[:idx] 172 n = n[idx+1:] 173 } 174 175 full := n 176 if s != "" { 177 full = s + "." + n 178 } 179 180 for _, part := range strings.Split(full, ".") { 181 if err := validateName(part); err != nil { 182 return name{}, err 183 } 184 } 185 186 return name{ 187 name: n, 188 space: s, 189 full: full, 190 }, nil 191 } 192 193 // Name returns the name of a schema. 194 func (n name) Name() string { 195 return n.name 196 } 197 198 // Namespace returns the namespace of a schema. 199 func (n name) Namespace() string { 200 return n.space 201 } 202 203 // FullName returns the full qualified name of a schema. 204 func (n name) FullName() string { 205 return n.full 206 } 207 208 type fingerprinter struct { 209 fingerprint atomic.Value // [32]byte 210 cache concurrent.Map // map[FingerprintType][]byte 211 } 212 213 // Fingerprint returns the SHA256 fingerprint of the schema. 214 func (f *fingerprinter) Fingerprint(stringer fmt.Stringer) [32]byte { 215 if v := f.fingerprint.Load(); v != nil { 216 return v.([32]byte) 217 } 218 219 fingerprint := sha256.Sum256([]byte(stringer.String())) 220 f.fingerprint.Store(fingerprint) 221 return fingerprint 222 } 223 224 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 225 func (f *fingerprinter) FingerprintUsing(typ FingerprintType, stringer fmt.Stringer) ([]byte, error) { 226 if v, ok := f.cache.Load(typ); ok { 227 return v.([]byte), nil 228 } 229 230 h, ok := fingerprinters[typ] 231 if !ok { 232 return nil, fmt.Errorf("avro: unknown fingerprint algorithm %s", typ) 233 } 234 235 h.Reset() 236 _, _ = h.Write([]byte(stringer.String())) 237 fingerprint := h.Sum(make([]byte, 0, h.Size())) 238 f.cache.Store(typ, fingerprint) 239 return fingerprint, nil 240 } 241 242 type properties struct { 243 reserved []string 244 props map[string]interface{} 245 } 246 247 // AddProp adds a property to the schema. 248 // 249 // AddProp will not overwrite existing properties. 250 func (p *properties) AddProp(name string, value interface{}) { 251 // Create the props is needed. 252 if p.props == nil { 253 p.props = map[string]interface{}{} 254 } 255 256 // Dont allow reserved properties 257 for _, res := range p.reserved { 258 if name == res { 259 return 260 } 261 } 262 263 // Dont overwrite a property 264 if _, ok := p.props[name]; ok { 265 return 266 } 267 268 p.props[name] = value 269 } 270 271 // Prop gets a property from the schema. 272 func (p *properties) Prop(name string) interface{} { 273 if p.props == nil { 274 return nil 275 } 276 277 return p.props[name] 278 } 279 280 // PrimitiveSchema is an Avro primitive type schema. 281 type PrimitiveSchema struct { 282 properties 283 fingerprinter 284 285 typ Type 286 logical LogicalSchema 287 } 288 289 // NewPrimitiveSchema creates a new PrimitiveSchema. 290 func NewPrimitiveSchema(t Type, l LogicalSchema) *PrimitiveSchema { 291 return &PrimitiveSchema{ 292 properties: properties{reserved: schemaReserved}, 293 typ: t, 294 logical: l, 295 } 296 } 297 298 // Type returns the type of the schema. 299 func (s *PrimitiveSchema) Type() Type { 300 return s.typ 301 } 302 303 // Logical returns the logical schema or nil. 304 func (s *PrimitiveSchema) Logical() LogicalSchema { 305 return s.logical 306 } 307 308 // String returns the canonical form of the schema. 309 func (s *PrimitiveSchema) String() string { 310 if s.logical == nil { 311 return `"` + string(s.typ) + `"` 312 } 313 314 return `{"type":"` + string(s.typ) + `",` + s.logical.String() + `}` 315 } 316 317 // MarshalJSON marshals the schema to json. 318 func (s *PrimitiveSchema) MarshalJSON() ([]byte, error) { 319 return []byte(s.String()), nil 320 } 321 322 // Fingerprint returns the SHA256 fingerprint of the schema. 323 func (s *PrimitiveSchema) Fingerprint() [32]byte { 324 return s.fingerprinter.Fingerprint(s) 325 } 326 327 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 328 func (s *PrimitiveSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 329 return s.fingerprinter.FingerprintUsing(typ, s) 330 } 331 332 // RecordSchema is an Avro record type schema. 333 type RecordSchema struct { 334 name 335 properties 336 fingerprinter 337 338 isError bool 339 fields []*Field 340 doc string 341 } 342 343 // NewRecordSchema creates a new record schema instance. 344 func NewRecordSchema(name, space string, fields []*Field) (*RecordSchema, error) { 345 n, err := newName(name, space) 346 if err != nil { 347 return nil, err 348 } 349 350 return &RecordSchema{ 351 name: n, 352 properties: properties{reserved: schemaReserved}, 353 fields: fields, 354 }, nil 355 } 356 357 // NewErrorRecordSchema creates a new error record schema instance. 358 func NewErrorRecordSchema(name, space string, fields []*Field) (*RecordSchema, error) { 359 n, err := newName(name, space) 360 if err != nil { 361 return nil, err 362 } 363 364 return &RecordSchema{ 365 name: n, 366 properties: properties{reserved: schemaReserved}, 367 isError: true, 368 fields: fields, 369 }, nil 370 } 371 372 // Type returns the type of the schema. 373 func (s *RecordSchema) Type() Type { 374 return Record 375 } 376 377 // Doc returns the documentation of a record. 378 func (s *RecordSchema) Doc() string { 379 return s.doc 380 } 381 382 // IsError determines is this is an error record. 383 func (s *RecordSchema) IsError() bool { 384 return s.isError 385 } 386 387 // Fields returns the fields of a record. 388 func (s *RecordSchema) Fields() []*Field { 389 return s.fields 390 } 391 392 // String returns the canonical form of the schema. 393 func (s *RecordSchema) String() string { 394 typ := "record" 395 if s.isError { 396 typ = "error" 397 } 398 399 fields := "" 400 for _, f := range s.fields { 401 fields += f.String() + "," 402 } 403 if len(fields) > 0 { 404 fields = fields[:len(fields)-1] 405 } 406 407 return `{"name":"` + s.FullName() + `","type":"` + typ + `","fields":[` + fields + `]}` 408 } 409 410 // MarshalJSON marshals the schema to json. 411 func (s *RecordSchema) MarshalJSON() ([]byte, error) { 412 typ := "record" 413 if s.isError { 414 typ = "error" 415 } 416 417 ss := struct { 418 Name string `json:"name"` 419 Type string `json:"type"` 420 Fields []*Field `json:"fields"` 421 }{ 422 Name: s.FullName(), 423 Type: typ, 424 Fields: s.fields, 425 } 426 427 return jsoniter.Marshal(ss) 428 } 429 430 // Fingerprint returns the SHA256 fingerprint of the schema. 431 func (s *RecordSchema) Fingerprint() [32]byte { 432 return s.fingerprinter.Fingerprint(s) 433 } 434 435 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 436 func (s *RecordSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 437 return s.fingerprinter.FingerprintUsing(typ, s) 438 } 439 440 // AddDoc add documentation to the record. 441 func (s *RecordSchema) AddDoc(doc string) { 442 s.doc = doc 443 } 444 445 // Field is an Avro record type field. 446 type Field struct { 447 properties 448 449 name string 450 doc string 451 typ Schema 452 hasDef bool 453 def interface{} 454 } 455 456 type noDef struct{} 457 458 // NoDefault is used when no default exists for a field. 459 var NoDefault = noDef{} 460 461 // NewField creates a new field instance. 462 func NewField(name string, typ Schema, def interface{}) (*Field, error) { 463 if err := validateName(name); err != nil { 464 return nil, err 465 } 466 467 f := &Field{ 468 properties: properties{reserved: fieldReserved}, 469 name: name, 470 typ: typ, 471 } 472 473 if def != NoDefault { 474 def, err := validateDefault(name, typ, def) 475 if err != nil { 476 return nil, err 477 } 478 f.def = def 479 f.hasDef = true 480 } 481 482 return f, nil 483 } 484 485 // Name returns the name of a field. 486 func (f *Field) Name() string { 487 return f.name 488 } 489 490 // Type returns the schema of a field. 491 func (f *Field) Type() Schema { 492 return f.typ 493 } 494 495 // HasDefault determines if the field has a default value. 496 func (f *Field) HasDefault() bool { 497 return f.hasDef 498 } 499 500 // AddDoc add documentation to the field. 501 func (f *Field) AddDoc(doc string) { 502 f.doc = doc 503 } 504 505 // Default returns the default of a field or nil. 506 // 507 // The only time a nil default is valid is for a Null Type. 508 func (f *Field) Default() interface{} { 509 if f.def == nullDefault { 510 return nil 511 } 512 513 return f.def 514 } 515 516 // Doc returns the documentation of a field. 517 func (f *Field) Doc() string { 518 return f.doc 519 } 520 521 // String returns the canonical form of a field. 522 func (f *Field) String() string { 523 return `{"name":"` + f.name + `","type":` + f.typ.String() + `}` 524 } 525 526 // MarshalJSON marshals the schema to json. 527 func (f *Field) MarshalJSON() ([]byte, error) { 528 type base struct { 529 Name string `json:"name"` 530 Type Schema `json:"type"` 531 } 532 type ext struct { 533 base 534 Default interface{} `json:"default"` 535 } 536 var s interface{} = base{ 537 Name: f.name, 538 Type: f.typ, 539 } 540 if f.hasDef { 541 s = ext{ 542 base: s.(base), 543 Default: f.Default(), 544 } 545 } 546 return jsoniter.Marshal(s) 547 } 548 549 // EnumSchema is an Avro enum type schema. 550 type EnumSchema struct { 551 name 552 properties 553 fingerprinter 554 555 symbols []string 556 def string 557 } 558 559 // NewEnumSchema creates a new enum schema instance. 560 func NewEnumSchema(name, namespace string, symbols []string) (*EnumSchema, error) { 561 n, err := newName(name, namespace) 562 if err != nil { 563 return nil, err 564 } 565 566 if len(symbols) == 0 { 567 return nil, errors.New("avro: enum must have a non-empty array of symbols") 568 } 569 for _, symbol := range symbols { 570 if err = validateName(symbol); err != nil { 571 return nil, fmt.Errorf("avro: invalid symnol %s", symbol) 572 } 573 } 574 575 return &EnumSchema{ 576 name: n, 577 properties: properties{reserved: schemaReserved}, 578 symbols: symbols, 579 }, nil 580 } 581 582 // Type returns the type of the schema. 583 func (s *EnumSchema) Type() Type { 584 return Enum 585 } 586 587 // Symbols returns the symbols of an enum. 588 func (s *EnumSchema) Symbols() []string { 589 return s.symbols 590 } 591 592 // String returns the canonical form of the schema. 593 func (s *EnumSchema) String() string { 594 symbols := "" 595 for _, sym := range s.symbols { 596 symbols += `"` + sym + `",` 597 } 598 if len(symbols) > 0 { 599 symbols = symbols[:len(symbols)-1] 600 } 601 602 return `{"name":"` + s.FullName() + `","type":"enum","symbols":[` + symbols + `]}` 603 } 604 605 // MarshalJSON marshals the schema to json. 606 func (s *EnumSchema) MarshalJSON() ([]byte, error) { 607 ss := struct { 608 Name string `json:"name"` 609 Type string `json:"type"` 610 Symbols []string `json:"symbols"` 611 Default string `json:"default,omitempty"` 612 }{ 613 Name: s.FullName(), 614 Type: "enum", 615 Symbols: s.symbols, 616 Default: s.def, 617 } 618 return jsoniter.Marshal(ss) 619 } 620 621 // Fingerprint returns the SHA256 fingerprint of the schema. 622 func (s *EnumSchema) Fingerprint() [32]byte { 623 return s.fingerprinter.Fingerprint(s) 624 } 625 626 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 627 func (s *EnumSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 628 return s.fingerprinter.FingerprintUsing(typ, s) 629 } 630 631 // ArraySchema is an Avro array type schema. 632 type ArraySchema struct { 633 properties 634 fingerprinter 635 636 items Schema 637 } 638 639 // NewArraySchema creates an array schema instance. 640 func NewArraySchema(items Schema) *ArraySchema { 641 return &ArraySchema{ 642 properties: properties{reserved: schemaReserved}, 643 items: items, 644 } 645 } 646 647 // Type returns the type of the schema. 648 func (s *ArraySchema) Type() Type { 649 return Array 650 } 651 652 // Items returns the items schema of an array. 653 func (s *ArraySchema) Items() Schema { 654 return s.items 655 } 656 657 // String returns the canonical form of the schema. 658 func (s *ArraySchema) String() string { 659 return `{"type":"array","items":` + s.items.String() + `}` 660 } 661 662 // MarshalJSON marshals the schema to json. 663 func (s *ArraySchema) MarshalJSON() ([]byte, error) { 664 ss := struct { 665 Type string `json:"type"` 666 Items Schema `json:"items"` 667 }{ 668 Type: "array", 669 Items: s.items, 670 } 671 return jsoniter.Marshal(ss) 672 } 673 674 // Fingerprint returns the SHA256 fingerprint of the schema. 675 func (s *ArraySchema) Fingerprint() [32]byte { 676 return s.fingerprinter.Fingerprint(s) 677 } 678 679 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 680 func (s *ArraySchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 681 return s.fingerprinter.FingerprintUsing(typ, s) 682 } 683 684 // MapSchema is an Avro map type schema. 685 type MapSchema struct { 686 properties 687 fingerprinter 688 689 values Schema 690 } 691 692 // NewMapSchema creates a map schema instance. 693 func NewMapSchema(values Schema) *MapSchema { 694 return &MapSchema{ 695 properties: properties{reserved: schemaReserved}, 696 values: values, 697 } 698 } 699 700 // Type returns the type of the schema. 701 func (s *MapSchema) Type() Type { 702 return Map 703 } 704 705 // Values returns the values schema of a map. 706 func (s *MapSchema) Values() Schema { 707 return s.values 708 } 709 710 // String returns the canonical form of the schema. 711 func (s *MapSchema) String() string { 712 return `{"type":"map","values":` + s.values.String() + `}` 713 } 714 715 // MarshalJSON marshals the schema to json. 716 func (s *MapSchema) MarshalJSON() ([]byte, error) { 717 ss := struct { 718 Type string `json:"type"` 719 Values Schema `json:"values"` 720 }{ 721 Type: "map", 722 Values: s.values, 723 } 724 return jsoniter.Marshal(ss) 725 } 726 727 // Fingerprint returns the SHA256 fingerprint of the schema. 728 func (s *MapSchema) Fingerprint() [32]byte { 729 return s.fingerprinter.Fingerprint(s) 730 } 731 732 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 733 func (s *MapSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 734 return s.fingerprinter.FingerprintUsing(typ, s) 735 } 736 737 // UnionSchema is an Avro union type schema. 738 type UnionSchema struct { 739 fingerprinter 740 741 types Schemas 742 } 743 744 // NewUnionSchema creates a union schema instance. 745 func NewUnionSchema(types []Schema) (*UnionSchema, error) { 746 seen := map[string]bool{} 747 for _, schema := range types { 748 if schema.Type() == Union { 749 return nil, errors.New("avro: union type cannot be a union") 750 } 751 752 strType := schemaTypeName(schema) 753 754 if seen[strType] { 755 return nil, errors.New("avro: union type must be unique") 756 } 757 seen[strType] = true 758 } 759 760 return &UnionSchema{ 761 types: Schemas(types), 762 }, nil 763 } 764 765 // Type returns the type of the schema. 766 func (s *UnionSchema) Type() Type { 767 return Union 768 } 769 770 // Types returns the types of a union. 771 func (s *UnionSchema) Types() Schemas { 772 return s.types 773 } 774 775 // Nullable returns the Schema if the union is nullable, otherwise nil. 776 func (s *UnionSchema) Nullable() bool { 777 if len(s.types) != 2 || s.types[0].Type() != Null && s.types[1].Type() != Null { 778 return false 779 } 780 781 return true 782 } 783 784 // Indices returns the index of the null and type schemas for a 785 // nullable schema. For non-nullable schemas 0 is returned for 786 // both. 787 func (s *UnionSchema) Indices() (null, typ int) { 788 if !s.Nullable() { 789 return 0, 0 790 } 791 if s.types[0].Type() == Null { 792 return 0, 1 793 } 794 return 1, 0 795 } 796 797 // String returns the canonical form of the schema. 798 func (s *UnionSchema) String() string { 799 types := "" 800 for _, typ := range s.types { 801 types += typ.String() + "," 802 } 803 if len(types) > 0 { 804 types = types[:len(types)-1] 805 } 806 807 return `[` + types + `]` 808 } 809 810 // MarshalJSON marshals the schema to json. 811 func (s *UnionSchema) MarshalJSON() ([]byte, error) { 812 return jsoniter.Marshal(s.types) 813 } 814 815 // Fingerprint returns the SHA256 fingerprint of the schema. 816 func (s *UnionSchema) Fingerprint() [32]byte { 817 return s.fingerprinter.Fingerprint(s) 818 } 819 820 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 821 func (s *UnionSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 822 return s.fingerprinter.FingerprintUsing(typ, s) 823 } 824 825 // FixedSchema is an Avro fixed type schema. 826 type FixedSchema struct { 827 name 828 properties 829 fingerprinter 830 831 size int 832 logical LogicalSchema 833 } 834 835 // NewFixedSchema creates a new fixed schema instance. 836 func NewFixedSchema(name, namespace string, size int, logical LogicalSchema) (*FixedSchema, error) { 837 n, err := newName(name, namespace) 838 if err != nil { 839 return nil, err 840 } 841 842 return &FixedSchema{ 843 name: n, 844 properties: properties{reserved: schemaReserved}, 845 size: size, 846 logical: logical, 847 }, nil 848 } 849 850 // Type returns the type of the schema. 851 func (s *FixedSchema) Type() Type { 852 return Fixed 853 } 854 855 // Size returns the number of bytes of the fixed schema. 856 func (s *FixedSchema) Size() int { 857 return s.size 858 } 859 860 // Logical returns the logical schema or nil. 861 func (s *FixedSchema) Logical() LogicalSchema { 862 return s.logical 863 } 864 865 // String returns the canonical form of the schema. 866 func (s *FixedSchema) String() string { 867 size := strconv.Itoa(s.size) 868 869 var logical string 870 if s.logical != nil { 871 logical = "," + s.logical.String() 872 } 873 874 return `{"name":"` + s.FullName() + `","type":"fixed","size":` + size + logical + `}` 875 } 876 877 // MarshalJSON marshals the schema to json. 878 func (s *FixedSchema) MarshalJSON() ([]byte, error) { 879 return []byte(s.String()), nil 880 } 881 882 // Fingerprint returns the SHA256 fingerprint of the schema. 883 func (s *FixedSchema) Fingerprint() [32]byte { 884 return s.fingerprinter.Fingerprint(s) 885 } 886 887 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 888 func (s *FixedSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 889 return s.fingerprinter.FingerprintUsing(typ, s) 890 } 891 892 // NullSchema is an Avro null type schema. 893 type NullSchema struct { 894 fingerprinter 895 } 896 897 // Type returns the type of the schema. 898 func (s *NullSchema) Type() Type { 899 return Null 900 } 901 902 // String returns the canonical form of the schema. 903 func (s *NullSchema) String() string { 904 return `"null"` 905 } 906 907 // MarshalJSON marshals the schema to json. 908 func (s *NullSchema) MarshalJSON() ([]byte, error) { 909 return []byte(`"null"`), nil 910 } 911 912 // Fingerprint returns the SHA256 fingerprint of the schema. 913 func (s *NullSchema) Fingerprint() [32]byte { 914 return s.fingerprinter.Fingerprint(s) 915 } 916 917 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 918 func (s *NullSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 919 return s.fingerprinter.FingerprintUsing(typ, s) 920 } 921 922 // RefSchema is a reference to a named Avro schema. 923 type RefSchema struct { 924 actual NamedSchema 925 } 926 927 // NewRefSchema creates a ref schema instance. 928 func NewRefSchema(schema NamedSchema) *RefSchema { 929 return &RefSchema{ 930 actual: schema, 931 } 932 } 933 934 // Type returns the type of the schema. 935 func (s *RefSchema) Type() Type { 936 return Ref 937 } 938 939 // Schema returns the schema being referenced. 940 func (s *RefSchema) Schema() Schema { 941 return s.actual 942 } 943 944 // String returns the canonical form of the schema. 945 func (s *RefSchema) String() string { 946 return `"` + s.actual.FullName() + `"` 947 } 948 949 // MarshalJSON marshals the schema to json. 950 func (s *RefSchema) MarshalJSON() ([]byte, error) { 951 return []byte(`"` + s.actual.FullName() + `"`), nil 952 } 953 954 // Fingerprint returns the SHA256 fingerprint of the schema. 955 func (s *RefSchema) Fingerprint() [32]byte { 956 return s.actual.Fingerprint() 957 } 958 959 // FingerprintUsing returns the fingerprint of the schema using the given algorithm or an error. 960 func (s *RefSchema) FingerprintUsing(typ FingerprintType) ([]byte, error) { 961 return s.actual.FingerprintUsing(typ) 962 } 963 964 // PrimitiveLogicalSchema is a logical type with no properties. 965 type PrimitiveLogicalSchema struct { 966 typ LogicalType 967 } 968 969 // NewPrimitiveLogicalSchema creates a new primitive logical schema instance. 970 func NewPrimitiveLogicalSchema(typ LogicalType) *PrimitiveLogicalSchema { 971 return &PrimitiveLogicalSchema{ 972 typ: typ, 973 } 974 } 975 976 // Type returns the type of the logical schema. 977 func (s *PrimitiveLogicalSchema) Type() LogicalType { 978 return s.typ 979 } 980 981 // String returns the canonical form of the logical schema. 982 func (s *PrimitiveLogicalSchema) String() string { 983 return `"logicalType":"` + string(s.typ) + `"` 984 } 985 986 // DecimalLogicalSchema is a decimal logical type. 987 type DecimalLogicalSchema struct { 988 prec int 989 scale int 990 } 991 992 // NewDecimalLogicalSchema creates a new decimal logical schema instance. 993 func NewDecimalLogicalSchema(prec, scale int) *DecimalLogicalSchema { 994 return &DecimalLogicalSchema{ 995 prec: prec, 996 scale: scale, 997 } 998 } 999 1000 // Type returns the type of the logical schema. 1001 func (s *DecimalLogicalSchema) Type() LogicalType { 1002 return Decimal 1003 } 1004 1005 // Precision returns the precision of the decimal logical schema. 1006 func (s *DecimalLogicalSchema) Precision() int { 1007 return s.prec 1008 } 1009 1010 // Scale returns the scale of the decimal logical schema. 1011 func (s *DecimalLogicalSchema) Scale() int { 1012 return s.scale 1013 } 1014 1015 // String returns the canonical form of the logical schema. 1016 func (s *DecimalLogicalSchema) String() string { 1017 var scale string 1018 if s.scale > 0 { 1019 scale = `,"scale":` + strconv.Itoa(s.scale) 1020 } 1021 precision := strconv.Itoa(s.prec) 1022 1023 return `"logicalType":"` + string(Decimal) + `","precision":` + precision + scale 1024 } 1025 1026 func invalidNameFirstChar(r rune) bool { 1027 return (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && r != '_' 1028 } 1029 1030 func invalidNameOtherChar(r rune) bool { 1031 return invalidNameFirstChar(r) && (r < '0' || r > '9') 1032 } 1033 1034 func validateName(name string) error { 1035 if len(name) == 0 { 1036 return errors.New("avro: name must be a non-empty") 1037 } 1038 1039 if strings.IndexFunc(name[:1], invalidNameFirstChar) > -1 { 1040 return fmt.Errorf("avro: invalid name %s", name) 1041 } 1042 if strings.IndexFunc(name[1:], invalidNameOtherChar) > -1 { 1043 return fmt.Errorf("avro: invalid name %s", name) 1044 } 1045 1046 return nil 1047 } 1048 1049 func validateDefault(name string, schema Schema, def interface{}) (interface{}, error) { 1050 if def == nil { 1051 if schema.Type() != Null && !(schema.Type() == Union && schema.(*UnionSchema).Nullable()) { 1052 // This is an empty default value. 1053 return nil, nil 1054 } 1055 } 1056 1057 def, ok := isValidDefault(schema, def) 1058 if !ok { 1059 return nil, fmt.Errorf("avro: invalid default for field %s. %+v not a %s", name, def, schema.Type()) 1060 } 1061 1062 return def, nil 1063 } 1064 1065 func isValidDefault(schema Schema, def interface{}) (interface{}, bool) { 1066 switch schema.Type() { 1067 case Null: 1068 return nullDefault, def == nil 1069 1070 case String, Bytes, Enum, Fixed: 1071 if _, ok := def.(string); ok { 1072 return def, true 1073 } 1074 1075 case Boolean: 1076 if _, ok := def.(bool); ok { 1077 return def, true 1078 } 1079 1080 case Int: 1081 if i, ok := def.(int8); ok { 1082 return int(i), true 1083 } 1084 if i, ok := def.(int16); ok { 1085 return int(i), true 1086 } 1087 if i, ok := def.(int32); ok { 1088 return int(i), true 1089 } 1090 if _, ok := def.(int); ok { 1091 return def, true 1092 } 1093 if f, ok := def.(float64); ok { 1094 return int(f), true 1095 } 1096 1097 case Long: 1098 if _, ok := def.(int64); ok { 1099 return def, true 1100 } 1101 if f, ok := def.(float64); ok { 1102 return int64(f), true 1103 } 1104 1105 case Float: 1106 if _, ok := def.(float32); ok { 1107 return def, true 1108 } 1109 if f, ok := def.(float64); ok { 1110 return float32(f), true 1111 } 1112 1113 case Double: 1114 if _, ok := def.(float64); ok { 1115 return def, true 1116 } 1117 1118 case Array: 1119 arr, ok := def.([]interface{}) 1120 if !ok { 1121 return nil, false 1122 } 1123 1124 arrSchema := schema.(*ArraySchema) 1125 for i, v := range arr { 1126 v, ok := isValidDefault(arrSchema.Items(), v) 1127 if !ok { 1128 return nil, false 1129 } 1130 arr[i] = v 1131 } 1132 1133 return arr, true 1134 1135 case Map: 1136 m, ok := def.(map[string]interface{}) 1137 if !ok { 1138 return nil, false 1139 } 1140 1141 mapSchema := schema.(*MapSchema) 1142 for k, v := range m { 1143 v, ok := isValidDefault(mapSchema.Values(), v) 1144 if !ok { 1145 return nil, false 1146 } 1147 1148 m[k] = v 1149 } 1150 1151 return m, true 1152 1153 case Union: 1154 unionSchema := schema.(*UnionSchema) 1155 return isValidDefault(unionSchema.Types()[0], def) 1156 1157 case Record: 1158 m, ok := def.(map[string]interface{}) 1159 if !ok { 1160 return nil, false 1161 } 1162 1163 recordSchema := schema.(*RecordSchema) 1164 for _, field := range recordSchema.Fields() { 1165 fieldDef := field.Default() 1166 if newDef, ok := m[field.Name()]; ok { 1167 fieldDef = newDef 1168 } 1169 1170 v, ok := isValidDefault(field.Type(), fieldDef) 1171 if !ok { 1172 return nil, false 1173 } 1174 1175 m[field.Name()] = v 1176 } 1177 1178 return m, true 1179 } 1180 1181 return nil, false 1182 } 1183 1184 func schemaTypeName(schema Schema) string { 1185 if schema.Type() == Ref { 1186 schema = schema.(*RefSchema).Schema() 1187 } 1188 1189 if n, ok := schema.(NamedSchema); ok { 1190 return n.FullName() 1191 } 1192 1193 name := string(schema.Type()) 1194 if lt := getLogicalType(schema); lt != "" { 1195 name += "." + string(lt) 1196 } 1197 1198 return name 1199 }