github.com/qxnw/lib4go@v0.0.0-20180426074627-c80c7e84b925/influxdb/models/points.go (about) 1 // Package models implements basic objects used throughout the TICK stack. 2 package models 3 4 import ( 5 "bytes" 6 "encoding/binary" 7 "errors" 8 "fmt" 9 "io" 10 "math" 11 "sort" 12 "strconv" 13 "strings" 14 "time" 15 16 "github.com/qxnw/lib4go/influxdb/escape" 17 ) 18 19 var ( 20 measurementEscapeCodes = map[byte][]byte{ 21 ',': []byte(`\,`), 22 ' ': []byte(`\ `), 23 } 24 25 tagEscapeCodes = map[byte][]byte{ 26 ',': []byte(`\,`), 27 ' ': []byte(`\ `), 28 '=': []byte(`\=`), 29 } 30 31 // ErrPointMustHaveAField is returned when operating on a point that does not have any fields. 32 ErrPointMustHaveAField = errors.New("point without fields is unsupported") 33 34 // ErrInvalidNumber is returned when a number is expected but not provided. 35 ErrInvalidNumber = errors.New("invalid number") 36 37 // ErrInvalidPoint is returned when a point cannot be parsed correctly. 38 ErrInvalidPoint = errors.New("point is invalid") 39 ) 40 41 const ( 42 // MaxKeyLength is the largest allowed size of the combined measurement and tag keys. 43 MaxKeyLength = 65535 44 ) 45 46 // Point defines the values that will be written to the database. 47 type Point interface { 48 // Name return the measurement name for the point. 49 Name() string 50 51 // SetName updates the measurement name for the point. 52 SetName(string) 53 54 // Tags returns the tag set for the point. 55 Tags() Tags 56 57 // AddTag adds or replaces a tag value for a point. 58 AddTag(key, value string) 59 60 // SetTags replaces the tags for the point. 61 SetTags(tags Tags) 62 63 // Fields returns the fields for the point. 64 Fields() (Fields, error) 65 66 // Time return the timestamp for the point. 67 Time() time.Time 68 69 // SetTime updates the timestamp for the point. 70 SetTime(t time.Time) 71 72 // UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. 73 UnixNano() int64 74 75 // HashID returns a non-cryptographic checksum of the point's key. 76 HashID() uint64 77 78 // Key returns the key (measurement joined with tags) of the point. 79 Key() []byte 80 81 // String returns a string representation of the point. If there is a 82 // timestamp associated with the point then it will be specified with the default 83 // precision of nanoseconds. 84 String() string 85 86 // MarshalBinary returns a binary representation of the point. 87 MarshalBinary() ([]byte, error) 88 89 // PrecisionString returns a string representation of the point. If there 90 // is a timestamp associated with the point then it will be specified in the 91 // given unit. 92 PrecisionString(precision string) string 93 94 // RoundedString returns a string representation of the point. If there 95 // is a timestamp associated with the point, then it will be rounded to the 96 // given duration. 97 RoundedString(d time.Duration) string 98 99 // Split will attempt to return multiple points with the same timestamp whose 100 // string representations are no longer than size. Points with a single field or 101 // a point without a timestamp may exceed the requested size. 102 Split(size int) []Point 103 104 // Round will round the timestamp of the point to the given duration. 105 Round(d time.Duration) 106 107 // StringSize returns the length of the string that would be returned by String(). 108 StringSize() int 109 110 // AppendString appends the result of String() to the provided buffer and returns 111 // the result, potentially reducing string allocations. 112 AppendString(buf []byte) []byte 113 114 // FieldIterator retuns a FieldIterator that can be used to traverse the 115 // fields of a point without constructing the in-memory map. 116 FieldIterator() FieldIterator 117 } 118 119 // FieldType represents the type of a field. 120 type FieldType int 121 122 const ( 123 // Integer indicates the field's type is integer. 124 Integer FieldType = iota 125 126 // Float indicates the field's type is float. 127 Float 128 129 // Boolean indicates the field's type is boolean. 130 Boolean 131 132 // String indicates the field's type is string. 133 String 134 135 // Empty is used to indicate that there is no field. 136 Empty 137 ) 138 139 // FieldIterator provides a low-allocation interface to iterate through a point's fields. 140 type FieldIterator interface { 141 // Next indicates whether there any fields remaining. 142 Next() bool 143 144 // FieldKey returns the key of the current field. 145 FieldKey() []byte 146 147 // Type returns the FieldType of the current field. 148 Type() FieldType 149 150 // StringValue returns the string value of the current field. 151 StringValue() string 152 153 // IntegerValue returns the integer value of the current field. 154 IntegerValue() (int64, error) 155 156 // BooleanValue returns the boolean value of the current field. 157 BooleanValue() (bool, error) 158 159 // FloatValue returns the float value of the current field. 160 FloatValue() (float64, error) 161 162 // Delete deletes the current field. 163 Delete() 164 165 // Reset resets the iterator to its initial state. 166 Reset() 167 } 168 169 // Points represents a sortable list of points by timestamp. 170 type Points []Point 171 172 // Len implements sort.Interface. 173 func (a Points) Len() int { return len(a) } 174 175 // Less implements sort.Interface. 176 func (a Points) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) } 177 178 // Swap implements sort.Interface. 179 func (a Points) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 180 181 // point is the default implementation of Point. 182 type point struct { 183 time time.Time 184 185 // text encoding of measurement and tags 186 // key must always be stored sorted by tags, if the original line was not sorted, 187 // we need to resort it 188 key []byte 189 190 // text encoding of field data 191 fields []byte 192 193 // text encoding of timestamp 194 ts []byte 195 196 // cached version of parsed fields from data 197 cachedFields map[string]interface{} 198 199 // cached version of parsed name from key 200 cachedName string 201 202 // cached version of parsed tags 203 cachedTags Tags 204 205 it fieldIterator 206 } 207 208 const ( 209 // the number of characters for the largest possible int64 (9223372036854775807) 210 maxInt64Digits = 19 211 212 // the number of characters for the smallest possible int64 (-9223372036854775808) 213 minInt64Digits = 20 214 215 // the number of characters required for the largest float64 before a range check 216 // would occur during parsing 217 maxFloat64Digits = 25 218 219 // the number of characters required for smallest float64 before a range check occur 220 // would occur during parsing 221 minFloat64Digits = 27 222 ) 223 224 // ParsePoints returns a slice of Points from a text representation of a point 225 // with each point separated by newlines. If any points fail to parse, a non-nil error 226 // will be returned in addition to the points that parsed successfully. 227 func ParsePoints(buf []byte) ([]Point, error) { 228 return ParsePointsWithPrecision(buf, time.Now().UTC(), "n") 229 } 230 231 // ParsePointsString is identical to ParsePoints but accepts a string. 232 func ParsePointsString(buf string) ([]Point, error) { 233 return ParsePoints([]byte(buf)) 234 } 235 236 // ParseKey returns the measurement name and tags from a point. 237 // 238 // NOTE: to minimize heap allocations, the returned Tags will refer to subslices of buf. 239 // This can have the unintended effect preventing buf from being garbage collected. 240 func ParseKey(buf []byte) (string, Tags, error) { 241 // Ignore the error because scanMeasurement returns "missing fields" which we ignore 242 // when just parsing a key 243 state, i, _ := scanMeasurement(buf, 0) 244 245 var tags Tags 246 if state == tagKeyState { 247 tags = parseTags(buf) 248 // scanMeasurement returns the location of the comma if there are tags, strip that off 249 return string(buf[:i-1]), tags, nil 250 } 251 return string(buf[:i]), tags, nil 252 } 253 254 // ParsePointsWithPrecision is similar to ParsePoints, but allows the 255 // caller to provide a precision for time. 256 // 257 // NOTE: to minimize heap allocations, the returned Points will refer to subslices of buf. 258 // This can have the unintended effect preventing buf from being garbage collected. 259 func ParsePointsWithPrecision(buf []byte, defaultTime time.Time, precision string) ([]Point, error) { 260 points := make([]Point, 0, bytes.Count(buf, []byte{'\n'})+1) 261 var ( 262 pos int 263 block []byte 264 failed []string 265 ) 266 for pos < len(buf) { 267 pos, block = scanLine(buf, pos) 268 pos++ 269 270 if len(block) == 0 { 271 continue 272 } 273 274 // lines which start with '#' are comments 275 start := skipWhitespace(block, 0) 276 277 // If line is all whitespace, just skip it 278 if start >= len(block) { 279 continue 280 } 281 282 if block[start] == '#' { 283 continue 284 } 285 286 // strip the newline if one is present 287 if block[len(block)-1] == '\n' { 288 block = block[:len(block)-1] 289 } 290 291 pt, err := parsePoint(block[start:], defaultTime, precision) 292 if err != nil { 293 failed = append(failed, fmt.Sprintf("unable to parse '%s': %v", string(block[start:]), err)) 294 } else { 295 points = append(points, pt) 296 } 297 298 } 299 if len(failed) > 0 { 300 return points, fmt.Errorf("%s", strings.Join(failed, "\n")) 301 } 302 return points, nil 303 304 } 305 306 func parsePoint(buf []byte, defaultTime time.Time, precision string) (Point, error) { 307 // scan the first block which is measurement[,tag1=value1,tag2=value=2...] 308 pos, key, err := scanKey(buf, 0) 309 if err != nil { 310 return nil, err 311 } 312 313 // measurement name is required 314 if len(key) == 0 { 315 return nil, fmt.Errorf("missing measurement") 316 } 317 318 if len(key) > MaxKeyLength { 319 return nil, fmt.Errorf("max key length exceeded: %v > %v", len(key), MaxKeyLength) 320 } 321 322 // scan the second block is which is field1=value1[,field2=value2,...] 323 pos, fields, err := scanFields(buf, pos) 324 if err != nil { 325 return nil, err 326 } 327 328 // at least one field is required 329 if len(fields) == 0 { 330 return nil, fmt.Errorf("missing fields") 331 } 332 333 // scan the last block which is an optional integer timestamp 334 pos, ts, err := scanTime(buf, pos) 335 if err != nil { 336 return nil, err 337 } 338 339 pt := &point{ 340 key: key, 341 fields: fields, 342 ts: ts, 343 } 344 345 if len(ts) == 0 { 346 pt.time = defaultTime 347 pt.SetPrecision(precision) 348 } else { 349 ts, err := parseIntBytes(ts, 10, 64) 350 if err != nil { 351 return nil, err 352 } 353 pt.time, err = SafeCalcTime(ts, precision) 354 if err != nil { 355 return nil, err 356 } 357 358 // Determine if there are illegal non-whitespace characters after the 359 // timestamp block. 360 for pos < len(buf) { 361 if buf[pos] != ' ' { 362 return nil, ErrInvalidPoint 363 } 364 pos++ 365 } 366 } 367 return pt, nil 368 } 369 370 // GetPrecisionMultiplier will return a multiplier for the precision specified. 371 func GetPrecisionMultiplier(precision string) int64 { 372 d := time.Nanosecond 373 switch precision { 374 case "u": 375 d = time.Microsecond 376 case "ms": 377 d = time.Millisecond 378 case "s": 379 d = time.Second 380 case "m": 381 d = time.Minute 382 case "h": 383 d = time.Hour 384 } 385 return int64(d) 386 } 387 388 // scanKey scans buf starting at i for the measurement and tag portion of the point. 389 // It returns the ending position and the byte slice of key within buf. If there 390 // are tags, they will be sorted if they are not already. 391 func scanKey(buf []byte, i int) (int, []byte, error) { 392 start := skipWhitespace(buf, i) 393 394 i = start 395 396 // Determines whether the tags are sort, assume they are 397 sorted := true 398 399 // indices holds the indexes within buf of the start of each tag. For example, 400 // a buf of 'cpu,host=a,region=b,zone=c' would have indices slice of [4,11,20] 401 // which indicates that the first tag starts at buf[4], seconds at buf[11], and 402 // last at buf[20] 403 indices := make([]int, 100) 404 405 // tracks how many commas we've seen so we know how many values are indices. 406 // Since indices is an arbitrarily large slice, 407 // we need to know how many values in the buffer are in use. 408 commas := 0 409 410 // First scan the Point's measurement. 411 state, i, err := scanMeasurement(buf, i) 412 if err != nil { 413 return i, buf[start:i], err 414 } 415 416 // Optionally scan tags if needed. 417 if state == tagKeyState { 418 i, commas, indices, err = scanTags(buf, i, indices) 419 if err != nil { 420 return i, buf[start:i], err 421 } 422 } 423 424 // Now we know where the key region is within buf, and the location of tags, we 425 // need to determine if duplicate tags exist and if the tags are sorted. This iterates 426 // over the list comparing each tag in the sequence with each other. 427 for j := 0; j < commas-1; j++ { 428 // get the left and right tags 429 _, left := scanTo(buf[indices[j]:indices[j+1]-1], 0, '=') 430 _, right := scanTo(buf[indices[j+1]:indices[j+2]-1], 0, '=') 431 432 // If left is greater than right, the tags are not sorted. We do not have to 433 // continue because the short path no longer works. 434 // If the tags are equal, then there are duplicate tags, and we should abort. 435 // If the tags are not sorted, this pass may not find duplicate tags and we 436 // need to do a more exhaustive search later. 437 if cmp := bytes.Compare(left, right); cmp > 0 { 438 sorted = false 439 break 440 } else if cmp == 0 { 441 return i, buf[start:i], fmt.Errorf("duplicate tags") 442 } 443 } 444 445 // If the tags are not sorted, then sort them. This sort is inline and 446 // uses the tag indices we created earlier. The actual buffer is not sorted, the 447 // indices are using the buffer for value comparison. After the indices are sorted, 448 // the buffer is reconstructed from the sorted indices. 449 if !sorted && commas > 0 { 450 // Get the measurement name for later 451 measurement := buf[start : indices[0]-1] 452 453 // Sort the indices 454 indices := indices[:commas] 455 insertionSort(0, commas, buf, indices) 456 457 // Create a new key using the measurement and sorted indices 458 b := make([]byte, len(buf[start:i])) 459 pos := copy(b, measurement) 460 for _, i := range indices { 461 b[pos] = ',' 462 pos++ 463 _, v := scanToSpaceOr(buf, i, ',') 464 pos += copy(b[pos:], v) 465 } 466 467 // Check again for duplicate tags now that the tags are sorted. 468 for j := 0; j < commas-1; j++ { 469 // get the left and right tags 470 _, left := scanTo(buf[indices[j]:], 0, '=') 471 _, right := scanTo(buf[indices[j+1]:], 0, '=') 472 473 // If the tags are equal, then there are duplicate tags, and we should abort. 474 // If the tags are not sorted, this pass may not find duplicate tags and we 475 // need to do a more exhaustive search later. 476 if bytes.Equal(left, right) { 477 return i, b, fmt.Errorf("duplicate tags") 478 } 479 } 480 481 return i, b, nil 482 } 483 484 return i, buf[start:i], nil 485 } 486 487 // The following constants allow us to specify which state to move to 488 // next, when scanning sections of a Point. 489 const ( 490 tagKeyState = iota 491 tagValueState 492 fieldsState 493 ) 494 495 // scanMeasurement examines the measurement part of a Point, returning 496 // the next state to move to, and the current location in the buffer. 497 func scanMeasurement(buf []byte, i int) (int, int, error) { 498 // Check first byte of measurement, anything except a comma is fine. 499 // It can't be a space, since whitespace is stripped prior to this 500 // function call. 501 if i >= len(buf) || buf[i] == ',' { 502 return -1, i, fmt.Errorf("missing measurement") 503 } 504 505 for { 506 i++ 507 if i >= len(buf) { 508 // cpu 509 return -1, i, fmt.Errorf("missing fields") 510 } 511 512 if buf[i-1] == '\\' { 513 // Skip character (it's escaped). 514 continue 515 } 516 517 // Unescaped comma; move onto scanning the tags. 518 if buf[i] == ',' { 519 return tagKeyState, i + 1, nil 520 } 521 522 // Unescaped space; move onto scanning the fields. 523 if buf[i] == ' ' { 524 // cpu value=1.0 525 return fieldsState, i, nil 526 } 527 } 528 } 529 530 // scanTags examines all the tags in a Point, keeping track of and 531 // returning the updated indices slice, number of commas and location 532 // in buf where to start examining the Point fields. 533 func scanTags(buf []byte, i int, indices []int) (int, int, []int, error) { 534 var ( 535 err error 536 commas int 537 state = tagKeyState 538 ) 539 540 for { 541 switch state { 542 case tagKeyState: 543 // Grow our indices slice if we have too many tags. 544 if commas >= len(indices) { 545 newIndics := make([]int, cap(indices)*2) 546 copy(newIndics, indices) 547 indices = newIndics 548 } 549 indices[commas] = i 550 commas++ 551 552 i, err = scanTagsKey(buf, i) 553 state = tagValueState // tag value always follows a tag key 554 case tagValueState: 555 state, i, err = scanTagsValue(buf, i) 556 case fieldsState: 557 indices[commas] = i + 1 558 return i, commas, indices, nil 559 } 560 561 if err != nil { 562 return i, commas, indices, err 563 } 564 } 565 } 566 567 // scanTagsKey scans each character in a tag key. 568 func scanTagsKey(buf []byte, i int) (int, error) { 569 // First character of the key. 570 if i >= len(buf) || buf[i] == ' ' || buf[i] == ',' || buf[i] == '=' { 571 // cpu,{'', ' ', ',', '='} 572 return i, fmt.Errorf("missing tag key") 573 } 574 575 // Examine each character in the tag key until we hit an unescaped 576 // equals (the tag value), or we hit an error (i.e., unescaped 577 // space or comma). 578 for { 579 i++ 580 581 // Either we reached the end of the buffer or we hit an 582 // unescaped comma or space. 583 if i >= len(buf) || 584 ((buf[i] == ' ' || buf[i] == ',') && buf[i-1] != '\\') { 585 // cpu,tag{'', ' ', ','} 586 return i, fmt.Errorf("missing tag value") 587 } 588 589 if buf[i] == '=' && buf[i-1] != '\\' { 590 // cpu,tag= 591 return i + 1, nil 592 } 593 } 594 } 595 596 // scanTagsValue scans each character in a tag value. 597 func scanTagsValue(buf []byte, i int) (int, int, error) { 598 // Tag value cannot be empty. 599 if i >= len(buf) || buf[i] == ',' || buf[i] == ' ' { 600 // cpu,tag={',', ' '} 601 return -1, i, fmt.Errorf("missing tag value") 602 } 603 604 // Examine each character in the tag value until we hit an unescaped 605 // comma (move onto next tag key), an unescaped space (move onto 606 // fields), or we error out. 607 for { 608 i++ 609 if i >= len(buf) { 610 // cpu,tag=value 611 return -1, i, fmt.Errorf("missing fields") 612 } 613 614 // An unescaped equals sign is an invalid tag value. 615 if buf[i] == '=' && buf[i-1] != '\\' { 616 // cpu,tag={'=', 'fo=o'} 617 return -1, i, fmt.Errorf("invalid tag format") 618 } 619 620 if buf[i] == ',' && buf[i-1] != '\\' { 621 // cpu,tag=foo, 622 return tagKeyState, i + 1, nil 623 } 624 625 // cpu,tag=foo value=1.0 626 // cpu, tag=foo\= value=1.0 627 if buf[i] == ' ' && buf[i-1] != '\\' { 628 return fieldsState, i, nil 629 } 630 } 631 } 632 633 func insertionSort(l, r int, buf []byte, indices []int) { 634 for i := l + 1; i < r; i++ { 635 for j := i; j > l && less(buf, indices, j, j-1); j-- { 636 indices[j], indices[j-1] = indices[j-1], indices[j] 637 } 638 } 639 } 640 641 func less(buf []byte, indices []int, i, j int) bool { 642 // This grabs the tag names for i & j, it ignores the values 643 _, a := scanTo(buf, indices[i], '=') 644 _, b := scanTo(buf, indices[j], '=') 645 return bytes.Compare(a, b) < 0 646 } 647 648 // scanFields scans buf, starting at i for the fields section of a point. It returns 649 // the ending position and the byte slice of the fields within buf. 650 func scanFields(buf []byte, i int) (int, []byte, error) { 651 start := skipWhitespace(buf, i) 652 i = start 653 quoted := false 654 655 // tracks how many '=' we've seen 656 equals := 0 657 658 // tracks how many commas we've seen 659 commas := 0 660 661 for { 662 // reached the end of buf? 663 if i >= len(buf) { 664 break 665 } 666 667 // escaped characters? 668 if buf[i] == '\\' && i+1 < len(buf) { 669 i += 2 670 continue 671 } 672 673 // If the value is quoted, scan until we get to the end quote 674 // Only quote values in the field value since quotes are not significant 675 // in the field key 676 if buf[i] == '"' && equals > commas { 677 quoted = !quoted 678 i++ 679 continue 680 } 681 682 // If we see an =, ensure that there is at least on char before and after it 683 if buf[i] == '=' && !quoted { 684 equals++ 685 686 // check for "... =123" but allow "a\ =123" 687 if buf[i-1] == ' ' && buf[i-2] != '\\' { 688 return i, buf[start:i], fmt.Errorf("missing field key") 689 } 690 691 // check for "...a=123,=456" but allow "a=123,a\,=456" 692 if buf[i-1] == ',' && buf[i-2] != '\\' { 693 return i, buf[start:i], fmt.Errorf("missing field key") 694 } 695 696 // check for "... value=" 697 if i+1 >= len(buf) { 698 return i, buf[start:i], fmt.Errorf("missing field value") 699 } 700 701 // check for "... value=,value2=..." 702 if buf[i+1] == ',' || buf[i+1] == ' ' { 703 return i, buf[start:i], fmt.Errorf("missing field value") 704 } 705 706 if isNumeric(buf[i+1]) || buf[i+1] == '-' || buf[i+1] == 'N' || buf[i+1] == 'n' { 707 var err error 708 i, err = scanNumber(buf, i+1) 709 if err != nil { 710 return i, buf[start:i], err 711 } 712 continue 713 } 714 // If next byte is not a double-quote, the value must be a boolean 715 if buf[i+1] != '"' { 716 var err error 717 i, _, err = scanBoolean(buf, i+1) 718 if err != nil { 719 return i, buf[start:i], err 720 } 721 continue 722 } 723 } 724 725 if buf[i] == ',' && !quoted { 726 commas++ 727 } 728 729 // reached end of block? 730 if buf[i] == ' ' && !quoted { 731 break 732 } 733 i++ 734 } 735 736 if quoted { 737 return i, buf[start:i], fmt.Errorf("unbalanced quotes") 738 } 739 740 // check that all field sections had key and values (e.g. prevent "a=1,b" 741 if equals == 0 || commas != equals-1 { 742 return i, buf[start:i], fmt.Errorf("invalid field format") 743 } 744 745 return i, buf[start:i], nil 746 } 747 748 // scanTime scans buf, starting at i for the time section of a point. It 749 // returns the ending position and the byte slice of the timestamp within buf 750 // and and error if the timestamp is not in the correct numeric format. 751 func scanTime(buf []byte, i int) (int, []byte, error) { 752 start := skipWhitespace(buf, i) 753 i = start 754 755 for { 756 // reached the end of buf? 757 if i >= len(buf) { 758 break 759 } 760 761 // Reached end of block or trailing whitespace? 762 if buf[i] == '\n' || buf[i] == ' ' { 763 break 764 } 765 766 // Handle negative timestamps 767 if i == start && buf[i] == '-' { 768 i++ 769 continue 770 } 771 772 // Timestamps should be integers, make sure they are so we don't need 773 // to actually parse the timestamp until needed. 774 if buf[i] < '0' || buf[i] > '9' { 775 return i, buf[start:i], fmt.Errorf("bad timestamp") 776 } 777 i++ 778 } 779 return i, buf[start:i], nil 780 } 781 782 func isNumeric(b byte) bool { 783 return (b >= '0' && b <= '9') || b == '.' 784 } 785 786 // scanNumber returns the end position within buf, start at i after 787 // scanning over buf for an integer, or float. It returns an 788 // error if a invalid number is scanned. 789 func scanNumber(buf []byte, i int) (int, error) { 790 start := i 791 var isInt bool 792 793 // Is negative number? 794 if i < len(buf) && buf[i] == '-' { 795 i++ 796 // There must be more characters now, as just '-' is illegal. 797 if i == len(buf) { 798 return i, ErrInvalidNumber 799 } 800 } 801 802 // how many decimal points we've see 803 decimal := false 804 805 // indicates the number is float in scientific notation 806 scientific := false 807 808 for { 809 if i >= len(buf) { 810 break 811 } 812 813 if buf[i] == ',' || buf[i] == ' ' { 814 break 815 } 816 817 if buf[i] == 'i' && i > start && !isInt { 818 isInt = true 819 i++ 820 continue 821 } 822 823 if buf[i] == '.' { 824 // Can't have more than 1 decimal (e.g. 1.1.1 should fail) 825 if decimal { 826 return i, ErrInvalidNumber 827 } 828 decimal = true 829 } 830 831 // `e` is valid for floats but not as the first char 832 if i > start && (buf[i] == 'e' || buf[i] == 'E') { 833 scientific = true 834 i++ 835 continue 836 } 837 838 // + and - are only valid at this point if they follow an e (scientific notation) 839 if (buf[i] == '+' || buf[i] == '-') && (buf[i-1] == 'e' || buf[i-1] == 'E') { 840 i++ 841 continue 842 } 843 844 // NaN is an unsupported value 845 if i+2 < len(buf) && (buf[i] == 'N' || buf[i] == 'n') { 846 return i, ErrInvalidNumber 847 } 848 849 if !isNumeric(buf[i]) { 850 return i, ErrInvalidNumber 851 } 852 i++ 853 } 854 855 if isInt && (decimal || scientific) { 856 return i, ErrInvalidNumber 857 } 858 859 numericDigits := i - start 860 if isInt { 861 numericDigits-- 862 } 863 if decimal { 864 numericDigits-- 865 } 866 if buf[start] == '-' { 867 numericDigits-- 868 } 869 870 if numericDigits == 0 { 871 return i, ErrInvalidNumber 872 } 873 874 // It's more common that numbers will be within min/max range for their type but we need to prevent 875 // out or range numbers from being parsed successfully. This uses some simple heuristics to decide 876 // if we should parse the number to the actual type. It does not do it all the time because it incurs 877 // extra allocations and we end up converting the type again when writing points to disk. 878 if isInt { 879 // Make sure the last char is an 'i' for integers (e.g. 9i10 is not valid) 880 if buf[i-1] != 'i' { 881 return i, ErrInvalidNumber 882 } 883 // Parse the int to check bounds the number of digits could be larger than the max range 884 // We subtract 1 from the index to remove the `i` from our tests 885 if len(buf[start:i-1]) >= maxInt64Digits || len(buf[start:i-1]) >= minInt64Digits { 886 if _, err := parseIntBytes(buf[start:i-1], 10, 64); err != nil { 887 return i, fmt.Errorf("unable to parse integer %s: %s", buf[start:i-1], err) 888 } 889 } 890 } else { 891 // Parse the float to check bounds if it's scientific or the number of digits could be larger than the max range 892 if scientific || len(buf[start:i]) >= maxFloat64Digits || len(buf[start:i]) >= minFloat64Digits { 893 if _, err := parseFloatBytes(buf[start:i], 10); err != nil { 894 return i, fmt.Errorf("invalid float") 895 } 896 } 897 } 898 899 return i, nil 900 } 901 902 // scanBoolean returns the end position within buf, start at i after 903 // scanning over buf for boolean. Valid values for a boolean are 904 // t, T, true, TRUE, f, F, false, FALSE. It returns an error if a invalid boolean 905 // is scanned. 906 func scanBoolean(buf []byte, i int) (int, []byte, error) { 907 start := i 908 909 if i < len(buf) && (buf[i] != 't' && buf[i] != 'f' && buf[i] != 'T' && buf[i] != 'F') { 910 return i, buf[start:i], fmt.Errorf("invalid boolean") 911 } 912 913 i++ 914 for { 915 if i >= len(buf) { 916 break 917 } 918 919 if buf[i] == ',' || buf[i] == ' ' { 920 break 921 } 922 i++ 923 } 924 925 // Single char bool (t, T, f, F) is ok 926 if i-start == 1 { 927 return i, buf[start:i], nil 928 } 929 930 // length must be 4 for true or TRUE 931 if (buf[start] == 't' || buf[start] == 'T') && i-start != 4 { 932 return i, buf[start:i], fmt.Errorf("invalid boolean") 933 } 934 935 // length must be 5 for false or FALSE 936 if (buf[start] == 'f' || buf[start] == 'F') && i-start != 5 { 937 return i, buf[start:i], fmt.Errorf("invalid boolean") 938 } 939 940 // Otherwise 941 valid := false 942 switch buf[start] { 943 case 't': 944 valid = bytes.Equal(buf[start:i], []byte("true")) 945 case 'f': 946 valid = bytes.Equal(buf[start:i], []byte("false")) 947 case 'T': 948 valid = bytes.Equal(buf[start:i], []byte("TRUE")) || bytes.Equal(buf[start:i], []byte("True")) 949 case 'F': 950 valid = bytes.Equal(buf[start:i], []byte("FALSE")) || bytes.Equal(buf[start:i], []byte("False")) 951 } 952 953 if !valid { 954 return i, buf[start:i], fmt.Errorf("invalid boolean") 955 } 956 957 return i, buf[start:i], nil 958 959 } 960 961 // skipWhitespace returns the end position within buf, starting at i after 962 // scanning over spaces in tags. 963 func skipWhitespace(buf []byte, i int) int { 964 for i < len(buf) { 965 if buf[i] != ' ' && buf[i] != '\t' && buf[i] != 0 { 966 break 967 } 968 i++ 969 } 970 return i 971 } 972 973 // scanLine returns the end position in buf and the next line found within 974 // buf. 975 func scanLine(buf []byte, i int) (int, []byte) { 976 start := i 977 quoted := false 978 fields := false 979 980 // tracks how many '=' and commas we've seen 981 // this duplicates some of the functionality in scanFields 982 equals := 0 983 commas := 0 984 for { 985 // reached the end of buf? 986 if i >= len(buf) { 987 break 988 } 989 990 // skip past escaped characters 991 if buf[i] == '\\' { 992 i += 2 993 continue 994 } 995 996 if buf[i] == ' ' { 997 fields = true 998 } 999 1000 // If we see a double quote, makes sure it is not escaped 1001 if fields { 1002 if !quoted && buf[i] == '=' { 1003 i++ 1004 equals++ 1005 continue 1006 } else if !quoted && buf[i] == ',' { 1007 i++ 1008 commas++ 1009 continue 1010 } else if buf[i] == '"' && equals > commas { 1011 i++ 1012 quoted = !quoted 1013 continue 1014 } 1015 } 1016 1017 if buf[i] == '\n' && !quoted { 1018 break 1019 } 1020 1021 i++ 1022 } 1023 1024 return i, buf[start:i] 1025 } 1026 1027 // scanTo returns the end position in buf and the next consecutive block 1028 // of bytes, starting from i and ending with stop byte, where stop byte 1029 // has not been escaped. 1030 // 1031 // If there are leading spaces, they are skipped. 1032 func scanTo(buf []byte, i int, stop byte) (int, []byte) { 1033 start := i 1034 for { 1035 // reached the end of buf? 1036 if i >= len(buf) { 1037 break 1038 } 1039 1040 // Reached unescaped stop value? 1041 if buf[i] == stop && (i == 0 || buf[i-1] != '\\') { 1042 break 1043 } 1044 i++ 1045 } 1046 1047 return i, buf[start:i] 1048 } 1049 1050 // scanTo returns the end position in buf and the next consecutive block 1051 // of bytes, starting from i and ending with stop byte. If there are leading 1052 // spaces, they are skipped. 1053 func scanToSpaceOr(buf []byte, i int, stop byte) (int, []byte) { 1054 start := i 1055 if buf[i] == stop || buf[i] == ' ' { 1056 return i, buf[start:i] 1057 } 1058 1059 for { 1060 i++ 1061 if buf[i-1] == '\\' { 1062 continue 1063 } 1064 1065 // reached the end of buf? 1066 if i >= len(buf) { 1067 return i, buf[start:i] 1068 } 1069 1070 // reached end of block? 1071 if buf[i] == stop || buf[i] == ' ' { 1072 return i, buf[start:i] 1073 } 1074 } 1075 } 1076 1077 func scanTagValue(buf []byte, i int) (int, []byte) { 1078 start := i 1079 for { 1080 if i >= len(buf) { 1081 break 1082 } 1083 1084 if buf[i] == ',' && buf[i-1] != '\\' { 1085 break 1086 } 1087 i++ 1088 } 1089 if i > len(buf) { 1090 return i, nil 1091 } 1092 return i, buf[start:i] 1093 } 1094 1095 func scanFieldValue(buf []byte, i int) (int, []byte) { 1096 start := i 1097 quoted := false 1098 for i < len(buf) { 1099 // Only escape char for a field value is a double-quote and backslash 1100 if buf[i] == '\\' && i+1 < len(buf) && (buf[i+1] == '"' || buf[i+1] == '\\') { 1101 i += 2 1102 continue 1103 } 1104 1105 // Quoted value? (e.g. string) 1106 if buf[i] == '"' { 1107 i++ 1108 quoted = !quoted 1109 continue 1110 } 1111 1112 if buf[i] == ',' && !quoted { 1113 break 1114 } 1115 i++ 1116 } 1117 return i, buf[start:i] 1118 } 1119 1120 func escapeMeasurement(in []byte) []byte { 1121 for b, esc := range measurementEscapeCodes { 1122 in = bytes.Replace(in, []byte{b}, esc, -1) 1123 } 1124 return in 1125 } 1126 1127 func unescapeMeasurement(in []byte) []byte { 1128 for b, esc := range measurementEscapeCodes { 1129 in = bytes.Replace(in, esc, []byte{b}, -1) 1130 } 1131 return in 1132 } 1133 1134 func escapeTag(in []byte) []byte { 1135 for b, esc := range tagEscapeCodes { 1136 if bytes.IndexByte(in, b) != -1 { 1137 in = bytes.Replace(in, []byte{b}, esc, -1) 1138 } 1139 } 1140 return in 1141 } 1142 1143 func unescapeTag(in []byte) []byte { 1144 if bytes.IndexByte(in, '\\') == -1 { 1145 return in 1146 } 1147 1148 for b, esc := range tagEscapeCodes { 1149 if bytes.IndexByte(in, b) != -1 { 1150 in = bytes.Replace(in, esc, []byte{b}, -1) 1151 } 1152 } 1153 return in 1154 } 1155 1156 // escapeStringFieldReplacer replaces double quotes and backslashes 1157 // with the same character preceded by a backslash. 1158 // As of Go 1.7 this benchmarked better in allocations and CPU time 1159 // compared to iterating through a string byte-by-byte and appending to a new byte slice, 1160 // calling strings.Replace twice, and better than (*Regex).ReplaceAllString. 1161 var escapeStringFieldReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) 1162 1163 // EscapeStringField returns a copy of in with any double quotes or 1164 // backslashes with escaped values. 1165 func EscapeStringField(in string) string { 1166 return escapeStringFieldReplacer.Replace(in) 1167 } 1168 1169 // unescapeStringField returns a copy of in with any escaped double-quotes 1170 // or backslashes unescaped. 1171 func unescapeStringField(in string) string { 1172 if strings.IndexByte(in, '\\') == -1 { 1173 return in 1174 } 1175 1176 var out []byte 1177 i := 0 1178 for { 1179 if i >= len(in) { 1180 break 1181 } 1182 // unescape backslashes 1183 if in[i] == '\\' && i+1 < len(in) && in[i+1] == '\\' { 1184 out = append(out, '\\') 1185 i += 2 1186 continue 1187 } 1188 // unescape double-quotes 1189 if in[i] == '\\' && i+1 < len(in) && in[i+1] == '"' { 1190 out = append(out, '"') 1191 i += 2 1192 continue 1193 } 1194 out = append(out, in[i]) 1195 i++ 1196 1197 } 1198 return string(out) 1199 } 1200 1201 // NewPoint returns a new point with the given measurement name, tags, fields and timestamp. If 1202 // an unsupported field value (NaN) or out of range time is passed, this function returns an error. 1203 func NewPoint(name string, tags Tags, fields Fields, t time.Time) (Point, error) { 1204 key, err := pointKey(name, tags, fields, t) 1205 if err != nil { 1206 return nil, err 1207 } 1208 1209 return &point{ 1210 key: key, 1211 time: t, 1212 fields: fields.MarshalBinary(), 1213 }, nil 1214 } 1215 1216 // pointKey checks some basic requirements for valid points, and returns the 1217 // key, along with an possible error. 1218 func pointKey(measurement string, tags Tags, fields Fields, t time.Time) ([]byte, error) { 1219 if len(fields) == 0 { 1220 return nil, ErrPointMustHaveAField 1221 } 1222 1223 if !t.IsZero() { 1224 if err := CheckTime(t); err != nil { 1225 return nil, err 1226 } 1227 } 1228 1229 for key, value := range fields { 1230 switch value := value.(type) { 1231 case float64: 1232 // Ensure the caller validates and handles invalid field values 1233 if math.IsNaN(value) { 1234 return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) 1235 } 1236 case float32: 1237 // Ensure the caller validates and handles invalid field values 1238 if math.IsNaN(float64(value)) { 1239 return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) 1240 } 1241 } 1242 if len(key) == 0 { 1243 return nil, fmt.Errorf("all fields must have non-empty names") 1244 } 1245 } 1246 1247 key := MakeKey([]byte(measurement), tags) 1248 if len(key) > MaxKeyLength { 1249 return nil, fmt.Errorf("max key length exceeded: %v > %v", len(key), MaxKeyLength) 1250 } 1251 1252 return key, nil 1253 } 1254 1255 // NewPointFromBytes returns a new Point from a marshalled Point. 1256 func NewPointFromBytes(b []byte) (Point, error) { 1257 p := &point{} 1258 if err := p.UnmarshalBinary(b); err != nil { 1259 return nil, err 1260 } 1261 fields, err := p.Fields() 1262 if err != nil { 1263 return nil, err 1264 } 1265 if len(fields) == 0 { 1266 return nil, ErrPointMustHaveAField 1267 } 1268 return p, nil 1269 } 1270 1271 // MustNewPoint returns a new point with the given measurement name, tags, fields and timestamp. If 1272 // an unsupported field value (NaN) is passed, this function panics. 1273 func MustNewPoint(name string, tags Tags, fields Fields, time time.Time) Point { 1274 pt, err := NewPoint(name, tags, fields, time) 1275 if err != nil { 1276 panic(err.Error()) 1277 } 1278 return pt 1279 } 1280 1281 // Key returns the key (measurement joined with tags) of the point. 1282 func (p *point) Key() []byte { 1283 return p.key 1284 } 1285 1286 func (p *point) name() []byte { 1287 _, name := scanTo(p.key, 0, ',') 1288 return name 1289 } 1290 1291 // Name return the measurement name for the point. 1292 func (p *point) Name() string { 1293 if p.cachedName != "" { 1294 return p.cachedName 1295 } 1296 p.cachedName = string(escape.Unescape(p.name())) 1297 return p.cachedName 1298 } 1299 1300 // SetName updates the measurement name for the point. 1301 func (p *point) SetName(name string) { 1302 p.cachedName = "" 1303 p.key = MakeKey([]byte(name), p.Tags()) 1304 } 1305 1306 // Time return the timestamp for the point. 1307 func (p *point) Time() time.Time { 1308 return p.time 1309 } 1310 1311 // SetTime updates the timestamp for the point. 1312 func (p *point) SetTime(t time.Time) { 1313 p.time = t 1314 } 1315 1316 // Round will round the timestamp of the point to the given duration. 1317 func (p *point) Round(d time.Duration) { 1318 p.time = p.time.Round(d) 1319 } 1320 1321 // Tags returns the tag set for the point. 1322 func (p *point) Tags() Tags { 1323 if p.cachedTags != nil { 1324 return p.cachedTags 1325 } 1326 p.cachedTags = parseTags(p.key) 1327 return p.cachedTags 1328 } 1329 1330 func parseTags(buf []byte) Tags { 1331 if len(buf) == 0 { 1332 return nil 1333 } 1334 1335 pos, name := scanTo(buf, 0, ',') 1336 1337 // it's an empty key, so there are no tags 1338 if len(name) == 0 { 1339 return nil 1340 } 1341 1342 tags := make(Tags, 0, bytes.Count(buf, []byte(","))) 1343 hasEscape := bytes.IndexByte(buf, '\\') != -1 1344 1345 i := pos + 1 1346 var key, value []byte 1347 for { 1348 if i >= len(buf) { 1349 break 1350 } 1351 i, key = scanTo(buf, i, '=') 1352 i, value = scanTagValue(buf, i+1) 1353 1354 if len(value) == 0 { 1355 continue 1356 } 1357 1358 if hasEscape { 1359 tags = append(tags, Tag{Key: unescapeTag(key), Value: unescapeTag(value)}) 1360 } else { 1361 tags = append(tags, Tag{Key: key, Value: value}) 1362 } 1363 1364 i++ 1365 } 1366 1367 return tags 1368 } 1369 1370 // MakeKey creates a key for a set of tags. 1371 func MakeKey(name []byte, tags Tags) []byte { 1372 // unescape the name and then re-escape it to avoid double escaping. 1373 // The key should always be stored in escaped form. 1374 return append(escapeMeasurement(unescapeMeasurement(name)), tags.HashKey()...) 1375 } 1376 1377 // SetTags replaces the tags for the point. 1378 func (p *point) SetTags(tags Tags) { 1379 p.key = MakeKey([]byte(p.Name()), tags) 1380 p.cachedTags = tags 1381 } 1382 1383 // AddTag adds or replaces a tag value for a point. 1384 func (p *point) AddTag(key, value string) { 1385 tags := p.Tags() 1386 tags = append(tags, Tag{Key: []byte(key), Value: []byte(value)}) 1387 sort.Sort(tags) 1388 p.cachedTags = tags 1389 p.key = MakeKey([]byte(p.Name()), tags) 1390 } 1391 1392 // Fields returns the fields for the point. 1393 func (p *point) Fields() (Fields, error) { 1394 if p.cachedFields != nil { 1395 return p.cachedFields, nil 1396 } 1397 cf, err := p.unmarshalBinary() 1398 if err != nil { 1399 return nil, err 1400 } 1401 p.cachedFields = cf 1402 return p.cachedFields, nil 1403 } 1404 1405 // SetPrecision will round a time to the specified precision. 1406 func (p *point) SetPrecision(precision string) { 1407 switch precision { 1408 case "n": 1409 case "u": 1410 p.SetTime(p.Time().Truncate(time.Microsecond)) 1411 case "ms": 1412 p.SetTime(p.Time().Truncate(time.Millisecond)) 1413 case "s": 1414 p.SetTime(p.Time().Truncate(time.Second)) 1415 case "m": 1416 p.SetTime(p.Time().Truncate(time.Minute)) 1417 case "h": 1418 p.SetTime(p.Time().Truncate(time.Hour)) 1419 } 1420 } 1421 1422 // String returns the string representation of the point. 1423 func (p *point) String() string { 1424 if p.Time().IsZero() { 1425 return string(p.Key()) + " " + string(p.fields) 1426 } 1427 return string(p.Key()) + " " + string(p.fields) + " " + strconv.FormatInt(p.UnixNano(), 10) 1428 } 1429 1430 // AppendString appends the string representation of the point to buf. 1431 func (p *point) AppendString(buf []byte) []byte { 1432 buf = append(buf, p.key...) 1433 buf = append(buf, ' ') 1434 buf = append(buf, p.fields...) 1435 1436 if !p.time.IsZero() { 1437 buf = append(buf, ' ') 1438 buf = strconv.AppendInt(buf, p.UnixNano(), 10) 1439 } 1440 1441 return buf 1442 } 1443 1444 // StringSize returns the length of the string that would be returned by String(). 1445 func (p *point) StringSize() int { 1446 size := len(p.key) + len(p.fields) + 1 1447 1448 if !p.time.IsZero() { 1449 digits := 1 // even "0" has one digit 1450 t := p.UnixNano() 1451 if t < 0 { 1452 // account for negative sign, then negate 1453 digits++ 1454 t = -t 1455 } 1456 for t > 9 { // already accounted for one digit 1457 digits++ 1458 t /= 10 1459 } 1460 size += digits + 1 // digits and a space 1461 } 1462 1463 return size 1464 } 1465 1466 // MarshalBinary returns a binary representation of the point. 1467 func (p *point) MarshalBinary() ([]byte, error) { 1468 if len(p.fields) == 0 { 1469 return nil, ErrPointMustHaveAField 1470 } 1471 1472 tb, err := p.time.MarshalBinary() 1473 if err != nil { 1474 return nil, err 1475 } 1476 1477 b := make([]byte, 8+len(p.key)+len(p.fields)+len(tb)) 1478 i := 0 1479 1480 binary.BigEndian.PutUint32(b[i:], uint32(len(p.key))) 1481 i += 4 1482 1483 i += copy(b[i:], p.key) 1484 1485 binary.BigEndian.PutUint32(b[i:i+4], uint32(len(p.fields))) 1486 i += 4 1487 1488 i += copy(b[i:], p.fields) 1489 1490 copy(b[i:], tb) 1491 return b, nil 1492 } 1493 1494 // UnmarshalBinary decodes a binary representation of the point into a point struct. 1495 func (p *point) UnmarshalBinary(b []byte) error { 1496 var n int 1497 1498 // Read key length. 1499 if len(b) < 4 { 1500 return io.ErrShortBuffer 1501 } 1502 n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] 1503 1504 // Read key. 1505 if len(b) < n { 1506 return io.ErrShortBuffer 1507 } 1508 p.key, b = b[:n], b[n:] 1509 1510 // Read fields length. 1511 if len(b) < 4 { 1512 return io.ErrShortBuffer 1513 } 1514 n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] 1515 1516 // Read fields. 1517 if len(b) < n { 1518 return io.ErrShortBuffer 1519 } 1520 p.fields, b = b[:n], b[n:] 1521 1522 // Read timestamp. 1523 if err := p.time.UnmarshalBinary(b); err != nil { 1524 return err 1525 } 1526 return nil 1527 } 1528 1529 // PrecisionString returns a string representation of the point. If there 1530 // is a timestamp associated with the point then it will be specified in the 1531 // given unit. 1532 func (p *point) PrecisionString(precision string) string { 1533 if p.Time().IsZero() { 1534 return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) 1535 } 1536 return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), 1537 p.UnixNano()/GetPrecisionMultiplier(precision)) 1538 } 1539 1540 // RoundedString returns a string representation of the point. If there 1541 // is a timestamp associated with the point, then it will be rounded to the 1542 // given duration. 1543 func (p *point) RoundedString(d time.Duration) string { 1544 if p.Time().IsZero() { 1545 return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) 1546 } 1547 return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), 1548 p.time.Round(d).UnixNano()) 1549 } 1550 1551 func (p *point) unmarshalBinary() (Fields, error) { 1552 iter := p.FieldIterator() 1553 fields := make(Fields, 8) 1554 for iter.Next() { 1555 if len(iter.FieldKey()) == 0 { 1556 continue 1557 } 1558 switch iter.Type() { 1559 case Float: 1560 v, err := iter.FloatValue() 1561 if err != nil { 1562 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1563 } 1564 fields[string(iter.FieldKey())] = v 1565 case Integer: 1566 v, err := iter.IntegerValue() 1567 if err != nil { 1568 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1569 } 1570 fields[string(iter.FieldKey())] = v 1571 case String: 1572 fields[string(iter.FieldKey())] = iter.StringValue() 1573 case Boolean: 1574 v, err := iter.BooleanValue() 1575 if err != nil { 1576 return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) 1577 } 1578 fields[string(iter.FieldKey())] = v 1579 } 1580 } 1581 return fields, nil 1582 } 1583 1584 // HashID returns a non-cryptographic checksum of the point's key. 1585 func (p *point) HashID() uint64 { 1586 h := NewInlineFNV64a() 1587 h.Write(p.key) 1588 sum := h.Sum64() 1589 return sum 1590 } 1591 1592 // UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. 1593 func (p *point) UnixNano() int64 { 1594 return p.Time().UnixNano() 1595 } 1596 1597 // Split will attempt to return multiple points with the same timestamp whose 1598 // string representations are no longer than size. Points with a single field or 1599 // a point without a timestamp may exceed the requested size. 1600 func (p *point) Split(size int) []Point { 1601 if p.time.IsZero() || len(p.String()) <= size { 1602 return []Point{p} 1603 } 1604 1605 // key string, timestamp string, spaces 1606 size -= len(p.key) + len(strconv.FormatInt(p.time.UnixNano(), 10)) + 2 1607 1608 var points []Point 1609 var start, cur int 1610 1611 for cur < len(p.fields) { 1612 end, _ := scanTo(p.fields, cur, '=') 1613 end, _ = scanFieldValue(p.fields, end+1) 1614 1615 if cur > start && end-start > size { 1616 points = append(points, &point{ 1617 key: p.key, 1618 time: p.time, 1619 fields: p.fields[start : cur-1], 1620 }) 1621 start = cur 1622 } 1623 1624 cur = end + 1 1625 } 1626 1627 points = append(points, &point{ 1628 key: p.key, 1629 time: p.time, 1630 fields: p.fields[start:], 1631 }) 1632 1633 return points 1634 } 1635 1636 // Tag represents a single key/value tag pair. 1637 type Tag struct { 1638 Key []byte 1639 Value []byte 1640 } 1641 1642 // Clone returns a shallow copy of Tag. 1643 // 1644 // Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. 1645 // Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. 1646 func (t Tag) Clone() Tag { 1647 other := Tag{ 1648 Key: make([]byte, len(t.Key)), 1649 Value: make([]byte, len(t.Value)), 1650 } 1651 1652 copy(other.Key, t.Key) 1653 copy(other.Value, t.Value) 1654 1655 return other 1656 } 1657 1658 // Tags represents a sorted list of tags. 1659 type Tags []Tag 1660 1661 // NewTags returns a new Tags from a map. 1662 func NewTags(m map[string]string) Tags { 1663 if len(m) == 0 { 1664 return nil 1665 } 1666 a := make(Tags, 0, len(m)) 1667 for k, v := range m { 1668 a = append(a, Tag{Key: []byte(k), Value: []byte(v)}) 1669 } 1670 sort.Sort(a) 1671 return a 1672 } 1673 1674 // Clone returns a copy of the slice where the elements are a result of calling `Clone` on the original elements 1675 // 1676 // Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. 1677 // Use Clone to create Tags with new byte slices that do not refer to the argument to ParsePointsWithPrecision. 1678 func (a Tags) Clone() Tags { 1679 if len(a) == 0 { 1680 return nil 1681 } 1682 1683 others := make(Tags, len(a)) 1684 for i := range a { 1685 others[i] = a[i].Clone() 1686 } 1687 1688 return others 1689 } 1690 1691 // Len implements sort.Interface. 1692 func (a Tags) Len() int { return len(a) } 1693 1694 // Less implements sort.Interface. 1695 func (a Tags) Less(i, j int) bool { return bytes.Compare(a[i].Key, a[j].Key) == -1 } 1696 1697 // Swap implements sort.Interface. 1698 func (a Tags) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 1699 1700 // Get returns the value for a key. 1701 func (a Tags) Get(key []byte) []byte { 1702 // OPTIMIZE: Use sort.Search if tagset is large. 1703 1704 for _, t := range a { 1705 if bytes.Equal(t.Key, key) { 1706 return t.Value 1707 } 1708 } 1709 return nil 1710 } 1711 1712 // GetString returns the string value for a string key. 1713 func (a Tags) GetString(key string) string { 1714 return string(a.Get([]byte(key))) 1715 } 1716 1717 // Set sets the value for a key. 1718 func (a *Tags) Set(key, value []byte) { 1719 for _, t := range *a { 1720 if bytes.Equal(t.Key, key) { 1721 t.Value = value 1722 return 1723 } 1724 } 1725 *a = append(*a, Tag{Key: key, Value: value}) 1726 sort.Sort(*a) 1727 } 1728 1729 // SetString sets the string value for a string key. 1730 func (a *Tags) SetString(key, value string) { 1731 a.Set([]byte(key), []byte(value)) 1732 } 1733 1734 // Delete removes a tag by key. 1735 func (a *Tags) Delete(key []byte) { 1736 for i, t := range *a { 1737 if bytes.Equal(t.Key, key) { 1738 copy((*a)[i:], (*a)[i+1:]) 1739 (*a)[len(*a)-1] = Tag{} 1740 *a = (*a)[:len(*a)-1] 1741 return 1742 } 1743 } 1744 } 1745 1746 // Map returns a map representation of the tags. 1747 func (a Tags) Map() map[string]string { 1748 m := make(map[string]string, len(a)) 1749 for _, t := range a { 1750 m[string(t.Key)] = string(t.Value) 1751 } 1752 return m 1753 } 1754 1755 // Merge merges the tags combining the two. If both define a tag with the 1756 // same key, the merged value overwrites the old value. 1757 // A new map is returned. 1758 func (a Tags) Merge(other map[string]string) Tags { 1759 merged := make(map[string]string, len(a)+len(other)) 1760 for _, t := range a { 1761 merged[string(t.Key)] = string(t.Value) 1762 } 1763 for k, v := range other { 1764 merged[k] = v 1765 } 1766 return NewTags(merged) 1767 } 1768 1769 // HashKey hashes all of a tag's keys. 1770 func (a Tags) HashKey() []byte { 1771 // Empty maps marshal to empty bytes. 1772 if len(a) == 0 { 1773 return nil 1774 } 1775 1776 escaped := make(Tags, 0, len(a)) 1777 for _, t := range a { 1778 ek := escapeTag(t.Key) 1779 ev := escapeTag(t.Value) 1780 1781 if len(ev) > 0 { 1782 escaped = append(escaped, Tag{Key: ek, Value: ev}) 1783 } 1784 } 1785 1786 // Extract keys and determine final size. 1787 sz := len(escaped) + (len(escaped) * 2) // separators 1788 keys := make([][]byte, len(escaped)+1) 1789 for i, t := range escaped { 1790 keys[i] = t.Key 1791 sz += len(t.Key) + len(t.Value) 1792 } 1793 keys = keys[:len(escaped)] 1794 sort.Sort(byteSlices(keys)) 1795 1796 // Generate marshaled bytes. 1797 b := make([]byte, sz) 1798 buf := b 1799 idx := 0 1800 for i, k := range keys { 1801 buf[idx] = ',' 1802 idx++ 1803 copy(buf[idx:idx+len(k)], k) 1804 idx += len(k) 1805 buf[idx] = '=' 1806 idx++ 1807 v := escaped[i].Value 1808 copy(buf[idx:idx+len(v)], v) 1809 idx += len(v) 1810 } 1811 return b[:idx] 1812 } 1813 1814 // Fields represents a mapping between a Point's field names and their 1815 // values. 1816 type Fields map[string]interface{} 1817 1818 // FieldIterator retuns a FieldIterator that can be used to traverse the 1819 // fields of a point without constructing the in-memory map. 1820 func (p *point) FieldIterator() FieldIterator { 1821 p.Reset() 1822 return p 1823 } 1824 1825 type fieldIterator struct { 1826 start, end int 1827 key, keybuf []byte 1828 valueBuf []byte 1829 fieldType FieldType 1830 } 1831 1832 // Next indicates whether there any fields remaining. 1833 func (p *point) Next() bool { 1834 p.it.start = p.it.end 1835 if p.it.start >= len(p.fields) { 1836 return false 1837 } 1838 1839 p.it.end, p.it.key = scanTo(p.fields, p.it.start, '=') 1840 if escape.IsEscaped(p.it.key) { 1841 p.it.keybuf = escape.AppendUnescaped(p.it.keybuf[:0], p.it.key) 1842 p.it.key = p.it.keybuf 1843 } 1844 1845 p.it.end, p.it.valueBuf = scanFieldValue(p.fields, p.it.end+1) 1846 p.it.end++ 1847 1848 if len(p.it.valueBuf) == 0 { 1849 p.it.fieldType = Empty 1850 return true 1851 } 1852 1853 c := p.it.valueBuf[0] 1854 1855 if c == '"' { 1856 p.it.fieldType = String 1857 return true 1858 } 1859 1860 if strings.IndexByte(`0123456789-.nNiI`, c) >= 0 { 1861 if p.it.valueBuf[len(p.it.valueBuf)-1] == 'i' { 1862 p.it.fieldType = Integer 1863 p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] 1864 } else { 1865 p.it.fieldType = Float 1866 } 1867 return true 1868 } 1869 1870 // to keep the same behavior that currently exists, default to boolean 1871 p.it.fieldType = Boolean 1872 return true 1873 } 1874 1875 // FieldKey returns the key of the current field. 1876 func (p *point) FieldKey() []byte { 1877 return p.it.key 1878 } 1879 1880 // Type returns the FieldType of the current field. 1881 func (p *point) Type() FieldType { 1882 return p.it.fieldType 1883 } 1884 1885 // StringValue returns the string value of the current field. 1886 func (p *point) StringValue() string { 1887 return unescapeStringField(string(p.it.valueBuf[1 : len(p.it.valueBuf)-1])) 1888 } 1889 1890 // IntegerValue returns the integer value of the current field. 1891 func (p *point) IntegerValue() (int64, error) { 1892 n, err := parseIntBytes(p.it.valueBuf, 10, 64) 1893 if err != nil { 1894 return 0, fmt.Errorf("unable to parse integer value %q: %v", p.it.valueBuf, err) 1895 } 1896 return n, nil 1897 } 1898 1899 // BooleanValue returns the boolean value of the current field. 1900 func (p *point) BooleanValue() (bool, error) { 1901 b, err := parseBoolBytes(p.it.valueBuf) 1902 if err != nil { 1903 return false, fmt.Errorf("unable to parse bool value %q: %v", p.it.valueBuf, err) 1904 } 1905 return b, nil 1906 } 1907 1908 // FloatValue returns the float value of the current field. 1909 func (p *point) FloatValue() (float64, error) { 1910 f, err := parseFloatBytes(p.it.valueBuf, 64) 1911 if err != nil { 1912 return 0, fmt.Errorf("unable to parse floating point value %q: %v", p.it.valueBuf, err) 1913 } 1914 return f, nil 1915 } 1916 1917 // Delete deletes the current field. 1918 func (p *point) Delete() { 1919 switch { 1920 case p.it.end == p.it.start: 1921 case p.it.end >= len(p.fields): 1922 p.fields = p.fields[:p.it.start] 1923 case p.it.start == 0: 1924 p.fields = p.fields[p.it.end:] 1925 default: 1926 p.fields = append(p.fields[:p.it.start], p.fields[p.it.end:]...) 1927 } 1928 1929 p.it.end = p.it.start 1930 p.it.key = nil 1931 p.it.valueBuf = nil 1932 p.it.fieldType = Empty 1933 } 1934 1935 // Reset resets the iterator to its initial state. 1936 func (p *point) Reset() { 1937 p.it.fieldType = Empty 1938 p.it.key = nil 1939 p.it.valueBuf = nil 1940 p.it.start = 0 1941 p.it.end = 0 1942 } 1943 1944 // MarshalBinary encodes all the fields to their proper type and returns the binary 1945 // represenation 1946 // NOTE: uint64 is specifically not supported due to potential overflow when we decode 1947 // again later to an int64 1948 // NOTE2: uint is accepted, and may be 64 bits, and is for some reason accepted... 1949 func (p Fields) MarshalBinary() []byte { 1950 var b []byte 1951 keys := make([]string, 0, len(p)) 1952 1953 for k := range p { 1954 keys = append(keys, k) 1955 } 1956 1957 // Not really necessary, can probably be removed. 1958 sort.Strings(keys) 1959 1960 for i, k := range keys { 1961 if i > 0 { 1962 b = append(b, ',') 1963 } 1964 b = appendField(b, k, p[k]) 1965 } 1966 1967 return b 1968 } 1969 1970 func appendField(b []byte, k string, v interface{}) []byte { 1971 b = append(b, []byte(escape.String(k))...) 1972 b = append(b, '=') 1973 1974 // check popular types first 1975 switch v := v.(type) { 1976 case float64: 1977 b = strconv.AppendFloat(b, v, 'f', -1, 64) 1978 case int64: 1979 b = strconv.AppendInt(b, v, 10) 1980 b = append(b, 'i') 1981 case string: 1982 b = append(b, '"') 1983 b = append(b, []byte(EscapeStringField(v))...) 1984 b = append(b, '"') 1985 case bool: 1986 b = strconv.AppendBool(b, v) 1987 case int32: 1988 b = strconv.AppendInt(b, int64(v), 10) 1989 b = append(b, 'i') 1990 case int16: 1991 b = strconv.AppendInt(b, int64(v), 10) 1992 b = append(b, 'i') 1993 case int8: 1994 b = strconv.AppendInt(b, int64(v), 10) 1995 b = append(b, 'i') 1996 case int: 1997 b = strconv.AppendInt(b, int64(v), 10) 1998 b = append(b, 'i') 1999 case uint32: 2000 b = strconv.AppendInt(b, int64(v), 10) 2001 b = append(b, 'i') 2002 case uint16: 2003 b = strconv.AppendInt(b, int64(v), 10) 2004 b = append(b, 'i') 2005 case uint8: 2006 b = strconv.AppendInt(b, int64(v), 10) 2007 b = append(b, 'i') 2008 // TODO: 'uint' should be considered just as "dangerous" as a uint64, 2009 // perhaps the value should be checked and capped at MaxInt64? We could 2010 // then include uint64 as an accepted value 2011 case uint: 2012 b = strconv.AppendInt(b, int64(v), 10) 2013 b = append(b, 'i') 2014 case float32: 2015 b = strconv.AppendFloat(b, float64(v), 'f', -1, 32) 2016 case []byte: 2017 b = append(b, v...) 2018 case nil: 2019 // skip 2020 default: 2021 // Can't determine the type, so convert to string 2022 b = append(b, '"') 2023 b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...) 2024 b = append(b, '"') 2025 2026 } 2027 2028 return b 2029 } 2030 2031 type byteSlices [][]byte 2032 2033 func (a byteSlices) Len() int { return len(a) } 2034 func (a byteSlices) Less(i, j int) bool { return bytes.Compare(a[i], a[j]) == -1 } 2035 func (a byteSlices) Swap(i, j int) { a[i], a[j] = a[j], a[i] }