github.com/niubaoshu/goutils@v0.0.0-20180828035119-e8e576f66c2b/utf8.go (about)

     1  package goutils
     2  
     3  import (
     4  	"unicode/utf8"
     5  )
     6  
     7  func encodeutf8(p []byte) []byte {
     8  	var buf = make([]byte, 0, 100)
     9  	for len(p) > 0 {
    10  		r, l := utf8.DecodeRune(p)
    11  		if r != utf8.RuneError {
    12  			buf = encUint32(uint32(r), buf)
    13  		}
    14  		p = p[l:]
    15  	}
    16  	return buf
    17  }
    18  
    19  func decodeutf8(buf []byte) []byte {
    20  	var p = make([]byte, 10000)
    21  	sl := 0
    22  	for len(buf) > 0 {
    23  		r, l := decUint32(buf)
    24  		if r != utf8.RuneError {
    25  			sl += utf8.EncodeRune(p[sl:], rune(r))
    26  		}
    27  		buf = buf[l:]
    28  	}
    29  	return p[:sl]
    30  }
    31  
    32  func encUint32(v uint32, buf []byte) []byte {
    33  	switch {
    34  	case v < 1<<7-1:
    35  		buf = append(buf, byte(v))
    36  	case v < 1<<14-1:
    37  		buf = append(buf, byte(v)|0x80, byte(v>>7))
    38  	case v < 1<<21-1:
    39  		buf = append(buf, byte(v)|0x80, byte(v>>7)|0x80, byte(v>>14))
    40  	case v < 1<<28-1:
    41  		buf = append(buf, byte(v)|0x80, byte(v>>7)|0x80, byte(v>>14)|0x80, byte(v>>21))
    42  	default:
    43  		buf = append(buf, byte(v)|0x80, byte(v>>7)|0x80, byte(v>>14)|0x80, byte(v>>21)|0x80, byte(v>>28))
    44  	}
    45  	return buf
    46  }
    47  
    48  func decUint32(buf []byte) (uint32, int) {
    49  	i := 0
    50  	x := uint32(buf[i])
    51  	if x < 0x80 {
    52  		i++
    53  		return x, i
    54  	}
    55  	x1 := buf[i+1]
    56  	x += uint32(x1) << 7
    57  	if x1 < 0x80 {
    58  		return x - 1<<7, i + 2
    59  	}
    60  	x2 := buf[i+2]
    61  	x += uint32(x2) << 14
    62  	if x2 < 0x80 {
    63  		return x - (1<<7 + 1<<14), i + 3
    64  	}
    65  	x3 := buf[i+3]
    66  	x += uint32(x3) << 21
    67  	if x3 < 0x80 {
    68  		return x - (1<<7 + 1<<14 + 1<<21), i + 4
    69  	}
    70  	x4 := buf[i+4]
    71  	x += uint32(x4) << 28
    72  	return x - (1<<7 + 1<<14 + 1<<21 + 1<<28), i + 5
    73  }