github.com/graybobo/golang.org-package-offline-cache@v0.0.0-20200626051047-6608995c132f/x/text/encoding/charmap/charmap.go (about) 1 // Copyright 2013 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:generate go run maketables.go 6 7 // Package charmap provides simple character encodings such as IBM Code Page 437 8 // and Windows 1252. 9 package charmap // import "golang.org/x/text/encoding/charmap" 10 11 import ( 12 "unicode/utf8" 13 14 "golang.org/x/text/encoding" 15 "golang.org/x/text/encoding/internal" 16 "golang.org/x/text/encoding/internal/identifier" 17 "golang.org/x/text/transform" 18 ) 19 20 // These encodings vary only in the way clients should interpret them. Their 21 // coded character set is identical and a single implementation can be shared. 22 var ( 23 // ISO8859_6E is the ISO 8859-6E encoding. 24 ISO8859_6E encoding.Encoding = &iso8859_6E 25 26 // ISO8859_6I is the ISO 8859-6I encoding. 27 ISO8859_6I encoding.Encoding = &iso8859_6I 28 29 // ISO8859_8E is the ISO 8859-8E encoding. 30 ISO8859_8E encoding.Encoding = &iso8859_8E 31 32 // ISO8859_8I is the ISO 8859-8I encoding. 33 ISO8859_8I encoding.Encoding = &iso8859_8I 34 35 iso8859_6E = internal.Encoding{ 36 ISO8859_6, 37 "ISO-8859-6E", 38 identifier.ISO88596E, 39 } 40 41 iso8859_6I = internal.Encoding{ 42 ISO8859_6, 43 "ISO-8859-6I", 44 identifier.ISO88596I, 45 } 46 47 iso8859_8E = internal.Encoding{ 48 ISO8859_8, 49 "ISO-8859-8E", 50 identifier.ISO88598E, 51 } 52 53 iso8859_8I = internal.Encoding{ 54 ISO8859_8, 55 "ISO-8859-8I", 56 identifier.ISO88598I, 57 } 58 ) 59 60 // All is a list of all defined encodings in this package. 61 var All = listAll 62 63 // TODO: implement these encodings, in order of importance. 64 // ASCII, ISO8859_1: Rather common. Close to Windows 1252. 65 // ISO8859_9: Close to Windows 1254. 66 67 // utf8Enc holds a rune's UTF-8 encoding in data[:len]. 68 type utf8Enc struct { 69 len uint8 70 data [3]byte 71 } 72 73 // charmap describes an 8-bit character set encoding. 74 type charmap struct { 75 // name is the encoding's name. 76 name string 77 // mib is the encoding type of this encoder. 78 mib identifier.MIB 79 // asciiSuperset states whether the encoding is a superset of ASCII. 80 asciiSuperset bool 81 // low is the lower bound of the encoded byte for a non-ASCII rune. If 82 // charmap.asciiSuperset is true then this will be 0x80, otherwise 0x00. 83 low uint8 84 // replacement is the encoded replacement character. 85 replacement byte 86 // decode is the map from encoded byte to UTF-8. 87 decode [256]utf8Enc 88 // encoding is the map from runes to encoded bytes. Each entry is a 89 // uint32: the high 8 bits are the encoded byte and the low 24 bits are 90 // the rune. The table entries are sorted by ascending rune. 91 encode [256]uint32 92 } 93 94 func (m *charmap) NewDecoder() *encoding.Decoder { 95 return &encoding.Decoder{Transformer: charmapDecoder{charmap: m}} 96 } 97 98 func (m *charmap) NewEncoder() *encoding.Encoder { 99 return &encoding.Encoder{Transformer: charmapEncoder{charmap: m}} 100 } 101 102 func (m *charmap) String() string { 103 return m.name 104 } 105 106 func (m *charmap) ID() (mib identifier.MIB, other string) { 107 return m.mib, "" 108 } 109 110 // charmapDecoder implements transform.Transformer by decoding to UTF-8. 111 type charmapDecoder struct { 112 transform.NopResetter 113 charmap *charmap 114 } 115 116 func (m charmapDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 117 for i, c := range src { 118 if m.charmap.asciiSuperset && c < utf8.RuneSelf { 119 if nDst >= len(dst) { 120 err = transform.ErrShortDst 121 break 122 } 123 dst[nDst] = c 124 nDst++ 125 nSrc = i + 1 126 continue 127 } 128 129 decode := &m.charmap.decode[c] 130 n := int(decode.len) 131 if nDst+n > len(dst) { 132 err = transform.ErrShortDst 133 break 134 } 135 // It's 15% faster to avoid calling copy for these tiny slices. 136 for j := 0; j < n; j++ { 137 dst[nDst] = decode.data[j] 138 nDst++ 139 } 140 nSrc = i + 1 141 } 142 return nDst, nSrc, err 143 } 144 145 // charmapEncoder implements transform.Transformer by encoding from UTF-8. 146 type charmapEncoder struct { 147 transform.NopResetter 148 charmap *charmap 149 } 150 151 func (m charmapEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 152 r, size := rune(0), 0 153 loop: 154 for nSrc < len(src) { 155 if nDst >= len(dst) { 156 err = transform.ErrShortDst 157 break 158 } 159 r = rune(src[nSrc]) 160 161 // Decode a 1-byte rune. 162 if r < utf8.RuneSelf { 163 if m.charmap.asciiSuperset { 164 nSrc++ 165 dst[nDst] = uint8(r) 166 nDst++ 167 continue 168 } 169 size = 1 170 171 } else { 172 // Decode a multi-byte rune. 173 r, size = utf8.DecodeRune(src[nSrc:]) 174 if size == 1 { 175 // All valid runes of size 1 (those below utf8.RuneSelf) were 176 // handled above. We have invalid UTF-8 or we haven't seen the 177 // full character yet. 178 if !atEOF && !utf8.FullRune(src[nSrc:]) { 179 err = transform.ErrShortSrc 180 } else { 181 err = internal.RepertoireError(m.charmap.replacement) 182 } 183 break 184 } 185 } 186 187 // Binary search in [low, high) for that rune in the m.charmap.encode table. 188 for low, high := int(m.charmap.low), 0x100; ; { 189 if low >= high { 190 err = internal.RepertoireError(m.charmap.replacement) 191 break loop 192 } 193 mid := (low + high) / 2 194 got := m.charmap.encode[mid] 195 gotRune := rune(got & (1<<24 - 1)) 196 if gotRune < r { 197 low = mid + 1 198 } else if gotRune > r { 199 high = mid 200 } else { 201 dst[nDst] = byte(got >> 24) 202 nDst++ 203 break 204 } 205 } 206 nSrc += size 207 } 208 return nDst, nSrc, err 209 }