github.com/goproxy0/go@v0.0.0-20171111080102-49cc0c489d2c/src/encoding/csv/reader.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package csv reads and writes comma-separated values (CSV) files. 6 // There are many kinds of CSV files; this package supports the format 7 // described in RFC 4180. 8 // 9 // A csv file contains zero or more records of one or more fields per record. 10 // Each record is separated by the newline character. The final record may 11 // optionally be followed by a newline character. 12 // 13 // field1,field2,field3 14 // 15 // White space is considered part of a field. 16 // 17 // Carriage returns before newline characters are silently removed. 18 // 19 // Blank lines are ignored. A line with only whitespace characters (excluding 20 // the ending newline character) is not considered a blank line. 21 // 22 // Fields which start and stop with the quote character " are called 23 // quoted-fields. The beginning and ending quote are not part of the 24 // field. 25 // 26 // The source: 27 // 28 // normal string,"quoted-field" 29 // 30 // results in the fields 31 // 32 // {`normal string`, `quoted-field`} 33 // 34 // Within a quoted-field a quote character followed by a second quote 35 // character is considered a single quote. 36 // 37 // "the ""word"" is true","a ""quoted-field""" 38 // 39 // results in 40 // 41 // {`the "word" is true`, `a "quoted-field"`} 42 // 43 // Newlines and commas may be included in a quoted-field 44 // 45 // "Multi-line 46 // field","comma is ," 47 // 48 // results in 49 // 50 // {`Multi-line 51 // field`, `comma is ,`} 52 package csv 53 54 import ( 55 "bufio" 56 "bytes" 57 "errors" 58 "fmt" 59 "io" 60 "unicode" 61 "unicode/utf8" 62 ) 63 64 // A ParseError is returned for parsing errors. 65 // Line numbers are 1-indexed and columns are 0-indexed. 66 type ParseError struct { 67 RecordLine int // Line where the record starts 68 Line int // Line where the error occurred 69 Column int // Column (rune index) where the error occurred 70 Err error // The actual error 71 } 72 73 func (e *ParseError) Error() string { 74 if e.Err == ErrFieldCount { 75 return fmt.Sprintf("record on line %d: %v", e.Line, e.Err) 76 } 77 if e.RecordLine != e.Line { 78 return fmt.Sprintf("record on line %d; parse error on line %d, column %d: %v", e.RecordLine, e.Line, e.Column, e.Err) 79 } 80 return fmt.Sprintf("parse error on line %d, column %d: %v", e.Line, e.Column, e.Err) 81 } 82 83 // These are the errors that can be returned in ParseError.Err. 84 var ( 85 ErrTrailingComma = errors.New("extra delimiter at end of line") // Deprecated: No longer used. 86 ErrBareQuote = errors.New("bare \" in non-quoted-field") 87 ErrQuote = errors.New("extraneous or missing \" in quoted-field") 88 ErrFieldCount = errors.New("wrong number of fields") 89 ) 90 91 var errInvalidDelim = errors.New("csv: invalid field or comment delimiter") 92 93 func validDelim(r rune) bool { 94 return r != 0 && r != '\r' && r != '\n' && utf8.ValidRune(r) && r != utf8.RuneError 95 } 96 97 // A Reader reads records from a CSV-encoded file. 98 // 99 // As returned by NewReader, a Reader expects input conforming to RFC 4180. 100 // The exported fields can be changed to customize the details before the 101 // first call to Read or ReadAll. 102 type Reader struct { 103 // Comma is the field delimiter. 104 // It is set to comma (',') by NewReader. 105 Comma rune 106 107 // Comment, if not 0, is the comment character. Lines beginning with the 108 // Comment character without preceding whitespace are ignored. 109 // With leading whitespace the Comment character becomes part of the 110 // field, even if TrimLeadingSpace is true. 111 Comment rune 112 113 // FieldsPerRecord is the number of expected fields per record. 114 // If FieldsPerRecord is positive, Read requires each record to 115 // have the given number of fields. If FieldsPerRecord is 0, Read sets it to 116 // the number of fields in the first record, so that future records must 117 // have the same field count. If FieldsPerRecord is negative, no check is 118 // made and records may have a variable number of fields. 119 FieldsPerRecord int 120 121 // If LazyQuotes is true, a quote may appear in an unquoted field and a 122 // non-doubled quote may appear in a quoted field. 123 LazyQuotes bool 124 125 // If TrimLeadingSpace is true, leading white space in a field is ignored. 126 // This is done even if the field delimiter, Comma, is white space. 127 TrimLeadingSpace bool 128 129 // ReuseRecord controls whether calls to Read may return a slice sharing 130 // the backing array of the previous call's returned slice for performance. 131 // By default, each call to Read returns newly allocated memory owned by the caller. 132 ReuseRecord bool 133 134 TrailingComma bool // Deprecated: No longer used. 135 136 r *bufio.Reader 137 138 // numLine is the current line being read in the CSV file. 139 numLine int 140 141 // rawBuffer is a line buffer only used by the readLine method. 142 rawBuffer []byte 143 144 // recordBuffer holds the unescaped fields, one after another. 145 // The fields can be accessed by using the indexes in fieldIndexes. 146 // E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de` 147 // and fieldIndexes will contain the indexes [1, 2, 5, 6]. 148 recordBuffer []byte 149 150 // fieldIndexes is an index of fields inside recordBuffer. 151 // The i'th field ends at offset fieldIndexes[i] in recordBuffer. 152 fieldIndexes []int 153 154 // lastRecord is a record cache and only used when ReuseRecord == true. 155 lastRecord []string 156 } 157 158 // NewReader returns a new Reader that reads from r. 159 func NewReader(r io.Reader) *Reader { 160 return &Reader{ 161 Comma: ',', 162 r: bufio.NewReader(r), 163 } 164 } 165 166 // Read reads one record (a slice of fields) from r. 167 // If the record has an unexpected number of fields, 168 // Read returns the record along with the error ErrFieldCount. 169 // Except for that case, Read always returns either a non-nil 170 // record or a non-nil error, but not both. 171 // If there is no data left to be read, Read returns nil, io.EOF. 172 // If ReuseRecord is true, the returned slice may be shared 173 // between multiple calls to Read. 174 func (r *Reader) Read() (record []string, err error) { 175 if r.ReuseRecord { 176 record, err = r.readRecord(r.lastRecord) 177 r.lastRecord = record 178 } else { 179 record, err = r.readRecord(nil) 180 } 181 return record, err 182 } 183 184 // ReadAll reads all the remaining records from r. 185 // Each record is a slice of fields. 186 // A successful call returns err == nil, not err == io.EOF. Because ReadAll is 187 // defined to read until EOF, it does not treat end of file as an error to be 188 // reported. 189 func (r *Reader) ReadAll() (records [][]string, err error) { 190 for { 191 record, err := r.readRecord(nil) 192 if err == io.EOF { 193 return records, nil 194 } 195 if err != nil { 196 return nil, err 197 } 198 records = append(records, record) 199 } 200 } 201 202 // readLine reads the next line (with the trailing endline). 203 // If EOF is hit without a trailing endline, it will be omitted. 204 // If some bytes were read, then the error is never io.EOF. 205 // The result is only valid until the next call to readLine. 206 func (r *Reader) readLine() ([]byte, error) { 207 line, err := r.r.ReadSlice('\n') 208 if err == bufio.ErrBufferFull { 209 r.rawBuffer = append(r.rawBuffer[:0], line...) 210 for err == bufio.ErrBufferFull { 211 line, err = r.r.ReadSlice('\n') 212 r.rawBuffer = append(r.rawBuffer, line...) 213 } 214 line = r.rawBuffer 215 } 216 if len(line) > 0 && err == io.EOF { 217 err = nil 218 } 219 r.numLine++ 220 return line, err 221 } 222 223 // lengthCRLF reports the number of bytes for a trailing "\r\n". 224 func lengthCRLF(b []byte) int { 225 if j := len(b) - 1; j >= 0 && b[j] == '\n' { 226 if j := len(b) - 2; j >= 0 && b[j] == '\r' { 227 return 2 228 } 229 return 1 230 } 231 return 0 232 } 233 234 // nextRune returns the next rune in b or utf8.RuneError. 235 func nextRune(b []byte) rune { 236 r, _ := utf8.DecodeRune(b) 237 return r 238 } 239 240 func (r *Reader) readRecord(dst []string) ([]string, error) { 241 if r.Comma == r.Comment || !validDelim(r.Comma) || (r.Comment != 0 && !validDelim(r.Comment)) { 242 return nil, errInvalidDelim 243 } 244 245 // Read line (automatically skipping past empty lines and any comments). 246 var line, fullLine []byte 247 var errRead error 248 for errRead == nil { 249 line, errRead = r.readLine() 250 if r.Comment != 0 && nextRune(line) == r.Comment { 251 line = nil 252 continue // Skip comment lines 253 } 254 if errRead == nil && len(line) == lengthCRLF(line) { 255 line = nil 256 continue // Skip empty lines 257 } 258 fullLine = line 259 break 260 } 261 if errRead == io.EOF { 262 return nil, errRead 263 } 264 265 // Parse each field in the record. 266 var err error 267 const quoteLen = len(`"`) 268 commaLen := utf8.RuneLen(r.Comma) 269 recLine := r.numLine // Starting line for record 270 r.recordBuffer = r.recordBuffer[:0] 271 r.fieldIndexes = r.fieldIndexes[:0] 272 parseField: 273 for { 274 if r.TrimLeadingSpace { 275 line = bytes.TrimLeftFunc(line, unicode.IsSpace) 276 } 277 if len(line) == 0 || line[0] != '"' { 278 // Non-quoted string field 279 i := bytes.IndexRune(line, r.Comma) 280 field := line 281 if i >= 0 { 282 field = field[:i] 283 } else { 284 field = field[:len(field)-lengthCRLF(field)] 285 } 286 // Check to make sure a quote does not appear in field. 287 if !r.LazyQuotes { 288 if j := bytes.IndexByte(field, '"'); j >= 0 { 289 col := utf8.RuneCount(fullLine[:len(fullLine)-len(line[j:])]) 290 err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote} 291 break parseField 292 } 293 } 294 r.recordBuffer = append(r.recordBuffer, field...) 295 r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) 296 if i >= 0 { 297 line = line[i+commaLen:] 298 continue parseField 299 } 300 break parseField 301 } else { 302 // Quoted string field 303 line = line[quoteLen:] 304 for { 305 i := bytes.IndexByte(line, '"') 306 if i >= 0 { 307 // Hit next quote. 308 r.recordBuffer = append(r.recordBuffer, line[:i]...) 309 line = line[i+quoteLen:] 310 switch rn := nextRune(line); { 311 case rn == '"': 312 // `""` sequence (append quote). 313 r.recordBuffer = append(r.recordBuffer, '"') 314 line = line[quoteLen:] 315 case rn == r.Comma: 316 // `",` sequence (end of field). 317 line = line[commaLen:] 318 r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) 319 continue parseField 320 case lengthCRLF(line) == len(line): 321 // `"\n` sequence (end of line). 322 r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) 323 break parseField 324 case r.LazyQuotes: 325 // `"` sequence (bare quote). 326 r.recordBuffer = append(r.recordBuffer, '"') 327 default: 328 // `"*` sequence (invalid non-escaped quote). 329 col := utf8.RuneCount(fullLine[:len(fullLine)-len(line)-quoteLen]) 330 err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote} 331 break parseField 332 } 333 } else if len(line) > 0 { 334 // Hit end of line (copy all data so far). 335 r.recordBuffer = append(r.recordBuffer, line...) 336 if errRead != nil { 337 break parseField 338 } 339 line, errRead = r.readLine() 340 if errRead == io.EOF { 341 errRead = nil 342 } 343 fullLine = line 344 } else { 345 // Abrupt end of file (EOF or error). 346 if !r.LazyQuotes && errRead == nil { 347 col := utf8.RuneCount(fullLine) 348 err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote} 349 break parseField 350 } 351 r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) 352 break parseField 353 } 354 } 355 } 356 } 357 if err == nil { 358 err = errRead 359 } 360 361 // Create a single string and create slices out of it. 362 // This pins the memory of the fields together, but allocates once. 363 str := string(r.recordBuffer) // Convert to string once to batch allocations 364 dst = dst[:0] 365 if cap(dst) < len(r.fieldIndexes) { 366 dst = make([]string, len(r.fieldIndexes)) 367 } 368 dst = dst[:len(r.fieldIndexes)] 369 var preIdx int 370 for i, idx := range r.fieldIndexes { 371 dst[i] = str[preIdx:idx] 372 preIdx = idx 373 } 374 375 // Check or update the expected fields per record. 376 if r.FieldsPerRecord > 0 { 377 if len(dst) != r.FieldsPerRecord && err == nil { 378 err = &ParseError{RecordLine: recLine, Line: recLine, Err: ErrFieldCount} 379 } 380 } else if r.FieldsPerRecord == 0 { 381 r.FieldsPerRecord = len(dst) 382 } 383 return dst, err 384 }