github.com/goproxy0/go@v0.0.0-20171111080102-49cc0c489d2c/src/encoding/csv/reader.go (about)

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package csv reads and writes comma-separated values (CSV) files.
     6  // There are many kinds of CSV files; this package supports the format
     7  // described in RFC 4180.
     8  //
     9  // A csv file contains zero or more records of one or more fields per record.
    10  // Each record is separated by the newline character. The final record may
    11  // optionally be followed by a newline character.
    12  //
    13  //	field1,field2,field3
    14  //
    15  // White space is considered part of a field.
    16  //
    17  // Carriage returns before newline characters are silently removed.
    18  //
    19  // Blank lines are ignored. A line with only whitespace characters (excluding
    20  // the ending newline character) is not considered a blank line.
    21  //
    22  // Fields which start and stop with the quote character " are called
    23  // quoted-fields. The beginning and ending quote are not part of the
    24  // field.
    25  //
    26  // The source:
    27  //
    28  //	normal string,"quoted-field"
    29  //
    30  // results in the fields
    31  //
    32  //	{`normal string`, `quoted-field`}
    33  //
    34  // Within a quoted-field a quote character followed by a second quote
    35  // character is considered a single quote.
    36  //
    37  //	"the ""word"" is true","a ""quoted-field"""
    38  //
    39  // results in
    40  //
    41  //	{`the "word" is true`, `a "quoted-field"`}
    42  //
    43  // Newlines and commas may be included in a quoted-field
    44  //
    45  //	"Multi-line
    46  //	field","comma is ,"
    47  //
    48  // results in
    49  //
    50  //	{`Multi-line
    51  //	field`, `comma is ,`}
    52  package csv
    53  
    54  import (
    55  	"bufio"
    56  	"bytes"
    57  	"errors"
    58  	"fmt"
    59  	"io"
    60  	"unicode"
    61  	"unicode/utf8"
    62  )
    63  
    64  // A ParseError is returned for parsing errors.
    65  // Line numbers are 1-indexed and columns are 0-indexed.
    66  type ParseError struct {
    67  	RecordLine int   // Line where the record starts
    68  	Line       int   // Line where the error occurred
    69  	Column     int   // Column (rune index) where the error occurred
    70  	Err        error // The actual error
    71  }
    72  
    73  func (e *ParseError) Error() string {
    74  	if e.Err == ErrFieldCount {
    75  		return fmt.Sprintf("record on line %d: %v", e.Line, e.Err)
    76  	}
    77  	if e.RecordLine != e.Line {
    78  		return fmt.Sprintf("record on line %d; parse error on line %d, column %d: %v", e.RecordLine, e.Line, e.Column, e.Err)
    79  	}
    80  	return fmt.Sprintf("parse error on line %d, column %d: %v", e.Line, e.Column, e.Err)
    81  }
    82  
    83  // These are the errors that can be returned in ParseError.Err.
    84  var (
    85  	ErrTrailingComma = errors.New("extra delimiter at end of line") // Deprecated: No longer used.
    86  	ErrBareQuote     = errors.New("bare \" in non-quoted-field")
    87  	ErrQuote         = errors.New("extraneous or missing \" in quoted-field")
    88  	ErrFieldCount    = errors.New("wrong number of fields")
    89  )
    90  
    91  var errInvalidDelim = errors.New("csv: invalid field or comment delimiter")
    92  
    93  func validDelim(r rune) bool {
    94  	return r != 0 && r != '\r' && r != '\n' && utf8.ValidRune(r) && r != utf8.RuneError
    95  }
    96  
    97  // A Reader reads records from a CSV-encoded file.
    98  //
    99  // As returned by NewReader, a Reader expects input conforming to RFC 4180.
   100  // The exported fields can be changed to customize the details before the
   101  // first call to Read or ReadAll.
   102  type Reader struct {
   103  	// Comma is the field delimiter.
   104  	// It is set to comma (',') by NewReader.
   105  	Comma rune
   106  
   107  	// Comment, if not 0, is the comment character. Lines beginning with the
   108  	// Comment character without preceding whitespace are ignored.
   109  	// With leading whitespace the Comment character becomes part of the
   110  	// field, even if TrimLeadingSpace is true.
   111  	Comment rune
   112  
   113  	// FieldsPerRecord is the number of expected fields per record.
   114  	// If FieldsPerRecord is positive, Read requires each record to
   115  	// have the given number of fields. If FieldsPerRecord is 0, Read sets it to
   116  	// the number of fields in the first record, so that future records must
   117  	// have the same field count. If FieldsPerRecord is negative, no check is
   118  	// made and records may have a variable number of fields.
   119  	FieldsPerRecord int
   120  
   121  	// If LazyQuotes is true, a quote may appear in an unquoted field and a
   122  	// non-doubled quote may appear in a quoted field.
   123  	LazyQuotes bool
   124  
   125  	// If TrimLeadingSpace is true, leading white space in a field is ignored.
   126  	// This is done even if the field delimiter, Comma, is white space.
   127  	TrimLeadingSpace bool
   128  
   129  	// ReuseRecord controls whether calls to Read may return a slice sharing
   130  	// the backing array of the previous call's returned slice for performance.
   131  	// By default, each call to Read returns newly allocated memory owned by the caller.
   132  	ReuseRecord bool
   133  
   134  	TrailingComma bool // Deprecated: No longer used.
   135  
   136  	r *bufio.Reader
   137  
   138  	// numLine is the current line being read in the CSV file.
   139  	numLine int
   140  
   141  	// rawBuffer is a line buffer only used by the readLine method.
   142  	rawBuffer []byte
   143  
   144  	// recordBuffer holds the unescaped fields, one after another.
   145  	// The fields can be accessed by using the indexes in fieldIndexes.
   146  	// E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de`
   147  	// and fieldIndexes will contain the indexes [1, 2, 5, 6].
   148  	recordBuffer []byte
   149  
   150  	// fieldIndexes is an index of fields inside recordBuffer.
   151  	// The i'th field ends at offset fieldIndexes[i] in recordBuffer.
   152  	fieldIndexes []int
   153  
   154  	// lastRecord is a record cache and only used when ReuseRecord == true.
   155  	lastRecord []string
   156  }
   157  
   158  // NewReader returns a new Reader that reads from r.
   159  func NewReader(r io.Reader) *Reader {
   160  	return &Reader{
   161  		Comma: ',',
   162  		r:     bufio.NewReader(r),
   163  	}
   164  }
   165  
   166  // Read reads one record (a slice of fields) from r.
   167  // If the record has an unexpected number of fields,
   168  // Read returns the record along with the error ErrFieldCount.
   169  // Except for that case, Read always returns either a non-nil
   170  // record or a non-nil error, but not both.
   171  // If there is no data left to be read, Read returns nil, io.EOF.
   172  // If ReuseRecord is true, the returned slice may be shared
   173  // between multiple calls to Read.
   174  func (r *Reader) Read() (record []string, err error) {
   175  	if r.ReuseRecord {
   176  		record, err = r.readRecord(r.lastRecord)
   177  		r.lastRecord = record
   178  	} else {
   179  		record, err = r.readRecord(nil)
   180  	}
   181  	return record, err
   182  }
   183  
   184  // ReadAll reads all the remaining records from r.
   185  // Each record is a slice of fields.
   186  // A successful call returns err == nil, not err == io.EOF. Because ReadAll is
   187  // defined to read until EOF, it does not treat end of file as an error to be
   188  // reported.
   189  func (r *Reader) ReadAll() (records [][]string, err error) {
   190  	for {
   191  		record, err := r.readRecord(nil)
   192  		if err == io.EOF {
   193  			return records, nil
   194  		}
   195  		if err != nil {
   196  			return nil, err
   197  		}
   198  		records = append(records, record)
   199  	}
   200  }
   201  
   202  // readLine reads the next line (with the trailing endline).
   203  // If EOF is hit without a trailing endline, it will be omitted.
   204  // If some bytes were read, then the error is never io.EOF.
   205  // The result is only valid until the next call to readLine.
   206  func (r *Reader) readLine() ([]byte, error) {
   207  	line, err := r.r.ReadSlice('\n')
   208  	if err == bufio.ErrBufferFull {
   209  		r.rawBuffer = append(r.rawBuffer[:0], line...)
   210  		for err == bufio.ErrBufferFull {
   211  			line, err = r.r.ReadSlice('\n')
   212  			r.rawBuffer = append(r.rawBuffer, line...)
   213  		}
   214  		line = r.rawBuffer
   215  	}
   216  	if len(line) > 0 && err == io.EOF {
   217  		err = nil
   218  	}
   219  	r.numLine++
   220  	return line, err
   221  }
   222  
   223  // lengthCRLF reports the number of bytes for a trailing "\r\n".
   224  func lengthCRLF(b []byte) int {
   225  	if j := len(b) - 1; j >= 0 && b[j] == '\n' {
   226  		if j := len(b) - 2; j >= 0 && b[j] == '\r' {
   227  			return 2
   228  		}
   229  		return 1
   230  	}
   231  	return 0
   232  }
   233  
   234  // nextRune returns the next rune in b or utf8.RuneError.
   235  func nextRune(b []byte) rune {
   236  	r, _ := utf8.DecodeRune(b)
   237  	return r
   238  }
   239  
   240  func (r *Reader) readRecord(dst []string) ([]string, error) {
   241  	if r.Comma == r.Comment || !validDelim(r.Comma) || (r.Comment != 0 && !validDelim(r.Comment)) {
   242  		return nil, errInvalidDelim
   243  	}
   244  
   245  	// Read line (automatically skipping past empty lines and any comments).
   246  	var line, fullLine []byte
   247  	var errRead error
   248  	for errRead == nil {
   249  		line, errRead = r.readLine()
   250  		if r.Comment != 0 && nextRune(line) == r.Comment {
   251  			line = nil
   252  			continue // Skip comment lines
   253  		}
   254  		if errRead == nil && len(line) == lengthCRLF(line) {
   255  			line = nil
   256  			continue // Skip empty lines
   257  		}
   258  		fullLine = line
   259  		break
   260  	}
   261  	if errRead == io.EOF {
   262  		return nil, errRead
   263  	}
   264  
   265  	// Parse each field in the record.
   266  	var err error
   267  	const quoteLen = len(`"`)
   268  	commaLen := utf8.RuneLen(r.Comma)
   269  	recLine := r.numLine // Starting line for record
   270  	r.recordBuffer = r.recordBuffer[:0]
   271  	r.fieldIndexes = r.fieldIndexes[:0]
   272  parseField:
   273  	for {
   274  		if r.TrimLeadingSpace {
   275  			line = bytes.TrimLeftFunc(line, unicode.IsSpace)
   276  		}
   277  		if len(line) == 0 || line[0] != '"' {
   278  			// Non-quoted string field
   279  			i := bytes.IndexRune(line, r.Comma)
   280  			field := line
   281  			if i >= 0 {
   282  				field = field[:i]
   283  			} else {
   284  				field = field[:len(field)-lengthCRLF(field)]
   285  			}
   286  			// Check to make sure a quote does not appear in field.
   287  			if !r.LazyQuotes {
   288  				if j := bytes.IndexByte(field, '"'); j >= 0 {
   289  					col := utf8.RuneCount(fullLine[:len(fullLine)-len(line[j:])])
   290  					err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote}
   291  					break parseField
   292  				}
   293  			}
   294  			r.recordBuffer = append(r.recordBuffer, field...)
   295  			r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
   296  			if i >= 0 {
   297  				line = line[i+commaLen:]
   298  				continue parseField
   299  			}
   300  			break parseField
   301  		} else {
   302  			// Quoted string field
   303  			line = line[quoteLen:]
   304  			for {
   305  				i := bytes.IndexByte(line, '"')
   306  				if i >= 0 {
   307  					// Hit next quote.
   308  					r.recordBuffer = append(r.recordBuffer, line[:i]...)
   309  					line = line[i+quoteLen:]
   310  					switch rn := nextRune(line); {
   311  					case rn == '"':
   312  						// `""` sequence (append quote).
   313  						r.recordBuffer = append(r.recordBuffer, '"')
   314  						line = line[quoteLen:]
   315  					case rn == r.Comma:
   316  						// `",` sequence (end of field).
   317  						line = line[commaLen:]
   318  						r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
   319  						continue parseField
   320  					case lengthCRLF(line) == len(line):
   321  						// `"\n` sequence (end of line).
   322  						r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
   323  						break parseField
   324  					case r.LazyQuotes:
   325  						// `"` sequence (bare quote).
   326  						r.recordBuffer = append(r.recordBuffer, '"')
   327  					default:
   328  						// `"*` sequence (invalid non-escaped quote).
   329  						col := utf8.RuneCount(fullLine[:len(fullLine)-len(line)-quoteLen])
   330  						err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote}
   331  						break parseField
   332  					}
   333  				} else if len(line) > 0 {
   334  					// Hit end of line (copy all data so far).
   335  					r.recordBuffer = append(r.recordBuffer, line...)
   336  					if errRead != nil {
   337  						break parseField
   338  					}
   339  					line, errRead = r.readLine()
   340  					if errRead == io.EOF {
   341  						errRead = nil
   342  					}
   343  					fullLine = line
   344  				} else {
   345  					// Abrupt end of file (EOF or error).
   346  					if !r.LazyQuotes && errRead == nil {
   347  						col := utf8.RuneCount(fullLine)
   348  						err = &ParseError{RecordLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote}
   349  						break parseField
   350  					}
   351  					r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
   352  					break parseField
   353  				}
   354  			}
   355  		}
   356  	}
   357  	if err == nil {
   358  		err = errRead
   359  	}
   360  
   361  	// Create a single string and create slices out of it.
   362  	// This pins the memory of the fields together, but allocates once.
   363  	str := string(r.recordBuffer) // Convert to string once to batch allocations
   364  	dst = dst[:0]
   365  	if cap(dst) < len(r.fieldIndexes) {
   366  		dst = make([]string, len(r.fieldIndexes))
   367  	}
   368  	dst = dst[:len(r.fieldIndexes)]
   369  	var preIdx int
   370  	for i, idx := range r.fieldIndexes {
   371  		dst[i] = str[preIdx:idx]
   372  		preIdx = idx
   373  	}
   374  
   375  	// Check or update the expected fields per record.
   376  	if r.FieldsPerRecord > 0 {
   377  		if len(dst) != r.FieldsPerRecord && err == nil {
   378  			err = &ParseError{RecordLine: recLine, Line: recLine, Err: ErrFieldCount}
   379  		}
   380  	} else if r.FieldsPerRecord == 0 {
   381  		r.FieldsPerRecord = len(dst)
   382  	}
   383  	return dst, err
   384  }