storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/pkg/s3select/csv/recordtransform.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package csv
    18  
    19  import (
    20  	"bytes"
    21  	"io"
    22  )
    23  
    24  // recordTransform will convert records to always have newline records.
    25  type recordTransform struct {
    26  	reader io.Reader
    27  	// recordDelimiter can be up to 2 characters.
    28  	recordDelimiter []byte
    29  	oneByte         []byte
    30  	useOneByte      bool
    31  }
    32  
    33  func (rr *recordTransform) Read(p []byte) (n int, err error) {
    34  	if rr.useOneByte {
    35  		p[0] = rr.oneByte[0]
    36  		rr.useOneByte = false
    37  		n, err = rr.reader.Read(p[1:])
    38  		n++
    39  	} else {
    40  		n, err = rr.reader.Read(p)
    41  	}
    42  
    43  	if err != nil {
    44  		return n, err
    45  	}
    46  
    47  	// Do nothing if record-delimiter is already newline.
    48  	if string(rr.recordDelimiter) == "\n" {
    49  		return n, nil
    50  	}
    51  
    52  	// Change record delimiters to newline.
    53  	if len(rr.recordDelimiter) == 1 {
    54  		for idx := 0; idx < len(p); {
    55  			i := bytes.Index(p[idx:], rr.recordDelimiter)
    56  			if i < 0 {
    57  				break
    58  			}
    59  			idx += i
    60  			p[idx] = '\n'
    61  		}
    62  		return n, nil
    63  	}
    64  
    65  	// 2 characters...
    66  	for idx := 0; idx < len(p); {
    67  		i := bytes.Index(p[idx:], rr.recordDelimiter)
    68  		if i < 0 {
    69  			break
    70  		}
    71  		idx += i
    72  
    73  		p[idx] = '\n'
    74  		p = append(p[:idx+1], p[idx+2:]...)
    75  		n--
    76  	}
    77  
    78  	if p[n-1] != rr.recordDelimiter[0] {
    79  		return n, nil
    80  	}
    81  
    82  	if _, err = rr.reader.Read(rr.oneByte); err != nil {
    83  		return n, err
    84  	}
    85  
    86  	if rr.oneByte[0] == rr.recordDelimiter[1] {
    87  		p[n-1] = '\n'
    88  		return n, nil
    89  	}
    90  
    91  	rr.useOneByte = true
    92  	return n, nil
    93  }