storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/pkg/s3select/json/reader.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package json
    18  
    19  import (
    20  	"errors"
    21  	"io"
    22  	"sync"
    23  
    24  	"storj.io/minio/pkg/s3select/sql"
    25  
    26  	"github.com/bcicen/jstream"
    27  )
    28  
    29  // Reader - JSON record reader for S3Select.
    30  type Reader struct {
    31  	args       *ReaderArgs
    32  	decoder    *jstream.Decoder
    33  	valueCh    chan *jstream.MetaValue
    34  	readCloser io.ReadCloser
    35  }
    36  
    37  // Read - reads single record.
    38  func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
    39  	v, ok := <-r.valueCh
    40  	if !ok {
    41  		if err := r.decoder.Err(); err != nil {
    42  			return nil, errJSONParsingError(err)
    43  		}
    44  		return nil, io.EOF
    45  	}
    46  
    47  	var kvs jstream.KVS
    48  	if v.ValueType == jstream.Object {
    49  		// This is a JSON object type (that preserves key
    50  		// order)
    51  		kvs = v.Value.(jstream.KVS)
    52  	} else {
    53  		// To be AWS S3 compatible Select for JSON needs to
    54  		// output non-object JSON as single column value
    55  		// i.e. a map with `_1` as key and value as the
    56  		// non-object.
    57  		kvs = jstream.KVS{jstream.KV{Key: "_1", Value: v.Value}}
    58  	}
    59  
    60  	dstRec, ok := dst.(*Record)
    61  	if !ok {
    62  		dstRec = &Record{}
    63  	}
    64  	dstRec.KVS = kvs
    65  	dstRec.SelectFormat = sql.SelectFmtJSON
    66  	return dstRec, nil
    67  }
    68  
    69  // Close - closes underlying reader.
    70  func (r *Reader) Close() error {
    71  	// Close the input.
    72  	err := r.readCloser.Close()
    73  	for range r.valueCh {
    74  		// Drain values so we don't leak a goroutine.
    75  		// Since we have closed the input, it should fail rather quickly.
    76  	}
    77  	return err
    78  }
    79  
    80  // NewReader - creates new JSON reader using readCloser.
    81  func NewReader(readCloser io.ReadCloser, args *ReaderArgs) *Reader {
    82  	readCloser = &syncReadCloser{rc: readCloser}
    83  	d := jstream.NewDecoder(readCloser, 0).ObjectAsKVS()
    84  	return &Reader{
    85  		args:       args,
    86  		decoder:    d,
    87  		valueCh:    d.Stream(),
    88  		readCloser: readCloser,
    89  	}
    90  }
    91  
    92  // syncReadCloser will wrap a readcloser and make it safe to call Close
    93  // while reads are running.
    94  // All read errors are also postponed until Close is called and
    95  // io.EOF is returned instead.
    96  type syncReadCloser struct {
    97  	rc    io.ReadCloser
    98  	errMu sync.Mutex
    99  	err   error
   100  }
   101  
   102  func (pr *syncReadCloser) Read(p []byte) (n int, err error) {
   103  	// This ensures that Close will block until Read has completed.
   104  	// This allows another goroutine to close the reader.
   105  	pr.errMu.Lock()
   106  	defer pr.errMu.Unlock()
   107  	if pr.err != nil {
   108  		return 0, io.EOF
   109  	}
   110  	n, pr.err = pr.rc.Read(p)
   111  	if pr.err != nil {
   112  		// Translate any error into io.EOF, so we don't crash:
   113  		// https://github.com/bcicen/jstream/blob/master/scanner.go#L48
   114  		return n, io.EOF
   115  	}
   116  
   117  	return n, nil
   118  }
   119  
   120  var errClosed = errors.New("read after close")
   121  
   122  func (pr *syncReadCloser) Close() error {
   123  	pr.errMu.Lock()
   124  	defer pr.errMu.Unlock()
   125  	if pr.err == errClosed {
   126  		return nil
   127  	}
   128  	if pr.err != nil {
   129  		return pr.err
   130  	}
   131  	pr.err = errClosed
   132  	return pr.rc.Close()
   133  }