github.com/matrixorigin/matrixone@v0.7.0/pkg/sql/plan/function/builtin/unary/load_file.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package unary
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  
    21  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    22  	"github.com/matrixorigin/matrixone/pkg/container/types"
    23  	"github.com/matrixorigin/matrixone/pkg/container/vector"
    24  	"github.com/matrixorigin/matrixone/pkg/fileservice"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/process"
    26  )
    27  
    28  const (
    29  	blobsize = 65536 // 2^16-1
    30  )
    31  
    32  func LoadFile(vectors []*vector.Vector, proc *process.Process) (*vector.Vector, error) {
    33  	inputVector := vectors[0]
    34  	resultType := types.New(types.T_text, 0, 0, 0)
    35  	resultVector := vector.New(resultType)
    36  	if inputVector.ConstVectorIsNull() {
    37  		return vector.NewConstNull(resultType, 1), nil
    38  	}
    39  	Filepath := vector.GetStrColumn(inputVector)[0]
    40  	fs := proc.FileService
    41  	r, err := ReadFromFile(Filepath, fs)
    42  	if err != nil {
    43  		return nil, err
    44  	}
    45  	ctx, err := io.ReadAll(r)
    46  	defer r.Close()
    47  	if len(ctx) > blobsize {
    48  		return nil, moerr.NewInternalError(proc.Ctx, "Data too long for blob")
    49  	}
    50  	var isNull bool
    51  	if len(ctx) == 0 {
    52  		isNull = true
    53  	}
    54  	if err := resultVector.Append(ctx, isNull, proc.Mp()); err != nil {
    55  		return nil, err
    56  	}
    57  	return resultVector, nil
    58  }
    59  
    60  func ReadFromFile(Filepath string, fs fileservice.FileService) (io.ReadCloser, error) {
    61  	fs, readPath, err := fileservice.GetForETL(fs, Filepath)
    62  	if fs == nil || err != nil {
    63  		return nil, err
    64  	}
    65  	var r io.ReadCloser
    66  	ctx := context.TODO()
    67  	vec := fileservice.IOVector{
    68  		FilePath: readPath,
    69  		Entries: []fileservice.IOEntry{
    70  			0: {
    71  				Offset:            0,
    72  				Size:              -1,
    73  				ReadCloserForRead: &r,
    74  			},
    75  		},
    76  	}
    77  	err = fs.Read(ctx, &vec)
    78  	if err != nil {
    79  		return nil, err
    80  	}
    81  	return r, nil
    82  }