github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/libraries/doltcore/mvdata/pipeline.go (about)

     1  // Copyright 2022 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package mvdata
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  
    21  	"github.com/dolthub/go-mysql-server/sql"
    22  	"golang.org/x/sync/errgroup"
    23  
    24  	"github.com/dolthub/dolt/go/libraries/doltcore/table"
    25  )
    26  
    27  // DataMoverPipeline is an errgroup based pipeline that reads rows from a reader and writes them to a destination with
    28  // a writer.
    29  type DataMoverPipeline struct {
    30  	g   *errgroup.Group
    31  	ctx context.Context
    32  	rd  table.SqlRowReader
    33  	wr  table.SqlRowWriter
    34  }
    35  
    36  func NewDataMoverPipeline(ctx context.Context, rd table.SqlRowReader, wr table.SqlRowWriter) *DataMoverPipeline {
    37  	g, ctx := errgroup.WithContext(ctx)
    38  	return &DataMoverPipeline{
    39  		g:   g,
    40  		ctx: ctx,
    41  		rd:  rd,
    42  		wr:  wr,
    43  	}
    44  }
    45  
    46  func (e *DataMoverPipeline) Execute() error {
    47  	parsedRowChan := make(chan sql.Row)
    48  
    49  	e.g.Go(func() (err error) {
    50  		defer func() {
    51  			close(parsedRowChan)
    52  			if cerr := e.rd.Close(e.ctx); cerr != nil {
    53  				err = cerr
    54  			}
    55  		}()
    56  
    57  		for {
    58  			row, err := e.rd.ReadSqlRow(e.ctx)
    59  			if err == io.EOF {
    60  				return nil
    61  			}
    62  
    63  			if err != nil {
    64  				return err
    65  			}
    66  
    67  			select {
    68  			case <-e.ctx.Done():
    69  				return e.ctx.Err()
    70  			case parsedRowChan <- row:
    71  			}
    72  		}
    73  	})
    74  
    75  	e.g.Go(func() (err error) {
    76  		defer func() {
    77  			if cerr := e.wr.Close(e.ctx); cerr != nil {
    78  				err = cerr
    79  			}
    80  		}()
    81  
    82  		for r := range parsedRowChan {
    83  			select {
    84  			case <-e.ctx.Done():
    85  				return e.ctx.Err()
    86  			default:
    87  				err := e.wr.WriteSqlRow(e.ctx, r)
    88  				if err != nil {
    89  					return err
    90  				}
    91  			}
    92  		}
    93  
    94  		return nil
    95  	})
    96  
    97  	return e.g.Wait()
    98  }