github.com/matrixorigin/matrixone@v1.2.0/pkg/util/export/etl/sql.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package etl
    16  
    17  import (
    18  	"context"
    19  	"sync"
    20  	"time"
    21  
    22  	db_holder "github.com/matrixorigin/matrixone/pkg/util/export/etl/db"
    23  	"github.com/matrixorigin/matrixone/pkg/util/export/table"
    24  
    25  	_ "github.com/go-sql-driver/mysql"
    26  )
    27  
    28  const MAX_INSERT_TIME = 3 * time.Second
    29  
    30  // DefaultSqlWriter SqlWriter is a writer that writes data to a SQL database.
    31  type DefaultSqlWriter struct {
    32  	ctx       context.Context
    33  	csvWriter *CSVWriter
    34  	tbl       *table.Table
    35  	buffer    [][]string
    36  	mux       sync.Mutex
    37  }
    38  
    39  func NewSqlWriter(ctx context.Context, tbl *table.Table, csv *CSVWriter) *DefaultSqlWriter {
    40  	return &DefaultSqlWriter{
    41  		ctx:       ctx,
    42  		csvWriter: csv,
    43  		tbl:       tbl,
    44  	}
    45  }
    46  
    47  func (sw *DefaultSqlWriter) GetContent() string {
    48  	return ""
    49  }
    50  
    51  func (sw *DefaultSqlWriter) WriteStrings(record []string) error {
    52  	return nil
    53  }
    54  
    55  func (sw *DefaultSqlWriter) WriteRow(row *table.Row) error {
    56  	sw.buffer = append(sw.buffer, row.ToStrings())
    57  	return nil
    58  }
    59  
    60  func (sw *DefaultSqlWriter) flushBuffer(force bool) (int, error) {
    61  	sw.mux.Lock()
    62  	defer sw.mux.Unlock()
    63  
    64  	var err error
    65  	var cnt int
    66  
    67  	cnt, err = db_holder.WriteRowRecords(sw.buffer, sw.tbl, MAX_INSERT_TIME)
    68  
    69  	if err != nil {
    70  		sw.dumpBufferToCSV()
    71  	}
    72  	_, err = sw.csvWriter.FlushAndClose()
    73  	return cnt, err
    74  }
    75  
    76  func (sw *DefaultSqlWriter) dumpBufferToCSV() error {
    77  	if len(sw.buffer) == 0 {
    78  		return nil
    79  	}
    80  	// write sw.buffer to csvWriter
    81  	for _, row := range sw.buffer {
    82  		sw.csvWriter.WriteStrings(row)
    83  	}
    84  	return nil
    85  }
    86  
    87  func (sw *DefaultSqlWriter) FlushAndClose() (int, error) {
    88  	if sw.buffer != nil && len(sw.buffer) == 0 {
    89  		return 0, nil
    90  	}
    91  	cnt, err := sw.flushBuffer(true)
    92  	sw.buffer = nil
    93  	sw.tbl = nil
    94  	sw.csvWriter = nil
    95  	return cnt, err
    96  }