github.com/erda-project/erda-infra@v1.0.9/providers/clickhouse/writer.go (about)

     1  // Copyright (c) 2021 Terminus, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package clickhouse
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  
    21  	ckdriver "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
    22  )
    23  
    24  // EncodeFunc .
    25  type EncodeFunc func(data interface{}) (item *WriteItem, err error)
    26  
    27  // WriteItem .
    28  type WriteItem struct {
    29  	Table string
    30  	Data  interface{}
    31  }
    32  
    33  // WriterOptions .
    34  type WriterOptions struct {
    35  	Encoder EncodeFunc
    36  }
    37  
    38  // Writer .
    39  type Writer struct {
    40  	client  ckdriver.Conn
    41  	Encoder EncodeFunc
    42  }
    43  
    44  // NewWriter .
    45  func NewWriter(client ckdriver.Conn, encoder EncodeFunc) *Writer {
    46  	w := &Writer{
    47  		client:  client,
    48  		Encoder: encoder,
    49  	}
    50  	return w
    51  }
    52  
    53  // Close .
    54  func (w *Writer) Close() error {
    55  	return nil
    56  }
    57  
    58  // WriteN .
    59  func (w *Writer) WriteN(list ...interface{}) (int, error) {
    60  	if len(list) <= 0 {
    61  		return 0, nil
    62  	}
    63  
    64  	items := map[string][]*WriteItem{}
    65  	for _, data := range list {
    66  		item, err := w.Encoder(data)
    67  		if err != nil {
    68  			return 0, err
    69  		}
    70  
    71  		items[item.Table] = append(items[item.Table], item)
    72  	}
    73  
    74  	succ := 0
    75  	for table, tItems := range items {
    76  		batch, err := w.client.PrepareBatch(context.Background(), fmt.Sprintf("insert into %s", table))
    77  		if err != nil {
    78  			return succ, err
    79  		}
    80  		for _, item := range tItems {
    81  			err = batch.AppendStruct(item.Data)
    82  			if err != nil {
    83  				_ = batch.Abort()
    84  				return succ, err
    85  			}
    86  		}
    87  		err = batch.Send()
    88  		if err != nil {
    89  			return succ, err
    90  		}
    91  		succ++
    92  	}
    93  
    94  	return succ, nil
    95  }