github.com/easysoft/zendata@v0.0.0-20240513203326-705bd5a7fd67/cmd/test/others/func/import/animal.go (about)

     1  package main
     2  
     3  import (
     4  	"fmt"
     5  	"strings"
     6  
     7  	"github.com/easysoft/zendata/cmd/test/others/func/comm"
     8  	"github.com/easysoft/zendata/cmd/test/others/func/model"
     9  	fileUtils "github.com/easysoft/zendata/pkg/utils/file"
    10  
    11  	"github.com/easysoft/zendata/cmd/test/others/func/comm"
    12  	"github.com/easysoft/zendata/cmd/test/others/func/model"
    13  	fileUtils "github.com/easysoft/zendata/pkg/utils/file"
    14  )
    15  
    16  type DataAnimal struct {
    17  	model.DataComm
    18  }
    19  
    20  func (DataAnimal) TableName() string {
    21  	return "biz_data_animal"
    22  }
    23  
    24  func main() {
    25  	tableName := "animal"
    26  	filePath := "/Users/aaron/work/zentao/product/zd/行业数据/动物.txt"
    27  
    28  	var insertTemplate = "INSERT INTO %s (name) VALUES %s;"
    29  	var createTableTempl = `CREATE TABLE IF NOT EXISTS %s (
    30  		id bigint auto_increment,
    31  		name varchar(1000) not null,
    32  
    33  		primary key(id)
    34  	) engine=innodb default charset=utf8 auto_increment=1;`
    35  
    36  	tableName = "biz_data_" + tableName
    37  	db := comm.GetDB()
    38  	db.Exec(fmt.Sprintf(comm.TruncateTable, tableName))
    39  
    40  	createTableSql := fmt.Sprintf(createTableTempl, tableName)
    41  	err := db.Exec(createTableSql).Error
    42  	if err != nil {
    43  		fmt.Printf("create table %s failed, err %s", tableName, err.Error())
    44  		return
    45  	}
    46  
    47  	content := fileUtils.ReadFile(filePath)
    48  	records := make([]DataAnimal, 0)
    49  
    50  	for _, line := range strings.Split(content, "\n") {
    51  		col1 := strings.Split(strings.TrimSpace(line), " ")[0]
    52  
    53  		po := DataAnimal{
    54  			DataComm: model.DataComm{
    55  				Name: col1,
    56  			},
    57  		}
    58  
    59  		records = append(records, po)
    60  	}
    61  
    62  	insertSqlArr := make([]string, 0)
    63  
    64  	for _, record := range records {
    65  		insert := fmt.Sprintf("('%s')", record.Name)
    66  		insertSqlArr = append(insertSqlArr, insert)
    67  	}
    68  
    69  	for i := 0; i < 1000; i++ {
    70  		start := i * 10000
    71  		end := (i + 1) * 10000
    72  
    73  		if end > len(insertSqlArr) {
    74  			end = len(insertSqlArr)
    75  		}
    76  
    77  		arr := insertSqlArr[start:end]
    78  
    79  		sql := fmt.Sprintf(insertTemplate, tableName, strings.Join(arr, ","))
    80  		err = db.Exec(sql).Error
    81  		if err != nil {
    82  			fmt.Printf("insert data failed, err %s", err.Error())
    83  			return
    84  		}
    85  
    86  		if end >= len(insertSqlArr) {
    87  			break
    88  		}
    89  	}
    90  }