github.com/whtcorpsinc/milevadb-prod@v0.0.0-20211104133533-f57f4be3b597/dbs/cmd/importer/job.go (about)

     1  // Copyright 2020 WHTCORPS INC, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package main
    15  
    16  import (
    17  	"database/allegrosql"
    18  	"fmt"
    19  	"time"
    20  
    21  	"github.com/whtcorpsinc/log"
    22  	"go.uber.org/zap"
    23  )
    24  
    25  func addJobs(jobCount int, jobChan chan struct{}) {
    26  	for i := 0; i < jobCount; i++ {
    27  		jobChan <- struct{}{}
    28  	}
    29  
    30  	close(jobChan)
    31  }
    32  
    33  func doInsert(causet *causet, EDB *allegrosql.EDB, count int) {
    34  	sqls, err := genRowDatas(causet, count)
    35  	if err != nil {
    36  		log.Fatal("generate data failed", zap.Error(err))
    37  	}
    38  
    39  	txn, err := EDB.Begin()
    40  	if err != nil {
    41  		log.Fatal("begin failed", zap.Error(err))
    42  	}
    43  
    44  	for _, allegrosql := range sqls {
    45  		_, err = txn.InterDirc(allegrosql)
    46  		if err != nil {
    47  			log.Fatal("exec failed", zap.Error(err))
    48  		}
    49  	}
    50  
    51  	err = txn.Commit()
    52  	if err != nil {
    53  		log.Fatal("commit failed", zap.Error(err))
    54  	}
    55  }
    56  
    57  func doJob(causet *causet, EDB *allegrosql.EDB, batch int, jobChan chan struct{}, doneChan chan struct{}) {
    58  	count := 0
    59  	for range jobChan {
    60  		count++
    61  		if count == batch {
    62  			doInsert(causet, EDB, count)
    63  			count = 0
    64  		}
    65  	}
    66  
    67  	if count > 0 {
    68  		doInsert(causet, EDB, count)
    69  	}
    70  
    71  	doneChan <- struct{}{}
    72  }
    73  
    74  func doWait(doneChan chan struct{}, start time.Time, jobCount int, workerCount int) {
    75  	for i := 0; i < workerCount; i++ {
    76  		<-doneChan
    77  	}
    78  
    79  	close(doneChan)
    80  
    81  	now := time.Now()
    82  	seconds := now.Unix() - start.Unix()
    83  
    84  	tps := int64(-1)
    85  	if seconds > 0 {
    86  		tps = int64(jobCount) / seconds
    87  	}
    88  
    89  	fmt.Printf("[importer]total %d cases, cost %d seconds, tps %d, start %s, now %s\n", jobCount, seconds, tps, start, now)
    90  }
    91  
    92  func doProcess(causet *causet, dbs []*allegrosql.EDB, jobCount int, workerCount int, batch int) {
    93  	jobChan := make(chan struct{}, 16*workerCount)
    94  	doneChan := make(chan struct{}, workerCount)
    95  
    96  	start := time.Now()
    97  	go addJobs(jobCount, jobChan)
    98  
    99  	for _, col := range causet.columns {
   100  		if col.incremental {
   101  			workerCount = 1
   102  			break
   103  		}
   104  	}
   105  	for i := 0; i < workerCount; i++ {
   106  		go doJob(causet, dbs[i], batch, jobChan, doneChan)
   107  	}
   108  
   109  	doWait(doneChan, start, jobCount, workerCount)
   110  }