github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/utils/gen_kafka_big_messages/main.go (about)

     1  // Copyright 2021 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package main
    15  
    16  import (
    17  	"flag"
    18  	"fmt"
    19  	"log"
    20  	"os"
    21  	"strings"
    22  
    23  	"github.com/pingcap/errors"
    24  )
    25  
    26  // See: https://docs.pingcap.com/tidb/stable/tidb-limitations/#limitations-on-string-types
    27  const varcharColumnMaxLen = 16383
    28  
    29  // Value of col. Defined as a variable for testing.
    30  var colValue = strings.Repeat("a", varcharColumnMaxLen)
    31  
    32  type options struct {
    33  	// The size of each row.
    34  	// The default is 1MiB.
    35  	// FIXME: Currently it does not have precise control over the size of each row.
    36  	// The overhead needs to be calculated and processed accurately.
    37  	rowBytes int
    38  	// Total number of rows.
    39  	// The default is 1 line.
    40  	rowCount int
    41  	// Sql file path.
    42  	// The default is `./test.sql`.
    43  	sqlFilePath string
    44  	// Database name.
    45  	// The default is `kafka_big_messages`.
    46  	databaseName string
    47  	// Table name.
    48  	// The default is `test`.
    49  	tableName string
    50  }
    51  
    52  func (o *options) validate() error {
    53  	if o.rowBytes <= 0 {
    54  		return errors.New("rowBytes must be greater than zero")
    55  	}
    56  
    57  	if o.rowCount <= 0 {
    58  		return errors.New("rowCount must be greater than zero")
    59  	}
    60  
    61  	if o.sqlFilePath == "" {
    62  		return errors.New("please specify the correct file path")
    63  	}
    64  
    65  	if o.databaseName == "" {
    66  		return errors.New("please specify the database name")
    67  	}
    68  
    69  	if o.tableName == "" {
    70  		return errors.New("please specify the table name")
    71  	}
    72  
    73  	return nil
    74  }
    75  
    76  func gatherOptions() *options {
    77  	o := &options{}
    78  
    79  	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
    80  	fs.IntVar(&o.rowBytes, "row-bytes", 1024*1024, "Number of bytes per row.")
    81  	fs.IntVar(&o.rowCount, "row-count", 1, "Count of rows.")
    82  	fs.StringVar(&o.sqlFilePath, "sql-file-path", "./test.sql", "Sql file path.")
    83  	fs.StringVar(&o.databaseName, "database-name", "kafka_big_messages", "Database name.")
    84  	fs.StringVar(&o.tableName, "table-name", "test", "Table name.")
    85  
    86  	_ = fs.Parse(os.Args[1:])
    87  	return o
    88  }
    89  
    90  func main() {
    91  	o := gatherOptions()
    92  	if err := o.validate(); err != nil {
    93  		log.Panicf("Invalid options: %v", err)
    94  	}
    95  
    96  	file, err := os.OpenFile(o.sqlFilePath, os.O_CREATE|os.O_RDWR, os.ModePerm)
    97  	if err != nil {
    98  		log.Panicf("Open sql file failed: %v", err)
    99  	}
   100  
   101  	_, err = file.Write([]byte(genDatabaseSql(o.databaseName)))
   102  	if err != nil {
   103  		log.Panicf("Wirte create database sql failed: %v", err)
   104  	}
   105  
   106  	_, err = file.Write([]byte(genCreateTableSql(o.rowBytes, o.tableName)))
   107  	if err != nil {
   108  		log.Panicf("Wirte create table sql failed: %v", err)
   109  	}
   110  
   111  	for i := 0; i < o.rowCount; i++ {
   112  		_, err = file.Write([]byte(genInsertSql(o.rowBytes, o.tableName, i)))
   113  		if err != nil {
   114  			log.Panicf("Wirte insert sql failed: %v", err)
   115  		}
   116  	}
   117  }
   118  
   119  func genDatabaseSql(databaseName string) string {
   120  	return fmt.Sprintf(`DROP DATABASE IF EXISTS %s;
   121  CREATE DATABASE %s;
   122  USE %s;
   123  
   124  `, databaseName, databaseName, databaseName)
   125  }
   126  
   127  func genCreateTableSql(rawBytes int, tableName string) string {
   128  	var cols string
   129  
   130  	for i := 0; i < rawBytes/varcharColumnMaxLen; i++ {
   131  		cols = fmt.Sprintf("%s, a%d VARCHAR(%d)", cols, i, varcharColumnMaxLen)
   132  	}
   133  
   134  	return fmt.Sprintf("CREATE TABLE %s(id int primary key %s);\n", tableName, cols)
   135  }
   136  
   137  func genInsertSql(rawBytes int, tableName string, id int) string {
   138  	var values string
   139  
   140  	for i := 0; i < rawBytes/varcharColumnMaxLen; i++ {
   141  		values = fmt.Sprintf("%s, '%s'", values, colValue)
   142  	}
   143  
   144  	return fmt.Sprintf("INSERT INTO %s VALUES (%d%s);\n", tableName, id, values)
   145  }