github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/internal/sqlsmith/sqlsmith_test.go (about)

     1  // Copyright 2019 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package sqlsmith
    12  
    13  import (
    14  	"context"
    15  	"flag"
    16  	"fmt"
    17  	"strings"
    18  	"testing"
    19  
    20  	"github.com/cockroachdb/cockroach/pkg/base"
    21  	"github.com/cockroachdb/cockroach/pkg/ccl/utilccl"
    22  	"github.com/cockroachdb/cockroach/pkg/sql/parser"
    23  	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
    24  	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
    25  	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
    26  	"github.com/cockroachdb/cockroach/pkg/util/randutil"
    27  )
    28  
    29  var (
    30  	flagExec     = flag.Bool("ex", false, "execute (instead of just parse) generated statements")
    31  	flagNum      = flag.Int("num", 100, "number of statements to generate")
    32  	flagSetup    = flag.String("setup", "", "setup for TestGenerateParse, empty for random")
    33  	flagSetting  = flag.String("setting", "", "setting for TestGenerateParse, empty for random")
    34  	flagCheckVec = flag.Bool("check-vec", false, "fail if a generated statement cannot be vectorized")
    35  )
    36  
    37  // TestSetups verifies that all setups generate executable SQL.
    38  func TestSetups(t *testing.T) {
    39  	defer leaktest.AfterTest(t)()
    40  
    41  	for name, setup := range Setups {
    42  		t.Run(name, func(t *testing.T) {
    43  			ctx := context.Background()
    44  			s, sqlDB, _ := serverutils.StartServer(t, base.TestServerArgs{})
    45  			defer s.Stopper().Stop(ctx)
    46  
    47  			rnd, _ := randutil.NewPseudoRand()
    48  
    49  			sql := setup(rnd)
    50  			if _, err := sqlDB.Exec(sql); err != nil {
    51  				t.Log(sql)
    52  				t.Fatal(err)
    53  			}
    54  		})
    55  	}
    56  }
    57  
    58  // TestGenerateParse verifies that statements produced by Generate can be
    59  // parsed. This is useful because since we make AST nodes directly we can
    60  // sometimes put them into bad states that the parser would never do.
    61  func TestGenerateParse(t *testing.T) {
    62  	defer leaktest.AfterTest(t)()
    63  	defer utilccl.TestingEnableEnterprise()()
    64  
    65  	ctx := context.Background()
    66  	s, sqlDB, _ := serverutils.StartServer(t, base.TestServerArgs{})
    67  	defer s.Stopper().Stop(ctx)
    68  
    69  	rnd, seed := randutil.NewPseudoRand()
    70  	t.Log("seed:", seed)
    71  
    72  	db := sqlutils.MakeSQLRunner(sqlDB)
    73  
    74  	setupName := *flagSetup
    75  	if setupName == "" {
    76  		setupName = RandSetup(rnd)
    77  	}
    78  	setup, ok := Setups[setupName]
    79  	if !ok {
    80  		t.Fatalf("unknown setup %s", setupName)
    81  	}
    82  	t.Log("setup:", setupName)
    83  	settingName := *flagSetting
    84  	if settingName == "" {
    85  		settingName = RandSetting(rnd)
    86  	}
    87  	setting, ok := Settings[settingName]
    88  	if !ok {
    89  		t.Fatalf("unknown setting %s", settingName)
    90  	}
    91  	settings := setting(rnd)
    92  	t.Log("setting:", settingName, settings.Options)
    93  	setupSQL := setup(rnd)
    94  	t.Log(setupSQL)
    95  	db.Exec(t, setupSQL)
    96  
    97  	smither, err := NewSmither(sqlDB, rnd, settings.Options...)
    98  	if err != nil {
    99  		t.Fatal(err)
   100  	}
   101  	defer smither.Close()
   102  
   103  	seen := map[string]bool{}
   104  	for i := 0; i < *flagNum; i++ {
   105  		stmt := smither.Generate()
   106  		if err != nil {
   107  			t.Fatalf("%v: %v", stmt, err)
   108  		}
   109  		parsed, err := parser.ParseOne(stmt)
   110  		if err != nil {
   111  			t.Fatalf("%v: %v", stmt, err)
   112  		}
   113  		stmt = prettyCfg.Pretty(parsed.AST)
   114  		fmt.Print("STMT: ", i, "\n", stmt, ";\n\n")
   115  		if *flagCheckVec {
   116  			if _, err := sqlDB.Exec(fmt.Sprintf("EXPLAIN (vec) %s", stmt)); err != nil {
   117  				es := err.Error()
   118  				ok := false
   119  				// It is hard to make queries that can always
   120  				// be vectorized. Hard code a list of error
   121  				// messages we are ok with.
   122  				for _, s := range []string{
   123  					// If the optimizer removes stuff due
   124  					// to something like a `WHERE false`,
   125  					// vec will fail with an error message
   126  					// like this. This is hard to fix
   127  					// because things like `WHERE true AND
   128  					// false` similarly remove rows but are
   129  					// harder to detect.
   130  					"num_rows:0",
   131  					"unsorted distinct",
   132  				} {
   133  					if strings.Contains(es, s) {
   134  						ok = true
   135  						break
   136  					}
   137  				}
   138  				if !ok {
   139  					t.Fatal(err)
   140  				}
   141  			}
   142  		}
   143  		if *flagExec {
   144  			db.Exec(t, `SET statement_timeout = '9s'`)
   145  			if _, err := sqlDB.Exec(stmt); err != nil {
   146  				es := err.Error()
   147  				if !seen[es] {
   148  					seen[es] = true
   149  					fmt.Printf("ERR (%d): %v\n", i, err)
   150  				}
   151  			}
   152  		}
   153  	}
   154  }