github.com/cockroachdb/cockroachdb-parser@v0.23.3-0.20240213214944-911057d40c9a/pkg/sql/parser/scanner.go (about)

     1  // Copyright 2021 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package parser
    12  
    13  import (
    14  	"github.com/cockroachdb/cockroachdb-parser/pkg/sql/lexbase"
    15  	"github.com/cockroachdb/cockroachdb-parser/pkg/sql/scanner"
    16  )
    17  
    18  func makeSQLScanner(str string) scanner.SQLScanner {
    19  	var s scanner.SQLScanner
    20  	s.Init(str)
    21  	return s
    22  }
    23  
    24  // SplitFirstStatement returns the length of the prefix of the string up to and
    25  // including the first semicolon that separates statements. If there is no
    26  // including the first semicolon that separates statements. If there is no
    27  // semicolon, returns ok=false.
    28  func SplitFirstStatement(sql string) (pos int, ok bool) {
    29  	s := makeSQLScanner(sql)
    30  	var lval = &sqlSymType{}
    31  	for {
    32  		s.Scan(lval)
    33  		switch lval.ID() {
    34  		case 0, lexbase.ERROR:
    35  			return 0, false
    36  		case ';':
    37  			return s.Pos(), true
    38  		}
    39  	}
    40  }
    41  
    42  // Tokens decomposes the input into lexical tokens.
    43  func Tokens(sql string) (tokens []TokenString, ok bool) {
    44  	s := makeSQLScanner(sql)
    45  	for {
    46  		var lval = &sqlSymType{}
    47  		s.Scan(lval)
    48  		if lval.ID() == lexbase.ERROR {
    49  			return nil, false
    50  		}
    51  		if lval.ID() == 0 {
    52  			break
    53  		}
    54  		tokens = append(tokens, TokenString{TokenID: lval.ID(), Str: lval.Str()})
    55  	}
    56  	return tokens, true
    57  }
    58  
    59  // TokensIgnoreErrors decomposes the input into lexical tokens and
    60  // ignores errors.
    61  func TokensIgnoreErrors(sql string) (tokens []TokenString) {
    62  	s := makeSQLScanner(sql)
    63  	for {
    64  		var lval = &sqlSymType{}
    65  		s.Scan(lval)
    66  		if lval.ID() == 0 {
    67  			break
    68  		}
    69  		tokens = append(tokens, TokenString{TokenID: lval.ID(), Str: lval.Str()})
    70  	}
    71  	return tokens
    72  }
    73  
    74  // TokenString is the unit value returned by Tokens.
    75  type TokenString struct {
    76  	TokenID int32
    77  	Str     string
    78  }