/// An end-of-file marker, not a real token

EOF,

/// A keyword (like SELECT) or an optionally quoted SQL identifier

Word(Word),

/// An unsigned numeric literal

Number(String, bool),

/// A character that could not be tokenized

Char(char),

/// Single quoted string: i.e: 'string'

SingleQuotedString(String),

/// Double quoted string: i.e: "string"

DoubleQuotedString(String),

/// Triple single quoted strings: Example '''abc'''

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

TripleSingleQuotedString(String),

/// Triple double quoted strings: Example """abc"""

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

TripleDoubleQuotedString(String),

/// Dollar quoted string: i.e: $$string$$ or $tag_name$string$tag_name$

DollarQuotedString(DollarQuotedString),

/// Byte string literal: i.e: b'string' or B'string' (note that some backends, such as

/// PostgreSQL, may treat this syntax as a bit string literal instead, i.e: b'10010101')

SingleQuotedByteStringLiteral(String),

/// Byte string literal: i.e: b"string" or B"string"

DoubleQuotedByteStringLiteral(String),

/// Triple single quoted literal with byte string prefix. Example `B'''abc'''`

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

TripleSingleQuotedByteStringLiteral(String),

/// Triple double quoted literal with byte string prefix. Example `B"""abc"""`

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

TripleDoubleQuotedByteStringLiteral(String),

/// Single quoted literal with raw string prefix. Example `R'abc'`

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

SingleQuotedRawStringLiteral(String),

/// Double quoted literal with raw string prefix. Example `R"abc"`

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

DoubleQuotedRawStringLiteral(String),

/// Triple single quoted literal with raw string prefix. Example `R'''abc'''`

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

TripleSingleQuotedRawStringLiteral(String),

/// Triple double quoted literal with raw string prefix. Example `R"""abc"""`

/// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#quoted_literals)

TripleDoubleQuotedRawStringLiteral(String),

/// "National" string literal: i.e: N'string'

NationalStringLiteral(String),

/// "escaped" string literal, which are an extension to the SQL standard: i.e: e'first \n second' or E 'first \n second'

EscapedStringLiteral(String),

/// Hexadecimal string literal: i.e.: X'deadbeef'

HexStringLiteral(String),

/// Comma

Comma,

/// Whitespace (space, tab, etc)

Whitespace(Whitespace),

/// Double equals sign `==`

DoubleEq,

/// Equality operator `=`

Eq,

/// Not Equals operator `<>` (or `!=` in some dialects)

Neq,

/// Less Than operator `<`

Lt,

/// Greater Than operator `>`

Gt,

/// Less Than Or Equals operator `<=`

LtEq,

/// Greater Than Or Equals operator `>=`

GtEq,

/// Spaceship operator <=>

Spaceship,

/// Plus operator `+`

Plus,

/// Minus operator `-`

Minus,

/// Multiplication operator `*`

Mul,

/// Division operator `/`

Div,

/// Integer division operator `//` in DuckDB

DuckIntDiv,

/// Modulo Operator `%`

Mod,

/// String concatenation `||`

StringConcat,

/// Left parenthesis `(`

LParen,

/// Right parenthesis `)`

RParen,

/// Period (used for compound identifiers or projections into nested types)

Period,

/// Colon `:`

Colon,

/// DoubleColon `::` (used for casting in PostgreSQL)

DoubleColon,

/// Assignment `:=` (used for keyword argument in DuckDB macros and some functions, and for variable declarations in DuckDB and Snowflake)

Assignment,

/// SemiColon `;` used as separator for COPY and payload

SemiColon,

/// Backslash `\` used in terminating the COPY payload with `\.`

Backslash,

/// Left bracket `[`

LBracket,

/// Right bracket `]`

RBracket,

/// Ampersand `&`

Ampersand,

/// Pipe `|`

Pipe,

/// Caret `^`

Caret,

/// Left brace `{`

LBrace,

/// Right brace `}`

RBrace,

/// Right Arrow `=>`

RArrow,

/// Sharp `#` used for PostgreSQL Bitwise XOR operator

Sharp,

/// Tilde `~` used for PostgreSQL Bitwise NOT operator or case sensitive match regular expression operator

Tilde,

/// `~*` , a case insensitive match regular expression operator in PostgreSQL

TildeAsterisk,

/// `!~` , a case sensitive not match regular expression operator in PostgreSQL

ExclamationMarkTilde,

/// `!~*` , a case insensitive not match regular expression operator in PostgreSQL

ExclamationMarkTildeAsterisk,

/// `~~`, a case sensitive match pattern operator in PostgreSQL

DoubleTilde,

/// `~~*`, a case insensitive match pattern operator in PostgreSQL

DoubleTildeAsterisk,

/// `!~~`, a case sensitive not match pattern operator in PostgreSQL

ExclamationMarkDoubleTilde,

/// `!~~*`, a case insensitive not match pattern operator in PostgreSQL

ExclamationMarkDoubleTildeAsterisk,

/// `<<`, a bitwise shift left operator in PostgreSQL

ShiftLeft,

/// `>>`, a bitwise shift right operator in PostgreSQL

ShiftRight,

/// `&&`, an overlap operator in PostgreSQL

Overlap,

/// Exclamation Mark `!` used for PostgreSQL factorial operator

ExclamationMark,

/// Double Exclamation Mark `!!` used for PostgreSQL prefix factorial operator

DoubleExclamationMark,

/// AtSign `@` used for PostgreSQL abs operator

AtSign,

/// `^@`, a "starts with" string operator in PostgreSQL

CaretAt,

/// `|/`, a square root math operator in PostgreSQL

PGSquareRoot,

/// `||/`, a cube root math operator in PostgreSQL

PGCubeRoot,

/// `?` or `$` , a prepared statement arg placeholder

Placeholder(String),

/// `->`, used as a operator to extract json field in PostgreSQL

Arrow,

/// `->>`, used as a operator to extract json field as text in PostgreSQL

LongArrow,

/// `#>`, extracts JSON sub-object at the specified path

HashArrow,

/// `#>>`, extracts JSON sub-object at the specified path as text

HashLongArrow,

/// jsonb @> jsonb -> boolean: Test whether left json contains the right json

AtArrow,

/// jsonb <@ jsonb -> boolean: Test whether right json contains the left json

ArrowAt,

/// jsonb #- text[] -> jsonb: Deletes the field or array element at the specified

/// path, where path elements can be either field keys or array indexes.

HashMinus,

/// jsonb @? jsonpath -> boolean: Does JSON path return any item for the specified

/// JSON value?

AtQuestion,

/// jsonb @@ jsonpath → boolean: Returns the result of a JSON path predicate check

/// for the specified JSON value. Only the first item of the result is taken into

/// account. If the result is not Boolean, then NULL is returned.

AtAt,

/// jsonb ? text -> boolean: Checks whether the string exists as a top-level key within the

/// jsonb object

Question,

/// jsonb ?& text[] -> boolean: Check whether all members of the text array exist as top-level

/// keys within the jsonb object

QuestionAnd,

/// jsonb ?| text[] -> boolean: Check whether any member of the text array exists as top-level

/// keys within the jsonb object

QuestionPipe,

/// Custom binary operator

/// This is used to represent any custom binary operator that is not part of the SQL standard.

/// PostgreSQL allows defining custom binary operators using CREATE OPERATOR.

CustomBinaryOperator(String),

fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

match self {

Token::EOF => f.write_str("EOF"),

Token::Word(ref w) => write!(f, "{w}"),

Token::Number(ref n, l) => write!(f, "{}{long}", n, long = if *l { "L" } else { "" }),

Token::Char(ref c) => write!(f, "{c}"),

Token::SingleQuotedString(ref s) => write!(f, "'{s}'"),

Token::TripleSingleQuotedString(ref s) => write!(f, "'''{s}'''"),

Token::DoubleQuotedString(ref s) => write!(f, "\"{s}\""),

Token::TripleDoubleQuotedString(ref s) => write!(f, "\"\"\"{s}\"\"\""),

Token::DollarQuotedString(ref s) => write!(f, "{s}"),

Token::NationalStringLiteral(ref s) => write!(f, "N'{s}'"),

Token::EscapedStringLiteral(ref s) => write!(f, "E'{s}'"),

Token::HexStringLiteral(ref s) => write!(f, "X'{s}'"),

Token::SingleQuotedByteStringLiteral(ref s) => write!(f, "B'{s}'"),

Token::TripleSingleQuotedByteStringLiteral(ref s) => write!(f, "B'''{s}'''"),

Token::DoubleQuotedByteStringLiteral(ref s) => write!(f, "B\"{s}\""),

Token::TripleDoubleQuotedByteStringLiteral(ref s) => write!(f, "B\"\"\"{s}\"\"\""),

Token::SingleQuotedRawStringLiteral(ref s) => write!(f, "R'{s}'"),

Token::DoubleQuotedRawStringLiteral(ref s) => write!(f, "R\"{s}\""),

Token::TripleSingleQuotedRawStringLiteral(ref s) => write!(f, "R'''{s}'''"),

Token::TripleDoubleQuotedRawStringLiteral(ref s) => write!(f, "R\"\"\"{s}\"\"\""),

Token::Comma => f.write_str(","),

Token::Whitespace(ws) => write!(f, "{ws}"),

Token::DoubleEq => f.write_str("=="),

Token::Spaceship => f.write_str("<=>"),

Token::Eq => f.write_str("="),

Token::Neq => f.write_str("<>"),

Token::Lt => f.write_str("<"),

Token::Gt => f.write_str(">"),

Token::LtEq => f.write_str("<="),

Token::GtEq => f.write_str(">="),

Token::Plus => f.write_str("+"),

Token::Minus => f.write_str("-"),

Token::Mul => f.write_str("*"),

Token::Div => f.write_str("/"),

Token::DuckIntDiv => f.write_str("//"),

Token::StringConcat => f.write_str("||"),

Token::Mod => f.write_str("%"),

Token::LParen => f.write_str("("),

Token::RParen => f.write_str(")"),

Token::Period => f.write_str("."),

Token::Colon => f.write_str(":"),

Token::DoubleColon => f.write_str("::"),

Token::Assignment => f.write_str(":="),

Token::SemiColon => f.write_str(";"),

Token::Backslash => f.write_str("\\"),

Token::LBracket => f.write_str("["),

Token::RBracket => f.write_str("]"),

Token::Ampersand => f.write_str("&"),

Token::Caret => f.write_str("^"),

Token::Pipe => f.write_str("|"),

Token::LBrace => f.write_str("{"),

Token::RBrace => f.write_str("}"),

Token::RArrow => f.write_str("=>"),

Token::Sharp => f.write_str("#"),

Token::ExclamationMark => f.write_str("!"),

Token::DoubleExclamationMark => f.write_str("!!"),

Token::Tilde => f.write_str("~"),

Token::TildeAsterisk => f.write_str("~*"),

Token::ExclamationMarkTilde => f.write_str("!~"),

Token::ExclamationMarkTildeAsterisk => f.write_str("!~*"),

Token::DoubleTilde => f.write_str("~~"),

Token::DoubleTildeAsterisk => f.write_str("~~*"),

Token::ExclamationMarkDoubleTilde => f.write_str("!~~"),

Token::ExclamationMarkDoubleTildeAsterisk => f.write_str("!~~*"),

Token::AtSign => f.write_str("@"),

Token::CaretAt => f.write_str("^@"),

Token::ShiftLeft => f.write_str("<<"),

Token::ShiftRight => f.write_str(">>"),

Token::Overlap => f.write_str("&&"),

Token::PGSquareRoot => f.write_str("|/"),

Token::PGCubeRoot => f.write_str("||/"),

Token::Placeholder(ref s) => write!(f, "{s}"),

Token::Arrow => write!(f, "->"),

Token::LongArrow => write!(f, "->>"),

Token::HashArrow => write!(f, "#>"),

Token::HashLongArrow => write!(f, "#>>"),

Token::AtArrow => write!(f, "@>"),

Token::ArrowAt => write!(f, "<@"),

Token::HashMinus => write!(f, "#-"),

Token::AtQuestion => write!(f, "@?"),

Token::AtAt => write!(f, "@@"),

Token::Question => write!(f, "?"),

Token::QuestionAnd => write!(f, "?&"),

Token::QuestionPipe => write!(f, "?|"),

Token::CustomBinaryOperator(s) => f.write_str(s),

}

}

pub fn make_keyword(keyword: &str) -> Self {

Token::make_word(keyword, None)

}

pub fn make_word(word: &str, quote_style: Option<char>) -> Self {

let word_uppercase = word.to_uppercase();

Token::Word(Word {

value: word.to_string(),

quote_style,

keyword: if quote_style.is_none() {

let keyword = ALL_KEYWORDS.binary_search(&word_uppercase.as_str());

keyword.map_or(Keyword::NoKeyword, |x| ALL_KEYWORDS_INDEX[x])

} else {

Keyword::NoKeyword

},

})

}

/// The value of the token, without the enclosing quotes, and with the

/// escape sequences (if any) processed (TODO: escapes are not handled)

pub value: String,

/// An identifier can be "quoted" (&lt;delimited identifier> in ANSI parlance).

/// The standard and most implementations allow using double quotes for this,

/// but some implementations support other quoting styles as well (e.g. \[MS SQL])

pub quote_style: Option<char>,

/// If the word was not quoted and it matched one of the known keywords,

/// this will have one of the values from dialect::keywords, otherwise empty

pub keyword: Keyword,

fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

match self.quote_style {

Some(s) if s == '"' || s == '[' || s == '`' => {

write!(f, "{}{}{}", s, self.value, Word::matching_end_quote(s))

}

None => f.write_str(&self.value),

_ => panic!("Unexpected quote_style!"),

}

}

fn matching_end_quote(ch: char) -> char {

match ch {

'"' => '"', // ANSI and most dialects

'[' => ']', // MS SQL

'`' => '`', // MySQL

_ => panic!("unexpected quoting style!"),

}

}

Space,

Newline,

Tab,

SingleLineComment { comment: String, prefix: String },

MultiLineComment(String),

fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

match self {

Whitespace::Space => f.write_str(" "),

Whitespace::Newline => f.write_str("\n"),

Whitespace::Tab => f.write_str("\t"),

Whitespace::SingleLineComment { prefix, comment } => write!(f, "{prefix}{comment}"),

Whitespace::MultiLineComment(s) => write!(f, "/*{s}*/"),

}

}

/// Line number, starting from 1

pub line: u64,

/// Line column, starting from 1

pub column: u64,

fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

if self.line == 0 {

return Ok(());

}

write!(

f,

// TODO: use standard compiler location syntax (<path>:<line>:<col>)

" at Line: {}, Column: {}",

self.line, self.column,

)

}

pub fn new(token: Token, line: u64, column: u64) -> TokenWithLocation {

TokenWithLocation {

token,

location: Location { line, column },

}

}

pub fn wrap(token: Token) -> TokenWithLocation {

TokenWithLocation::new(token, 0, 0)

}

fn eq(&self, other: &Token) -> bool {

&self.token == other

}

fn eq(&self, other: &TokenWithLocation) -> bool {

self == &other.token

}

fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

self.token.fmt(f)

}

fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

write!(f, "{}{}", self.message, self.location,)

}

peekable: Peekable<Chars<'a>>,

pub line: u64,

pub col: u64,

/// return the next character and advance the stream

pub fn next(&mut self) -> Option<char> {

match self.peekable.next() {

None => None,

Some(s) => {

if s == '\n' {

self.line += 1;

self.col = 1;

} else {

self.col += 1;

}

Some(s)

}

}

}

/// return the next character but do not advance the stream

pub fn peek(&mut self) -> Option<&char> {

self.peekable.peek()

}

pub fn location(&self) -> Location {

Location {

line: self.line,

column: self.col,

}

}

/// e.g. `"abc"`, `'abc'`, `r'abc'`

One,

/// e.g. `"""abc"""`, `'''abc'''`, `r'''abc'''`

Many(NonZeroU8),

/// The character used to quote the string.

quote_style: char,

/// Represents how many quotes characters enclose the string literal.

num_quote_chars: NumStringQuoteChars,

/// The number of opening quotes left to consume, before parsing

/// the remaining string literal.

/// For example: given initial string `"""abc"""`. If the caller has

/// already parsed the first quote for some reason, then this value

/// is set to 1, flagging to look to consume only 2 leading quotes.

num_opening_quotes_to_consume: u8,

/// True if the string uses backslash escaping of special characters

/// e.g `'abc\ndef\'ghi'

backslash_escape: bool,

dialect: &'a dyn Dialect,

query: &'a str,

/// If true (the default), the tokenizer will un-escape literal

/// SQL strings See [`Tokenizer::with_unescape`] for more details.

unescape: bool,

/// Create a new SQL tokenizer for the specified SQL statement

///

/// ```

/// # use sqlparser::tokenizer::{Token, Whitespace, Tokenizer};

/// # use sqlparser::dialect::GenericDialect;

/// # let dialect = GenericDialect{};

/// let query = r#"SELECT 'foo'"#;

///

/// // Parsing the query

/// let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();

///

/// assert_eq!(tokens, vec![

/// Token::make_word("SELECT", None),

/// Token::Whitespace(Whitespace::Space),

/// Token::SingleQuotedString("foo".to_string()),

/// ]);

pub fn new(dialect: &'a dyn Dialect, query: &'a str) -> Self {

Self {

dialect,

query,

unescape: true,

}

}

/// Set unescape mode

///

/// When true (default) the tokenizer unescapes literal values

/// (for example, `""` in SQL is unescaped to the literal `"`).

///

/// When false, the tokenizer provides the raw strings as provided

/// in the query. This can be helpful for programs that wish to

/// recover the *exact* original query text without normalizing

/// the escaping

///

/// # Example

///

/// ```

/// # use sqlparser::tokenizer::{Token, Tokenizer};

/// # use sqlparser::dialect::GenericDialect;

/// # let dialect = GenericDialect{};

/// let query = r#""Foo "" Bar""#;

/// let unescaped = Token::make_word(r#"Foo " Bar"#, Some('"'));

/// let original = Token::make_word(r#"Foo "" Bar"#, Some('"'));

///

/// // Parsing with unescaping (default)

/// let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();

/// assert_eq!(tokens, vec![unescaped]);

///

/// // Parsing with unescape = false

/// let tokens = Tokenizer::new(&dialect, &query)

/// .with_unescape(false)

/// .tokenize().unwrap();

/// assert_eq!(tokens, vec![original]);

/// ```

pub fn with_unescape(mut self, unescape: bool) -> Self {

self.unescape = unescape;

self

}

/// Tokenize the statement and produce a vector of tokens

pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {

let twl = self.tokenize_with_location()?;

Ok(twl.into_iter().map(|t| t.token).collect())

}

/// Tokenize the statement and produce a vector of tokens with location information

pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> {

let mut tokens: Vec<TokenWithLocation> = vec![];

self.tokenize_with_location_into_buf(&mut tokens)

.map(|_| tokens)

}

/// Tokenize the statement and append tokens with location information into the provided buffer.

/// If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error.

pub fn tokenize_with_location_into_buf(

&mut self,

buf: &mut Vec<TokenWithLocation>,

) -> Result<(), TokenizerError> {

let mut state = State {

peekable: self.query.chars().peekable(),

line: 1,

col: 1,

};

let mut location = state.location();

while let Some(token) = self.next_token(&mut state)? {

buf.push(TokenWithLocation { token, location });

location = state.location();

}

Ok(())

}

// Tokenize the identifier or keywords in `ch`

fn tokenize_identifier_or_keyword(

&self,

ch: impl IntoIterator<Item = char>,

chars: &mut State,

) -> Result<Option<Token>, TokenizerError> {

chars.next(); // consume the first char

let ch: String = ch.into_iter().collect();

let word = self.tokenize_word(ch, chars);

// TODO: implement parsing of exponent here

if word.chars().all(|x| x.is_ascii_digit() || x == '.') {

let mut inner_state = State {

peekable: word.chars().peekable(),

line: 0,

col: 0,

};

let mut s = peeking_take_while(&mut inner_state, |ch| matches!(ch, '0'..='9' | '.'));

let s2 = peeking_take_while(chars, |ch| matches!(ch, '0'..='9' | '.'));

s += s2.as_str();

return Ok(Some(Token::Number(s, false)));

}

Ok(Some(Token::make_word(&word, None)))

}

/// Get the next token or return None

fn next_token(&self, chars: &mut State) -> Result<Option<Token>, TokenizerError> {

match chars.peek() {

Some(&ch) => match ch {

' ' => self.consume_and_return(chars, Token::Whitespace(Whitespace::Space)),

'\t' => self.consume_and_return(chars, Token::Whitespace(Whitespace::Tab)),

'\n' => self.consume_and_return(chars, Token::Whitespace(Whitespace::Newline)),

'\r' => {

// Emit a single Whitespace::Newline token for \r and \r\n

chars.next();

if let Some('\n') = chars.peek() {

chars.next();

}

Ok(Some(Token::Whitespace(Whitespace::Newline)))

}

// BigQuery uses b or B for byte string literal

b @ 'B' | b @ 'b' if dialect_of!(self is BigQueryDialect | GenericDialect) => {

chars.next(); // consume

match chars.peek() {

Some('\'') => {

if self.dialect.supports_triple_quoted_string() {

return self

.tokenize_single_or_triple_quoted_string::<fn(String) -> Token>(

chars,

'\'',

false,

Token::SingleQuotedByteStringLiteral,

Token::TripleSingleQuotedByteStringLiteral,

);

}

let s = self.tokenize_single_quoted_string(chars, '\'', false)?;

Ok(Some(Token::SingleQuotedByteStringLiteral(s)))

}

Some('\"') => {

if self.dialect.supports_triple_quoted_string() {

return self

.tokenize_single_or_triple_quoted_string::<fn(String) -> Token>(

chars,

'"',

false,

Token::DoubleQuotedByteStringLiteral,

Token::TripleDoubleQuotedByteStringLiteral,

);

}

let s = self.tokenize_single_quoted_string(chars, '\"', false)?;

Ok(Some(Token::DoubleQuotedByteStringLiteral(s)))

}

_ => {

// regular identifier starting with an "b" or "B"

let s = self.tokenize_word(b, chars);

Ok(Some(Token::make_word(&s, None)))

}

}

}

// BigQuery uses r or R for raw string literal

b @ 'R' | b @ 'r' if dialect_of!(self is BigQueryDialect | GenericDialect) => {

chars.next(); // consume

match chars.peek() {

Some('\'') => self

.tokenize_single_or_triple_quoted_string::<fn(String) -> Token>(

chars,

'\'',

false,

Token::SingleQuotedRawStringLiteral,

Token::TripleSingleQuotedRawStringLiteral,

),

Some('\"') => self

.tokenize_single_or_triple_quoted_string::<fn(String) -> Token>(

chars,

'"',

false,

Token::DoubleQuotedRawStringLiteral,

Token::TripleDoubleQuotedRawStringLiteral,

),

_ => {

// regular identifier starting with an "r" or "R"

let s = self.tokenize_word(b, chars);

Ok(Some(Token::make_word(&s, None)))

}

}

}

// Redshift uses lower case n for national string literal

n @ 'N' | n @ 'n' => {

chars.next(); // consume, to check the next char

match chars.peek() {

Some('\'') => {

// N'...' - a <national character string literal>

let s = self.tokenize_single_quoted_string(chars, '\'', true)?;

Ok(Some(Token::NationalStringLiteral(s)))

}

_ => {

// regular identifier starting with an "N"

let s = self.tokenize_word(n, chars);

Ok(Some(Token::make_word(&s, None)))

}

}

}

// PostgreSQL accepts "escape" string constants, which are an extension to the SQL standard.

x @ 'e' | x @ 'E' => {

let starting_loc = chars.location();

chars.next(); // consume, to check the next char

match chars.peek() {

Some('\'') => {

let s =

self.tokenize_escaped_single_quoted_string(starting_loc, chars)?;

Ok(Some(Token::EscapedStringLiteral(s)))

}

_ => {

// regular identifier starting with an "E" or "e"

let s = self.tokenize_word(x, chars);

Ok(Some(Token::make_word(&s, None)))

}

}

}

// The spec only allows an uppercase 'X' to introduce a hex

// string, but PostgreSQL, at least, allows a lowercase 'x' too.

x @ 'x' | x @ 'X' => {

chars.next(); // consume, to check the next char

match chars.peek() {

Some('\'') => {

// X'...' - a <binary string literal>

let s = self.tokenize_single_quoted_string(chars, '\'', true)?;

Ok(Some(Token::HexStringLiteral(s)))

}

_ => {

// regular identifier starting with an "X"

let s = self.tokenize_word(x, chars);

Ok(Some(Token::make_word(&s, None)))

}

}

}

// single quoted string

'\'' => {

if self.dialect.supports_triple_quoted_string() {

return self

.tokenize_single_or_triple_quoted_string::<fn(String) -> Token>(

chars,

'\'',

self.dialect.supports_string_literal_backslash_escape(),

Token::SingleQuotedString,

Token::TripleSingleQuotedString,

);

}

let s = self.tokenize_single_quoted_string(

chars,

'\'',

self.dialect.supports_string_literal_backslash_escape(),

)?;

Ok(Some(Token::SingleQuotedString(s)))

}

// double quoted string

'\"' if !self.dialect.is_delimited_identifier_start(ch)

&& !self.dialect.is_identifier_start(ch) =>

{

if self.dialect.supports_triple_quoted_string() {

return self

.tokenize_single_or_triple_quoted_string::<fn(String) -> Token>(

chars,

'"',

self.dialect.supports_string_literal_backslash_escape(),

Token::DoubleQuotedString,

Token::TripleDoubleQuotedString,

);

}

let s = self.tokenize_single_quoted_string(

chars,

'"',

self.dialect.supports_string_literal_backslash_escape(),

)?;

Ok(Some(Token::DoubleQuotedString(s)))

}

// delimited (quoted) identifier

quote_start

if self.dialect.is_delimited_identifier_start(ch)

&& self

.dialect

.is_proper_identifier_inside_quotes(chars.peekable.clone()) =>

{

let error_loc = chars.location();

chars.next(); // consume the opening quote

let quote_end = Word::matching_end_quote(quote_start);

let (s, last_char) = self.parse_quoted_ident(chars, quote_end);

if last_char == Some(quote_end) {

Ok(Some(Token::make_word(&s, Some(quote_start))))

} else {

self.tokenizer_error(

error_loc,

format!("Expected close delimiter '{quote_end}' before EOF."),

)

}

}

// numbers and period

'0'..='9' | '.' => {

let mut s = peeking_take_while(chars, |ch| ch.is_ascii_digit());

// match binary literal that starts with 0x

if s == "0" && chars.peek() == Some(&'x') {

chars.next();

let s2 = peeking_take_while(chars, |ch| ch.is_ascii_hexdigit());

return Ok(Some(Token::HexStringLiteral(s2)));

}

// match one period

if let Some('.') = chars.peek() {

s.push('.');

chars.next();

}

s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());

// No number -> Token::Period

if s == "." {

return Ok(Some(Token::Period));

}