get rid of tokens.Punctuation.ArrayIndex, add negative lookbehind for sqlite identifiers

darikg · darikg · commit c9dc9c8b79a3 · 2015-03-04T10:34:05.000-05:00
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
@@ -194,8 +194,10 @@ class Lexer(object):
             (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
             # not a real string literal in ANSI SQL:
             (r'(""|".*?[^\\]")', tokens.String.Symbol),
-            (r'(?<=[\w\]])(\[[^\]]*?\])', tokens.Punctuation.ArrayIndex),
-            (r'(\[[^\]]+\])', tokens.Name),
+            # sqlite names can be escaped with [square brackets]. left bracket
+            # cannot be preceded by word character or a right bracket --
+            # otherwise it's probably an array index
+            (r'(?<![\w\])])(\[[^\]]+\])', tokens.Name),
             (r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
             (r'END(\s+IF|\s+LOOP)?\b', tokens.Keyword),
             (r'NOT NULL\b', tokens.Keyword),
diff --git a/sqlparse/tokens.py b/sqlparse/tokens.py
@@ -57,7 +57,6 @@ def __repr__(self):
 String = Literal.String
 Number = Literal.Number
 Punctuation = Token.Punctuation
-ArrayIndex = Punctuation.ArrayIndex
 Operator = Token.Operator
 Comparison = Operator.Comparison
 Wildcard = Token.Wildcard
