Allow identifiers to start with an underscore (fixes andialbrecht#175).

andialbrecht · andialbrecht · commit 15b0cb9e75ca · 2015-03-01T11:52:34.000+01:00
diff --git a/CHANGES b/CHANGES
@@ -7,6 +7,7 @@ Bug Fixes
 * Add missing SQL types (issue154, issue155, issue156, by jukebox).
 * Fix parsing of multi-line comments (issue172, by JacekPliszka).
 * Fix parsing of escaped backslashes (issue174, by caseyching).
+* Fix parsing of identifiers starting with underscore (issue175).
 
 Enhancements
 * Improve formatting of HAVING statements.
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
@@ -202,7 +202,7 @@ class Lexer(object):
             (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
             (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
             (r'(?<=\.)[^\W\d_]\w*', tokens.Name),
-            (r'[^\W\d_]\w*', is_keyword),
+            (r'[^\W\d]\w*', is_keyword),
             (r'[;:()\[\],\.]', tokens.Punctuation),
             (r'[<>=~!]+', tokens.Operator.Comparison),
             (r'[+/@#%^&|`?^-]+', tokens.Operator),
@@ -292,7 +292,6 @@ def get_tokens_unprocessed(self, stream, stack=('root',)):
             for rexmatch, action, new_state in statetokens:
                 m = rexmatch(text, pos)
                 if m:
-                    # print rex.pattern
                     value = m.group()
                     if value in known_names:
                         yield pos, known_names[value], value
diff --git a/tests/test_parse.py b/tests/test_parse.py
@@ -145,6 +145,15 @@ def test_quoted_identifier():
     assert t[2].get_real_name() == 'y'
 
 
+@pytest.mark.parametrize('name', [
+    'foo',
+    '_foo',
+])
+def test_valid_identifier_names(name):  # issue175
+    t = sqlparse.parse(name)[0].tokens
+    assert isinstance(t[0], sqlparse.sql.Identifier)
+
+
 def test_psql_quotation_marks():  # issue83
     # regression: make sure plain $$ work
     t = sqlparse.split("""
