Fix incorrect parsing of string literals with line breaks (fixes andialbrecht#118).

andialbrecht · andialbrecht · commit 6e476cfded15 · 2013-11-06T11:59:52.000+01:00
diff --git a/CHANGES b/CHANGES
@@ -1,3 +1,10 @@
+Development
+-----------
+
+Bug Fixes
+* Fix incorrect parsing of string literals containing line breaks (issue118).
+
+
 Release 0.1.10 (Nov 02, 2013)
 -----------------------------
 
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
@@ -191,7 +191,7 @@ class Lexer(object):
             (r'[-]?[0-9]*\.[0-9]+', tokens.Number.Float),
             (r'[-]?[0-9]+', tokens.Number.Integer),
             # TODO: Backslash escapes?
-            (r"(''|'.*?[^\\]')", tokens.String.Single),
+            (r"'(''|\\'|[^'])*'", tokens.String.Single),
             # not a real string literal in ANSI SQL:
             (r'(""|".*?[^\\]")', tokens.String.Symbol),
             (r'(\[.*[^\]]\])', tokens.Name),
@@ -209,7 +209,7 @@ class Lexer(object):
             (r'/\*', tokens.Comment.Multiline, 'multiline-comments'),
             (r'\*/', tokens.Comment.Multiline, '#pop'),
             (r'[^/\*]+', tokens.Comment.Multiline),
-            (r'[/*]', tokens.Comment.Multiline)
+            (r'[/*]', tokens.Comment.Multiline),
         ]}
 
     def __init__(self):
diff --git a/tests/test_parse.py b/tests/test_parse.py
@@ -169,3 +169,9 @@ def test_double_quotes_are_identifiers():
     p = sqlparse.parse('"foo"')[0].tokens
     assert len(p) == 1
     assert isinstance(p[0], sqlparse.sql.Identifier)
+
+
+def test_single_quotes_with_linebreaks():  # issue118
+    p = sqlparse.parse("'f\nf'")[0].tokens
+    assert len(p) == 1
+    assert p[0].ttype is T.String.Single
