Improve performance of reindent engine a bit (targets issue41).

andialbrecht · andialbrecht · commit bac3c5156f5d · 2013-05-10T10:16:17.000+02:00
diff --git a/sqlparse/filters.py b/sqlparse/filters.py
@@ -250,10 +250,17 @@ def __init__(self, width=2, char=' ', line_width=None):
         self._curr_stmt = None
         self._last_stmt = None
 
+    def _flatten_up_to_token(self, token):
+        """Yields all tokens up to token plus the next one."""
+        # helper for _get_offset
+        iterator = self._curr_stmt.flatten()
+        for t in iterator:
+            yield t
+            if t == token:
+                raise StopIteration
+
     def _get_offset(self, token):
-        all_ = list(self._curr_stmt.flatten())
-        idx = all_.index(token)
-        raw = ''.join(unicode(x) for x in all_[:idx + 1])
+        raw = ''.join(map(unicode, self._flatten_up_to_token(token)))
         line = raw.splitlines()[-1]
         # Now take current offset into account and return relative offset.
         full_offset = len(line) - len(self.char * (self.width * self.indent))
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
@@ -171,7 +171,7 @@ class Lexer(object):
             # $ matches *before* newline, therefore we have two patterns
             # to match Comment.Single
             (r'--.*?$', tokens.Comment.Single),
-            (r'(\r|\n|\r\n)', tokens.Newline),
+            (r'(\r\n|\r|\n)', tokens.Newline),
             (r'\s+', tokens.Whitespace),
             (r'/\*', tokens.Comment.Multiline, 'multiline-comments'),
             (r':=', tokens.Assignment),
