Index-based token_idx_prev

sjoerdjob · vmuriart · commit 8f7968ed5c64 · 2016-06-12T17:38:22.000-07:00
Prevent some more calls to token_index in group_identifier_list. They
are now all gone.
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
@@ -166,10 +166,11 @@ def group_identifier_list(tlist):
 
     tidx, token = tlist.token_idx_next_by(m=M_COMMA)
     while token:
-        before, after = tlist.token_prev(tidx), tlist.token_next(tidx)
+        before_idx, before = tlist.token_idx_prev(tidx)
+        after = tlist.token_next(tidx)
 
         if func(before) and func(after):
-            tidx = tlist.token_index(before)
+            tidx = before_idx
             token = tlist.group_tokens_between(sql.IdentifierList, tidx, after, extend=True)
 
         tidx, token = tlist.token_idx_next_by(m=M_COMMA, idx=tidx + 1)
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
@@ -233,12 +233,18 @@ def _token_idx_matching(self, funcs, start=0, end=None, reverse=False):
         if not isinstance(funcs, (list, tuple)):
             funcs = (funcs,)
 
-        iterable = enumerate(self.tokens[start:end], start=start)
-
-        for idx, token in iterable:
-            for func in funcs:
-                if func(token):
-                    return idx, token
+        if reverse:
+            assert end is None
+            for idx in range(start - 2, -1, -1):
+                token = self.tokens[idx]
+                for func in funcs:
+                    if func(token):
+                        return idx, token
+        else:
+            for idx, token in enumerate(self.tokens[start:end], start=start):
+                for func in funcs:
+                    if func(token):
+                        return idx, token
         return None, None
 
     def _token_matching(self, funcs, start=0, end=None, reverse=False):
@@ -312,6 +318,16 @@ def token_not_matching(self, idx, funcs):
     def token_matching(self, idx, funcs):
         return self._token_matching(funcs, idx)
 
+    def token_idx_prev(self, idx, skip_ws=True):
+        """Returns the previous token relative to *idx*.
+
+        If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
+        ``None`` is returned if there's no previous token.
+        """
+        idx += 1  # alot of code usage current pre-compensates for this
+        funcs = lambda tk: not (tk.is_whitespace() and skip_ws)
+        return self._token_idx_matching(funcs, idx, reverse=True)
+
     def token_prev(self, idx, skip_ws=True):
         """Returns the previous token relative to *idx*.
 
