Use specialized token_idx_next_by in group_aliased.

sjoerdjob · vmuriart · commit 67dc823e1174 · 2016-06-12T17:38:13.000-07:00
The method group_aliased was making a lot of calls to token_index. By
specializing token_next_by to token_idx_next_by, the calls to
token_index became superfluous.

Also use token_idx_next_by in group_identifier_list.

It was making a lot of calls, which is now more than reduced in half.
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
@@ -163,17 +163,16 @@ def group_identifier_list(tlist):
                     (T.Keyword, T.Comment, T.Wildcard))
 
     func = lambda t: imt(t, i=I_IDENT_LIST, m=M_ROLE, t=T_IDENT_LIST)
-    token = tlist.token_next_by(m=M_COMMA)
 
+    tidx, token = tlist.token_idx_next_by(m=M_COMMA)
     while token:
-        tidx = tlist.token_index(token)
         before, after = tlist.token_prev(tidx), tlist.token_next(tidx)
 
         if func(before) and func(after):
             tidx = tlist.token_index(before)
             token = tlist.group_tokens_between(sql.IdentifierList, tidx, after, extend=True)
 
-        token = tlist.token_next_by(m=M_COMMA, idx=tidx + 1)
+        tidx, token = tlist.token_idx_next_by(m=M_COMMA, idx=tidx + 1)
 
 
 def group_brackets(tlist):
@@ -217,13 +216,12 @@ def group_aliased(tlist):
     I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier,
                )  # sql.Operation)
 
-    token = tlist.token_next_by(i=I_ALIAS, t=T.Number)
+    tidx, token = tlist.token_idx_next_by(i=I_ALIAS, t=T.Number)
     while token:
-        tidx = tlist.token_index(token)
         next_ = tlist.token_next(tidx)
         if imt(next_, i=sql.Identifier):
             token = tlist.group_tokens_between(sql.Identifier, tidx, next_, extend=True)
-        token = tlist.token_next_by(i=I_ALIAS, t=T.Number, idx=tidx + 1)
+        tidx, token = tlist.token_idx_next_by(i=I_ALIAS, t=T.Number, idx=tidx + 1)
 
 
 def group_typecasts(tlist):
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
@@ -225,6 +225,22 @@ def get_sublists(self):
     def _groupable_tokens(self):
         return self.tokens
 
+    def _token_idx_matching(self, funcs, start=0, end=None, reverse=False):
+        """next token that match functions"""
+        if start is None:
+            return None
+
+        if not isinstance(funcs, (list, tuple)):
+            funcs = (funcs,)
+
+        iterable = enumerate(self.tokens[start:end], start=start)
+
+        for idx, token in iterable:
+            for func in funcs:
+                if func(token):
+                    return idx, token
+        return None, None
+
     def _token_matching(self, funcs, start=0, end=None, reverse=False):
         """next token that match functions"""
         if start is None:
@@ -259,6 +275,10 @@ def token_first(self, ignore_whitespace=True, ignore_comments=False):
                                 (ignore_comments and imt(tk, i=Comment)))
         return self._token_matching(funcs)
 
+    def token_idx_next_by(self, i=None, m=None, t=None, idx=0, end=None):
+        funcs = lambda tk: imt(tk, i, m, t)
+        return self._token_idx_matching(funcs, idx, end)
+
     def token_next_by(self, i=None, m=None, t=None, idx=0, end=None):
         funcs = lambda tk: imt(tk, i, m, t)
         return self._token_matching(funcs, idx, end)
