Skip to content

Commit a7ffa64

Browse files
vmuriartvmuriart
committed
Merge remote-tracking branch 'core/long_live_indexes' into develop
2 parents fae3d94 + 89d4f68 commit a7ffa64

File tree

5 files changed

+181
-73
lines changed

5 files changed

+181
-73
lines changed

‎sqlparse/engine/grouping.py‎

Lines changed: 69 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
from sqlparse import sql
99
from sqlparse import tokens as T
10-
from sqlparse.utils import recurse, imt, find_matching
10+
from sqlparse.utils import recurse, imt
1111

1212
M_ROLE = (T.Keyword, ('null', 'role'))
1313
M_SEMICOLON = (T.Punctuation, ';')
@@ -23,40 +23,47 @@ def _group_left_right(tlist, m, cls,
2323
valid_right=lambda t: t is not None,
2424
semicolon=False):
2525
"""Groups together tokens that are joined by a middle token. ie. x < y"""
26+
[_group_left_right(sgroup, m, cls, valid_left, valid_right, semicolon)
27+
for sgroup in tlist.get_sublists() if not isinstance(sgroup, cls)]
2628

27-
for token in list(tlist):
28-
if token.is_group() and not isinstance(token, cls):
29-
_group_left_right(token, m, cls, valid_left, valid_right,
30-
semicolon)
31-
32-
if not token.match(*m):
33-
continue
34-
35-
left, right = tlist.token_prev(token), tlist.token_next(token)
29+
token = tlist.token_next_by(m=m)
30+
while token:
31+
tidx = tlist.token_index(token)
32+
left, right = tlist.token_prev(tidx), tlist.token_next(tidx)
3633

3734
if valid_left(left) and valid_right(right):
3835
if semicolon:
3936
# only overwrite if a semicolon present.
40-
sright = tlist.token_next_by(m=M_SEMICOLON, idx=right)
37+
sright = tlist.token_next_by(m=M_SEMICOLON, idx=tidx + 1)
4138
right = sright or right
42-
tokens = tlist.tokens_between(left, right)
43-
tlist.group_tokens(cls, tokens, extend=True)
39+
# Luckily, this leaves the position of `token` intact.
40+
token = tlist.group_tokens_between(cls, left, right, extend=True)
41+
token = tlist.token_next_by(m=m, idx=tidx + 1)
4442

4543

4644
def _group_matching(tlist, cls):
4745
"""Groups Tokens that have beginning and end."""
48-
[_group_matching(sgroup, cls) for sgroup in tlist.get_sublists()
49-
if not isinstance(sgroup, cls)]
50-
idx = 1 if isinstance(tlist, cls) else 0
46+
idx = 1 if imt(tlist, i=cls) else 0
5147

52-
token = tlist.token_next_by(m=cls.M_OPEN, idx=idx)
53-
while token:
54-
end = find_matching(tlist, token, cls.M_OPEN, cls.M_CLOSE)
55-
if end is not None:
56-
tokens = tlist.tokens_between(token, end)
57-
token = tlist.group_tokens(cls, tokens)
58-
_group_matching(token, cls)
59-
token = tlist.token_next_by(m=cls.M_OPEN, idx=token)
48+
opens = []
49+
50+
while True:
51+
try:
52+
token = tlist.tokens[idx]
53+
except IndexError:
54+
break
55+
56+
if token.match(*cls.M_OPEN):
57+
opens.append(idx)
58+
elif token.match(*cls.M_CLOSE):
59+
try:
60+
open_idx = opens.pop()
61+
except IndexError:
62+
break
63+
tlist.group_tokens_between(cls, open_idx, idx)
64+
idx = open_idx
65+
66+
idx += 1
6067

6168

6269
def group_if(tlist):
@@ -109,8 +116,9 @@ def group_identifier(tlist):
109116

110117
token = tlist.token_next_by(t=T_IDENT)
111118
while token:
112-
token = tlist.group_tokens(sql.Identifier, [token, ])
113-
token = tlist.token_next_by(t=T_IDENT, idx=token)
119+
tidx = tlist.token_index(token)
120+
token = tlist.group_tokens_between(sql.Identifier, tidx, tidx)
121+
token = tlist.token_next_by(t=T_IDENT, idx=tidx + 1)
114122

115123

116124
def group_period(tlist):
@@ -127,12 +135,11 @@ def group_period(tlist):
127135
def group_arrays(tlist):
128136
token = tlist.token_next_by(i=sql.SquareBrackets)
129137
while token:
130-
prev = tlist.token_prev(token)
138+
prev = tlist.token_prev(tlist.token_index(token))
131139
if imt(prev, i=(sql.SquareBrackets, sql.Identifier, sql.Function),
132140
t=(T.Name, T.String.Symbol,)):
133-
tokens = tlist.tokens_between(prev, token)
134-
token = tlist.group_tokens(sql.Identifier, tokens, extend=True)
135-
token = tlist.token_next_by(i=sql.SquareBrackets, idx=token)
141+
token = tlist.group_tokens_between(sql.Identifier, prev, token, extend=True)
142+
token = tlist.token_next_by(i=sql.SquareBrackets, idx=tlist.token_index(token) + 1)
136143

137144

138145
@recurse(sql.Identifier)
@@ -145,14 +152,13 @@ def group_operator(tlist):
145152

146153
token = tlist.token_next_by(t=(T.Operator, T.Wildcard))
147154
while token:
148-
left, right = tlist.token_prev(token), tlist.token_next(token)
155+
left, right = tlist.token_prev(tlist.token_index(token)), tlist.token_next(tlist.token_index(token))
149156

150157
if func(left) and func(right):
151158
token.ttype = T.Operator
152-
tokens = tlist.tokens_between(left, right)
153-
token = tlist.group_tokens(sql.Operation, tokens)
159+
token = tlist.group_tokens_between(sql.Operation, left, right)
154160

155-
token = tlist.token_next_by(t=(T.Operator, T.Wildcard), idx=token)
161+
token = tlist.token_next_by(t=(T.Operator, T.Wildcard), idx=tlist.token_index(token) + 1)
156162

157163

158164
@recurse(sql.IdentifierList)
@@ -163,15 +169,17 @@ def group_identifier_list(tlist):
163169
(T.Keyword, T.Comment, T.Wildcard))
164170

165171
func = lambda t: imt(t, i=I_IDENT_LIST, m=M_ROLE, t=T_IDENT_LIST)
166-
token = tlist.token_next_by(m=M_COMMA)
167172

173+
tidx, token = tlist.token_idx_next_by(m=M_COMMA)
168174
while token:
169-
before, after = tlist.token_prev(token), tlist.token_next(token)
175+
before_idx, before = tlist.token_idx_prev(tidx)
176+
after_idx, after = tlist.token_idx_next(tidx)
170177

171178
if func(before) and func(after):
172-
tokens = tlist.tokens_between(before, after)
173-
token = tlist.group_tokens(sql.IdentifierList, tokens, extend=True)
174-
token = tlist.token_next_by(m=M_COMMA, idx=token)
179+
tidx = before_idx
180+
token = tlist.group_tokens_between(sql.IdentifierList, tidx, after_idx, extend=True)
181+
182+
tidx, token = tlist.token_idx_next_by(m=M_COMMA, idx=tidx + 1)
175183

176184

177185
def group_brackets(tlist):
@@ -187,43 +195,40 @@ def group_comments(tlist):
187195
token = tlist.token_next_by(t=T.Comment)
188196
while token:
189197
end = tlist.token_not_matching(
190-
token, lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace())
198+
tlist.token_index(token) + 1, lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace())
191199
if end is not None:
192-
end = tlist.token_prev(end, False)
193-
tokens = tlist.tokens_between(token, end)
194-
token = tlist.group_tokens(sql.Comment, tokens)
200+
end = tlist.token_prev(tlist.token_index(end), False)
201+
token = tlist.group_tokens_between(sql.Comment, token, end)
195202

196-
token = tlist.token_next_by(t=T.Comment, idx=token)
203+
token = tlist.token_next_by(t=T.Comment, idx=tlist.token_index(token) + 1)
197204

198205

199206
@recurse(sql.Where)
200207
def group_where(tlist):
201208
token = tlist.token_next_by(m=sql.Where.M_OPEN)
202209
while token:
203-
end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=token)
210+
end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=tlist.token_index(token) + 1)
204211

205212
if end is None:
206-
tokens = tlist.tokens_between(token, tlist._groupable_tokens[-1])
213+
end = tlist._groupable_tokens[-1]
207214
else:
208-
tokens = tlist.tokens_between(
209-
token, tlist.tokens[tlist.token_index(end) - 1])
215+
end = tlist.tokens[tlist.token_index(end) - 1]
210216

211-
token = tlist.group_tokens(sql.Where, tokens)
212-
token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=token)
217+
token = tlist.group_tokens_between(sql.Where, token, end)
218+
token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=tlist.token_index(token) + 1)
213219

214220

215221
@recurse()
216222
def group_aliased(tlist):
217223
I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier,
218224
sql.Operation)
219225

220-
token = tlist.token_next_by(i=I_ALIAS, t=T.Number)
226+
tidx, token = tlist.token_idx_next_by(i=I_ALIAS, t=T.Number)
221227
while token:
222-
next_ = tlist.token_next(token)
228+
next_index_, next_ = tlist.token_idx_next(tidx)
223229
if imt(next_, i=sql.Identifier):
224-
tokens = tlist.tokens_between(token, next_)
225-
token = tlist.group_tokens(sql.Identifier, tokens, extend=True)
226-
token = tlist.token_next_by(i=I_ALIAS, t=T.Number, idx=token)
230+
token = tlist.group_tokens_between(sql.Identifier, tidx, next_index_, extend=True)
231+
tidx, token = tlist.token_idx_next_by(i=I_ALIAS, t=T.Number, idx=tidx + 1)
227232

228233

229234
def group_typecasts(tlist):
@@ -243,33 +248,30 @@ def group_functions(tlist):
243248
return
244249
token = tlist.token_next_by(t=T.Name)
245250
while token:
246-
next_ = tlist.token_next(token)
251+
next_ = tlist.token_next(tlist.token_index(token))
247252
if imt(next_, i=sql.Parenthesis):
248-
tokens = tlist.tokens_between(token, next_)
249-
token = tlist.group_tokens(sql.Function, tokens)
250-
token = tlist.token_next_by(t=T.Name, idx=token)
253+
token = tlist.group_tokens_between(sql.Function, token, next_)
254+
token = tlist.token_next_by(t=T.Name, idx=tlist.token_index(token) + 1)
251255

252256

253257
def group_order(tlist):
254258
"""Group together Identifier and Asc/Desc token"""
255259
token = tlist.token_next_by(t=T.Keyword.Order)
256260
while token:
257-
prev = tlist.token_prev(token)
261+
prev = tlist.token_prev(tlist.token_index(token))
258262
if imt(prev, i=sql.Identifier, t=T.Number):
259-
tokens = tlist.tokens_between(prev, token)
260-
token = tlist.group_tokens(sql.Identifier, tokens)
261-
token = tlist.token_next_by(t=T.Keyword.Order, idx=token)
263+
token = tlist.group_tokens_between(sql.Identifier, prev, token)
264+
token = tlist.token_next_by(t=T.Keyword.Order, idx=tlist.token_index(token) + 1)
262265

263266

264267
@recurse()
265268
def align_comments(tlist):
266269
token = tlist.token_next_by(i=sql.Comment)
267270
while token:
268-
before = tlist.token_prev(token)
271+
before = tlist.token_prev(tlist.token_index(token))
269272
if isinstance(before, sql.TokenList):
270-
tokens = tlist.tokens_between(before, token)
271-
token = tlist.group_tokens(sql.TokenList, tokens, extend=True)
272-
token = tlist.token_next_by(i=sql.Comment, idx=token)
273+
token = tlist.group_tokens_between(sql.TokenList, before, token, extend=True)
274+
token = tlist.token_next_by(i=sql.Comment, idx=tlist.token_index(token) + 1)
273275

274276

275277
def group(stmt):

0 commit comments

Comments
 (0)