Skip to content

Commit a795be1

Browse files
vmuriartvmuriart
committed
Change token_ funcs to token_idx funcs
1 parent 997f95b commit a795be1

File tree

7 files changed

+167
-137
lines changed

7 files changed

+167
-137
lines changed

‎examples/column_defs_lowlevel.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,16 @@ def extract_definitions(token_list):
1717
definitions = []
1818
tmp = []
1919
# grab the first token, ignoring whitespace. idx=1 to skip open (
20-
token = token_list.token_next(1)
20+
tidx, token = token_list.token_idx_next(1)
2121
while token and not token.match(sqlparse.tokens.Punctuation, ')'):
2222
tmp.append(token)
2323
# grab the next token, this times including whitespace
24-
token = token_list.token_next(token, skip_ws=False)
24+
tidx, token = token_list.token_idx_next(tidx, skip_ws=False)
2525
# split on ",", except when on end of statement
2626
if token and token.match(sqlparse.tokens.Punctuation, ','):
2727
definitions.append(tmp)
2828
tmp = []
29-
token = token_list.token_next(token)
29+
tidx, token = token_list.token_idx_next(tidx)
3030
if tmp and isinstance(tmp[0], sqlparse.sql.Identifier):
3131
definitions.append(tmp)
3232
return definitions
@@ -41,7 +41,7 @@ def extract_definitions(token_list):
4141
parsed = sqlparse.parse(SQL)[0]
4242

4343
# extract the parenthesis which holds column definitions
44-
par = parsed.token_next_by(i=sqlparse.sql.Parenthesis)
44+
_, par = parsed.token_idx_next_by(i=sqlparse.sql.Parenthesis)
4545
columns = extract_definitions(par)
4646

4747
for column in columns:

‎sqlparse/engine/grouping.py‎

Lines changed: 59 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,16 @@ def _group_left_right(tlist, m, cls,
3232
continue
3333

3434
tidx = tlist.token_index(token)
35-
left, right = tlist.token_prev(tidx), tlist.token_next(tidx)
35+
pidx, prev_ = tlist.token_idx_prev(tidx)
36+
nidx, next_ = tlist.token_idx_next(tidx)
3637

37-
if valid_left(left) and valid_right(right):
38+
if valid_left(prev_) and valid_right(next_):
3839
if semicolon:
3940
# only overwrite if a semicolon present.
40-
sright = tlist.token_next_by(m=M_SEMICOLON, idx=tidx + 1)
41-
right = sright or right
41+
snidx, _ = tlist.token_idx_next_by(m=M_SEMICOLON, idx=nidx)
42+
nidx = snidx or nidx
4243
# Luckily, this leaves the position of `token` intact.
43-
tlist.group_tokens_between(cls, left, right, extend=True)
44+
tlist.group_tokens_between(cls, pidx, nidx, extend=True)
4445

4546

4647
def _group_matching(tlist, cls):
@@ -114,11 +115,10 @@ def group_case(tlist):
114115
def group_identifier(tlist):
115116
T_IDENT = (T.String.Symbol, T.Name)
116117

117-
token = tlist.token_next_by(t=T_IDENT)
118+
tidx, token = tlist.token_idx_next_by(t=T_IDENT)
118119
while token:
119-
tidx = tlist.token_index(token)
120-
token = tlist.group_tokens_between(sql.Identifier, tidx, tidx)
121-
token = tlist.token_next_by(t=T_IDENT, idx=tidx + 1)
120+
tlist.group_tokens_between(sql.Identifier, tidx, tidx)
121+
tidx, token = tlist.token_idx_next_by(t=T_IDENT, idx=tidx + 1)
122122

123123

124124
def group_period(tlist):
@@ -133,13 +133,14 @@ def group_period(tlist):
133133

134134

135135
def group_arrays(tlist):
136-
token = tlist.token_next_by(i=sql.SquareBrackets)
136+
tidx, token = tlist.token_idx_next_by(i=sql.SquareBrackets)
137137
while token:
138-
prev = tlist.token_prev(tlist.token_index(token))
138+
pidx, prev = tlist.token_idx_prev(tidx)
139139
if imt(prev, i=(sql.SquareBrackets, sql.Identifier, sql.Function),
140140
t=(T.Name, T.String.Symbol,)):
141-
token = tlist.group_tokens_between(sql.Identifier, prev, token, extend=True)
142-
token = tlist.token_next_by(i=sql.SquareBrackets, idx=tlist.token_index(token) + 1)
141+
tlist.group_tokens_between(sql.Identifier, pidx, tidx, extend=True)
142+
tidx = pidx
143+
tidx, token = tlist.token_idx_next_by(i=sql.SquareBrackets, idx=tidx + 1)
143144

144145

145146
@recurse(sql.Identifier)
@@ -150,15 +151,18 @@ def group_operator(tlist):
150151
T_CYCLE = T_NUMERICAL + T_STRING + T_NAME
151152
func = lambda tk: imt(tk, i=I_CYCLE, t=T_CYCLE)
152153

153-
token = tlist.token_next_by(t=(T.Operator, T.Wildcard))
154+
tidx, token = tlist.token_idx_next_by(t=(T.Operator, T.Wildcard))
154155
while token:
155-
left, right = tlist.token_prev(tlist.token_index(token)), tlist.token_next(tlist.token_index(token))
156+
pidx, prev_ = tlist.token_idx_prev(tidx)
157+
nidx, next_ = tlist.token_idx_next(tidx)
156158

157-
if func(left) and func(right):
159+
if func(prev_) and func(next_):
158160
token.ttype = T.Operator
159-
token = tlist.group_tokens_between(sql.Operation, left, right)
161+
tlist.group_tokens_between(sql.Operation, pidx, nidx)
162+
tidx = pidx
160163

161-
token = tlist.token_next_by(t=(T.Operator, T.Wildcard), idx=tlist.token_index(token) + 1)
164+
tidx, token = tlist.token_idx_next_by(t=(T.Operator, T.Wildcard),
165+
idx=tidx + 1)
162166

163167

164168
@recurse(sql.IdentifierList)
@@ -172,13 +176,12 @@ def group_identifier_list(tlist):
172176

173177
tidx, token = tlist.token_idx_next_by(m=M_COMMA)
174178
while token:
175-
before_idx, before = tlist.token_idx_prev(tidx)
176-
after_idx, after = tlist.token_idx_next(tidx)
177-
178-
if func(before) and func(after):
179-
tidx = before_idx
180-
token = tlist.group_tokens_between(sql.IdentifierList, tidx, after_idx, extend=True)
179+
pidx, prev_ = tlist.token_idx_prev(tidx)
180+
nidx, next_ = tlist.token_idx_next(tidx)
181181

182+
if func(prev_) and func(next_):
183+
tlist.group_tokens_between(sql.IdentifierList, pidx, nidx, extend=True)
184+
tidx = pidx
182185
tidx, token = tlist.token_idx_next_by(m=M_COMMA, idx=tidx + 1)
183186

184187

@@ -192,31 +195,32 @@ def group_parenthesis(tlist):
192195

193196
@recurse(sql.Comment)
194197
def group_comments(tlist):
195-
token = tlist.token_next_by(t=T.Comment)
198+
tidx, token = tlist.token_idx_next_by(t=T.Comment)
196199
while token:
197200
end = tlist.token_not_matching(
198-
lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace(),
199-
idx=tlist.token_index(token) + 1)
201+
lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace(), idx=tidx + 1)
200202
if end is not None:
201-
end = tlist.token_prev(tlist.token_index(end), False)
202-
token = tlist.group_tokens_between(sql.Comment, token, end)
203+
eidx = tlist.token_index(end)
204+
eidx, end = tlist.token_idx_prev(eidx, skip_ws=False)
205+
tlist.group_tokens_between(sql.Comment, tidx, eidx)
203206

204-
token = tlist.token_next_by(t=T.Comment, idx=tlist.token_index(token) + 1)
207+
tidx, token = tlist.token_idx_next_by(t=T.Comment, idx=tidx + 1)
205208

206209

207210
@recurse(sql.Where)
208211
def group_where(tlist):
209-
token = tlist.token_next_by(m=sql.Where.M_OPEN)
212+
tidx, token = tlist.token_idx_next_by(m=sql.Where.M_OPEN)
210213
while token:
211-
end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=tlist.token_index(token) + 1)
214+
eidx, end = tlist.token_idx_next_by(m=sql.Where.M_CLOSE, idx=tidx + 1)
212215

213216
if end is None:
214217
end = tlist._groupable_tokens[-1]
215218
else:
216-
end = tlist.tokens[tlist.token_index(end) - 1]
217-
218-
token = tlist.group_tokens_between(sql.Where, token, end)
219-
token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=tlist.token_index(token) + 1)
219+
end = tlist.tokens[eidx - 1]
220+
# TODO: convert this to eidx instead of end token.
221+
# i think above values are len(tlist) and eidx-1
222+
tlist.group_tokens_between(sql.Where, tidx, end)
223+
tidx, token = tlist.token_idx_next_by(m=sql.Where.M_OPEN, idx=tidx + 1)
220224

221225

222226
@recurse()
@@ -226,9 +230,9 @@ def group_aliased(tlist):
226230

227231
tidx, token = tlist.token_idx_next_by(i=I_ALIAS, t=T.Number)
228232
while token:
229-
next_index_, next_ = tlist.token_idx_next(tidx)
233+
nidx, next_ = tlist.token_idx_next(tidx)
230234
if imt(next_, i=sql.Identifier):
231-
token = tlist.group_tokens_between(sql.Identifier, tidx, next_index_, extend=True)
235+
tlist.group_tokens_between(sql.Identifier, tidx, nidx, extend=True)
232236
tidx, token = tlist.token_idx_next_by(i=I_ALIAS, t=T.Number, idx=tidx + 1)
233237

234238

@@ -247,32 +251,35 @@ def group_functions(tlist):
247251
has_table = True
248252
if has_create and has_table:
249253
return
250-
token = tlist.token_next_by(t=T.Name)
254+
255+
tidx, token = tlist.token_idx_next_by(t=T.Name)
251256
while token:
252-
next_ = tlist.token_next(tlist.token_index(token))
253-
if imt(next_, i=sql.Parenthesis):
254-
token = tlist.group_tokens_between(sql.Function, token, next_)
255-
token = tlist.token_next_by(t=T.Name, idx=tlist.token_index(token) + 1)
257+
nidx, next_ = tlist.token_idx_next(tidx)
258+
if isinstance(next_, sql.Parenthesis):
259+
tlist.group_tokens_between(sql.Function, tidx, nidx)
260+
tidx, token = tlist.token_idx_next_by(t=T.Name, idx=tidx + 1)
256261

257262

258263
def group_order(tlist):
259264
"""Group together Identifier and Asc/Desc token"""
260-
token = tlist.token_next_by(t=T.Keyword.Order)
265+
tidx, token = tlist.token_idx_next_by(t=T.Keyword.Order)
261266
while token:
262-
prev = tlist.token_prev(tlist.token_index(token))
267+
pidx, prev = tlist.token_idx_prev(tidx)
263268
if imt(prev, i=sql.Identifier, t=T.Number):
264-
token = tlist.group_tokens_between(sql.Identifier, prev, token)
265-
token = tlist.token_next_by(t=T.Keyword.Order, idx=tlist.token_index(token) + 1)
269+
tlist.group_tokens_between(sql.Identifier, pidx, tidx)
270+
tidx = pidx
271+
tidx, token = tlist.token_idx_next_by(t=T.Keyword.Order, idx=tidx + 1)
266272

267273

268274
@recurse()
269275
def align_comments(tlist):
270-
token = tlist.token_next_by(i=sql.Comment)
276+
tidx, token = tlist.token_idx_next_by(i=sql.Comment)
271277
while token:
272-
before = tlist.token_prev(tlist.token_index(token))
273-
if isinstance(before, sql.TokenList):
274-
token = tlist.group_tokens_between(sql.TokenList, before, token, extend=True)
275-
token = tlist.token_next_by(i=sql.Comment, idx=tlist.token_index(token) + 1)
278+
pidx, prev = tlist.token_idx_prev(tidx)
279+
if isinstance(prev, sql.TokenList):
280+
tlist.group_tokens_between(sql.TokenList, pidx, tidx, extend=True)
281+
tidx = pidx
282+
tidx, token = tlist.token_idx_next_by(i=sql.Comment, idx=tidx + 1)
276283

277284

278285
def group(stmt):

‎sqlparse/filters/aligned_indent.py‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ def _process_statement(self, tlist):
4646

4747
def _process_parenthesis(self, tlist):
4848
# if this isn't a subquery, don't re-indent
49-
if tlist.token_next_by(m=(T.DML, 'SELECT')):
49+
_, token = tlist.token_idx_next_by(m=(T.DML, 'SELECT'))
50+
if token is not None:
5051
with indent(self):
5152
tlist.insert_after(tlist[0], self.nl('SELECT'))
5253
# process the inside of the parantheses
@@ -66,7 +67,7 @@ def _process_case(self, tlist):
6667
offset_ = len('case ') + len('when ')
6768
cases = tlist.get_cases(skip_ws=True)
6869
# align the end as well
69-
end_token = tlist.token_next_by(m=(T.Keyword, 'END'))
70+
_, end_token = tlist.token_idx_next_by(m=(T.Keyword, 'END'))
7071
cases.append((None, [end_token]))
7172

7273
condition_width = [len(' '.join(map(text_type, cond))) if cond else 0
@@ -87,30 +88,32 @@ def _process_case(self, tlist):
8788

8889
def _next_token(self, tlist, idx=0):
8990
split_words = T.Keyword, self.split_words, True
90-
token = tlist.token_next_by(m=split_words, idx=idx)
91+
tidx, token = tlist.token_idx_next_by(m=split_words, idx=idx)
9192
# treat "BETWEEN x and y" as a single statement
92-
if token and token.value.upper() == 'BETWEEN':
93-
token = self._next_token(tlist, token)
94-
if token and token.value.upper() == 'AND':
95-
token = self._next_token(tlist, token)
96-
return token
93+
if token and token.normalized == 'BETWEEN':
94+
tidx, token = self._next_token(tlist, tidx + 1)
95+
if token and token.normalized == 'AND':
96+
tidx, token = self._next_token(tlist, tidx + 1)
97+
return tidx, token
9798

9899
def _split_kwds(self, tlist):
99-
token = self._next_token(tlist)
100+
tidx, token = self._next_token(tlist)
100101
while token:
101102
# joins are special case. only consider the first word as aligner
102103
if token.match(T.Keyword, self.join_words, regex=True):
103104
token_indent = token.value.split()[0]
104105
else:
105106
token_indent = text_type(token)
106107
tlist.insert_before(token, self.nl(token_indent))
107-
token = self._next_token(tlist, token)
108+
tidx += 1
109+
tidx, token = self._next_token(tlist, tidx + 1)
108110

109111
def _process_default(self, tlist):
110112
self._split_kwds(tlist)
111113
# process any sub-sub statements
112114
for sgroup in tlist.get_sublists():
113-
prev = tlist.token_prev(sgroup)
115+
idx = tlist.token_index(sgroup)
116+
pidx, prev = tlist.token_idx_prev(idx)
114117
# HACK: make "group/order by" work. Longer than max_len.
115118
offset_ = 3 if (prev and prev.match(T.Keyword, 'BY')) else 0
116119
with offset(self, offset_):

‎sqlparse/filters/others.py‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,22 @@ class StripCommentsFilter(object):
1414
def _process(tlist):
1515
def get_next_comment():
1616
# TODO(andi) Comment types should be unified, see related issue38
17-
return tlist.token_next_by(i=sql.Comment, t=T.Comment)
17+
return tlist.token_idx_next_by(i=sql.Comment, t=T.Comment)
1818

19-
token = get_next_comment()
19+
tidx, token = get_next_comment()
2020
while token:
21-
prev = tlist.token_prev(token, skip_ws=False)
22-
next_ = tlist.token_next(token, skip_ws=False)
21+
pidx, prev_ = tlist.token_idx_prev(tidx, skip_ws=False)
22+
nidx, next_ = tlist.token_idx_next(tidx, skip_ws=False)
2323
# Replace by whitespace if prev and next exist and if they're not
2424
# whitespaces. This doesn't apply if prev or next is a paranthesis.
25-
if (prev is None or next_ is None or
26-
prev.is_whitespace() or prev.match(T.Punctuation, '(') or
25+
if (prev_ is None or next_ is None or
26+
prev_.is_whitespace() or prev_.match(T.Punctuation, '(') or
2727
next_.is_whitespace() or next_.match(T.Punctuation, ')')):
2828
tlist.tokens.remove(token)
2929
else:
30-
tidx = tlist.token_index(token)
3130
tlist.tokens[tidx] = sql.Token(T.Whitespace, ' ')
3231

33-
token = get_next_comment()
32+
tidx, token = get_next_comment()
3433

3534
def process(self, stmt):
3635
[self.process(sgroup) for sgroup in stmt.get_sublists()]
@@ -86,20 +85,21 @@ def process(self, stmt, depth=0):
8685
class SpacesAroundOperatorsFilter(object):
8786
@staticmethod
8887
def _process(tlist):
89-
def next_token(idx=0):
90-
return tlist.token_next_by(t=(T.Operator, T.Comparison), idx=idx)
9188

92-
token = next_token()
89+
ttypes = (T.Operator, T.Comparison)
90+
tidx, token = tlist.token_idx_next_by(t=ttypes)
9391
while token:
94-
prev_ = tlist.token_prev(token, skip_ws=False)
95-
if prev_ and prev_.ttype != T.Whitespace:
96-
tlist.insert_before(token, sql.Token(T.Whitespace, ' '))
97-
98-
next_ = tlist.token_next(token, skip_ws=False)
92+
nidx, next_ = tlist.token_idx_next(tidx, skip_ws=False)
9993
if next_ and next_.ttype != T.Whitespace:
100-
tlist.insert_after(token, sql.Token(T.Whitespace, ' '))
94+
tlist.insert_after(tidx, sql.Token(T.Whitespace, ' '))
95+
96+
pidx, prev_ = tlist.token_idx_prev(tidx, skip_ws=False)
97+
if prev_ and prev_.ttype != T.Whitespace:
98+
tlist.insert_before(tidx, sql.Token(T.Whitespace, ' '))
99+
tidx += 1 # has to shift since token inserted before it
101100

102-
token = next_token(idx=token)
101+
# assert tlist.token_index(token) == tidx
102+
tidx, token = tlist.token_idx_next_by(t=ttypes, idx=tidx + 1)
103103

104104
def process(self, stmt):
105105
[self.process(sgroup) for sgroup in stmt.get_sublists()]

0 commit comments

Comments
 (0)