forked from danmar/cppcheck
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdonate-cpu.py
More file actions
executable file
·266 lines (255 loc) · 11.1 KB
/
donate-cpu.py
File metadata and controls
executable file
·266 lines (255 loc) · 11.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
#!/usr/bin/env python3
# Donate CPU
#
# A script a user can run to donate CPU to cppcheck project
#
# Syntax: donate-cpu.py [-jN] [--package=url] [--stop-time=HH:MM] [--work-path=path] [--test] [--bandwidth-limit=limit]
# -jN Use N threads in compilation/analysis. Default is 1.
# --package=url Check a specific package and then stop. Can be useful if you want to reproduce
# some warning/crash/exception/etc..
# --stop-time=HH:MM Stop analysis when time has passed. Default is that you must terminate the script.
# --work-path=path Work folder path. Default path is cppcheck-donate-cpu-workfolder in your home folder.
# --test Connect to a donate-cpu-server that is running locally on port 8001 for testing.
# --bandwidth-limit=limit Limit download rate for packages. Format for limit is the same that wget uses.
# Examples: --bandwidth-limit=250k => max. 250 kilobytes per second
# --bandwidth-limit=2m => max. 2 megabytes per second
# --max-packages=N Process N packages and then exit. A value of 0 means infinitely.
# --no-upload Do not upload anything. Defaults to False.
# --packages Process a list of given packages.
#
# What this script does:
# 1. Check requirements
# 2. Pull & compile Cppcheck
# 3. Select a package
# 4. Download package
# 5. Analyze source code
# 6. Upload results
# 7. Repeat from step 2
#
# Quick start: just run this script without any arguments
import platform
from donate_cpu_lib import *
max_packages = None
package_urls = []
for arg in sys.argv[1:]:
# --stop-time=12:00 => run until ~12:00 and then stop
if arg.startswith('--stop-time='):
stop_time = arg[-5:]
print('Stop time:' + stop_time)
elif arg.startswith('-j'):
if not re.match(r'-j\d+', arg):
print('Argument "{}" is invalid.'.format(arg))
print('"-j" must be followed by a positive number.')
sys.exit(1)
jobs = arg
print('Jobs:' + jobs[2:])
elif arg.startswith('--package='):
pkg = arg[arg.find('=')+1:]
package_urls.append(pkg)
print('Added Package:' + pkg)
elif arg.startswith('--packages='):
pkg_cnt = len(package_urls)
with open(arg[arg.find('=')+1:], 'rt') as f:
for package_url in f:
package_url = package_url.strip()
if not package_url:
continue
package_urls.append(package_url)
print('Added Packages:' + str(len(package_urls) - pkg_cnt))
elif arg.startswith('--work-path='):
work_path = os.path.abspath(arg[arg.find('=')+1:])
print('work_path:' + work_path)
if not os.path.exists(work_path):
print('work path does not exist!')
sys.exit(1)
elif arg == '--test':
server_address = ('localhost', 8001)
elif arg.startswith('--bandwidth-limit='):
bandwidth_limit = arg[arg.find('=')+1:]
elif arg.startswith('--max-packages='):
arg_value = arg[arg.find('=')+1:]
try:
max_packages = int(arg_value)
except ValueError:
max_packages = None
if max_packages < 0:
max_packages = None
if max_packages is None:
print('Error: Max. packages value "{}" is invalid. Must be a positive number or 0.'.format(arg_value))
sys.exit(1)
# 0 means infinitely, no counting needed.
if max_packages == 0:
max_packages = None
elif arg.startswith('--no-upload'):
do_upload = False
elif arg == '--help':
print('Donate CPU to Cppcheck project')
print('')
print('Syntax: donate-cpu.py [-jN] [--stop-time=HH:MM] [--work-path=path]')
print(' -jN Use N threads in compilation/analysis. Default is 1.')
print(' --package=url Check a specific package and then stop. Can be useful if you want to reproduce')
print(' some warning/crash/exception/etc..')
print(' --stop-time=HH:MM Stop analysis when time has passed. Default is that you must terminate the script.')
print(' --work-path=path Work folder path. Default path is ' + work_path)
print(' --bandwidth-limit=limit Limit download rate for packages. Format for limit is the same that wget uses.')
print(' Examples: --bandwidth-limit=250k => max. 250 kilobytes per second')
print(' --bandwidth-limit=2m => max. 2 megabytes per second')
print(' --max-packages=N Process N packages and then exit. A value of 0 means infinitely.')
print(' --no-upload Do not upload anything. Defaults to False.')
print('')
print('Quick start: just run this script without any arguments')
sys.exit(0)
else:
print('Unhandled argument: ' + arg)
sys.exit(1)
if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 4):
print("#" * 80)
print("IMPORTANT")
print("Please run the client with at least Python 3.4, thanks!")
print("#" * 80)
time.sleep(2)
sys.exit(1)
print('Thank you!')
if not check_requirements():
sys.exit(1)
if bandwidth_limit and isinstance(bandwidth_limit, str):
if subprocess.call(['wget', '--limit-rate=' + bandwidth_limit, '-q', '--spider', 'cppcheck1.osuosl.org']) == 2:
print('Error: Bandwidth limit value "' + bandwidth_limit + '" is invalid.')
sys.exit(1)
else:
print('Bandwidth-limit: ' + bandwidth_limit)
if package_urls:
max_packages = len(package_urls)
if max_packages:
print('Maximum number of packages to download and analyze: {}'.format(max_packages))
if not os.path.exists(work_path):
os.mkdir(work_path)
repo_path = os.path.join(work_path, 'repo')
# This is a temporary migration step which should be removed in the future
migrate_repo_path = os.path.join(work_path, 'cppcheck')
packages_processed = 0
print('Get Cppcheck..')
try:
try_retry(clone_cppcheck, fargs=(repo_path, migrate_repo_path))
except:
print('Error: Failed to clone Cppcheck, retry later')
sys.exit(1)
while True:
if max_packages:
if packages_processed >= max_packages:
print('Processed the specified number of {} package(s). Exiting now.'.format(max_packages))
break
print('Processing package {} of the specified {} package(s).'.format(packages_processed + 1, max_packages))
packages_processed += 1
if stop_time:
print('stop_time:' + stop_time + '. Time:' + time.strftime('%H:%M') + '.')
if stop_time < time.strftime('%H:%M'):
print('Stopping. Thank you!')
sys.exit(0)
cppcheck_versions = get_cppcheck_versions(server_address)
if cppcheck_versions is None:
print('Failed to communicate with server, retry later')
sys.exit(1)
if len(cppcheck_versions) == 0:
print('Did not get any cppcheck versions from server, retry later')
sys.exit(1)
for ver in cppcheck_versions:
if ver == 'head':
ver = 'main'
current_cppcheck_dir = os.path.join(work_path, 'tree-'+ver)
try:
print('Fetching Cppcheck-{}..'.format(ver))
try_retry(checkout_cppcheck_version, fargs=(repo_path, ver, current_cppcheck_dir))
except KeyboardInterrupt as e:
# Passthrough for user abort
raise e
except:
print('Failed to update Cppcheck, retry later')
sys.exit(1)
if ver == 'main':
if not compile_cppcheck(current_cppcheck_dir, jobs):
print('Failed to compile Cppcheck-{}, retry later'.format(ver))
sys.exit(1)
else:
if not compile_version(current_cppcheck_dir, jobs):
print('Failed to compile Cppcheck-{}, retry later'.format(ver))
sys.exit(1)
if package_urls:
package = package_urls[packages_processed-1]
else:
package = get_package(server_address)
tgz = download_package(work_path, package, bandwidth_limit)
if tgz is None:
print("No package downloaded")
continue
if not unpack_package(work_path, tgz):
print("No files to process")
continue
crash = False
timeout = False
count = ''
elapsed_time = ''
results_to_diff = []
cppcheck_options = ''
head_info_msg = ''
head_timing_info = ''
old_timing_info = ''
cppcheck_head_info = ''
libraries = get_libraries()
for ver in cppcheck_versions:
tree_path = os.path.join(work_path, 'tree-'+ver)
capture_callstack = False
if ver == 'head':
tree_path = os.path.join(work_path, 'tree-main')
cppcheck_head_info = get_cppcheck_info(tree_path)
capture_callstack = True
c, errout, info, t, cppcheck_options, timing_info = scan_package(work_path, tree_path, jobs, libraries, capture_callstack)
if c < 0:
if c == -101 and 'error: could not find or open any of the paths given.' in errout:
# No sourcefile found (for example only headers present)
count += ' 0'
elif c == RETURN_CODE_TIMEOUT:
# Timeout
count += ' TO!'
timeout = True
else:
crash = True
count += ' Crash!'
else:
count += ' ' + str(c)
elapsed_time += " {:.1f}".format(t)
results_to_diff.append(errout)
if ver == 'head':
head_info_msg = info
head_timing_info = timing_info
else:
old_timing_info = timing_info
output = 'cppcheck-options: ' + cppcheck_options + '\n'
output += 'platform: ' + platform.platform() + '\n'
output += 'python: ' + platform.python_version() + '\n'
output += 'client-version: ' + CLIENT_VERSION + '\n'
output += 'compiler: ' + get_compiler_version() + '\n'
output += 'cppcheck: ' + ' '.join(cppcheck_versions) + '\n'
output += 'head-info: ' + cppcheck_head_info + '\n'
output += 'count:' + count + '\n'
output += 'elapsed-time:' + elapsed_time + '\n'
output += 'head-timing-info:\n' + head_timing_info + '\n'
output += 'old-timing-info:\n' + old_timing_info + '\n'
info_output = output
info_output += 'info messages:\n' + head_info_msg
if 'head' in cppcheck_versions:
output += 'head results:\n' + results_to_diff[cppcheck_versions.index('head')]
if not crash and not timeout:
output += 'diff:\n' + diff_results(cppcheck_versions[0], results_to_diff[0], cppcheck_versions[1], results_to_diff[1]) + '\n'
if package_urls:
print('=========================================================')
print(output)
print('=========================================================')
print(info_output)
print('=========================================================')
if do_upload:
upload_results(package, output, server_address)
upload_info(package, info_output, server_address)
if not max_packages or packages_processed < max_packages:
print('Sleep 5 seconds..')
time.sleep(5)