This repository was archived by the owner on Sep 12, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 51
Expand file tree
/
Copy pathplotStuckness.py
More file actions
executable file
·168 lines (148 loc) · 4.99 KB
/
Copy pathplotStuckness.py
File metadata and controls
executable file
·168 lines (148 loc) · 4.99 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
#!/usr/bin/env python
import sys,urllib,urllib2,re,time,os
try:
import json
except ImportError:
import simplejson as json
import optparse
import httplib
import datetime
import time
import zlib
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
from pprint import pprint
afs_base ='/afs/cern.ch/user/j/jbadillo/www/'
live_status = ['assigned','acquired','running-open','running-closed','completed']
ignoreCampaign = set(['CMSSW_7_1_0','CMSSW_7_2_0_pre4','TEST'])
def drawPlot(s, t, r, filename):
"""
draw the plot
"""
avg = t/r
threshold = 1.5*avg
#f.write('avg = %s<br>'%avg)
#f.write('threshold = %s<br>'%(threshold))
h = ['request','status','priority','relativeprio','dayssame','assignedon','stuckness']
c = 'rgbcmgyko'
i = 0
plt.ylabel('Relative Priority')
plt.xlabel('Days old')
handles = []
for rtype, reqs in s.items():
X = np.array([r['requestdays'] for r in reqs])
Y = np.array([r['relativeprio'] for r in reqs])
L = np.array([r['dayssame'] for r in reqs])
#add some random noise to see quantity
X += 0.3*(30)*np.random.random(X.shape)
Y += 0.3*np.random.random(Y.shape)
han = plt.scatter(X,Y, c=c[i], marker='o', s=10, lw = 0)
#add lines to represent dimension
for j in range(X.size):
plt.plot( [ X[j]-L[j], X[j]], [Y[j], Y[j]], '%s-'%c[i] )
handles.append(han)
i += 1
plt.legend(handles, s.keys(), loc='lower right', prop={'size':8}, scatterpoints = 1)
plt.savefig(filename)
#f.write(foot%time.strftime("%c"))
def getpriorities(s,campaign,zone,status):
"""
Get the different priority values, in decreasing order
filtered by campaign, zone and status
"""
p = set()
for r in s:
if r['priority'] not in p:
if (not campaign or campaign == getcampaign(r) ) and (not zone or zone == r['zone']) and r['status'] in status:
p.add(r['priority'])
p = sorted(p)
p.reverse()
return p
def main():
#read json files
now = datetime.datetime.now()
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat('%s/data.json' % afs_base)
datajsonmtime = "%s" % time.ctime(mtime)
d=open('%s/data.json' % afs_base).read()
s = json.loads(d)
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat('%s/pledged.json' % afs_base)
pledgedjsonmtime = "%s" % time.ctime(mtime)
d=open('%s/pledged.json' % afs_base).read()
pledged = json.loads(d)
totalpledged = 0
#calculate total pledge
for i in pledged.keys():
totalpledged = totalpledged + pledged[i]
s2 = []
#filter out test stuff
for r in s:
if r['status'] not in live_status:
continue
name = r['requestname']
status = r['status']
campaign = r['campaign']
rtype = r['type']
#ignore anything that says "Test"
if 'test' in name.lower() or 'test' in campaign.lower():
continue
#ignore anything that says "backfill"
if 'backfill' in name.lower() or 'backfill' in campaign.lower():
continue
#ignore resubmissions:
if rtype == 'Resubmission':
continue
#ignore dave mason's tests and stefan's tests
if 'dmason' in name or 'piperov' in name:
continue
#ignore relvals and test
if 'dmason' in name or 'piperov' in name:
continue
#ignore relvals
if 'RVCMSSW' in name:
continue
s2.append(r)
s = s2
urgent_requests = []
stuck_days = 5
old_days = 15
oldest = {}
highest_prio = max( getpriorities(s,None,None,live_status))
#for status in ['assigned','acquired','running-open','running-closed','completed']:
# for priority in getpriorities(s,'','',[status]):
totaldays = 0
reqs = 0
s2 = {}
#arrange them by type
for r in s:
name = r['requestname']
status = r['status']
campaign = r['campaign']
rtype = r['type']
days = []
for ods in r['outputdatasetinfo']:
if 'lastmodts' not in ods or not ods['lastmodts']:
days.append(0)
else:
lastmodts = datetime.datetime.fromtimestamp(ods['lastmodts'])
delta = now - lastmodts
days.append(delta.days + delta.seconds/3600.0/24.0)
dayssame = min(days) if days else 0
#TODO calculate stuckness
relativeprio = float(r['priority'])/highest_prio
stuckness = relativeprio*dayssame
r['relativeprio'] = relativeprio
r['stuckness'] = stuckness
r['dayssame'] = dayssame
if dayssame:
totaldays += dayssame
reqs += 1
if rtype not in s2:
s2[rtype] = []
s2[rtype].append(r)
s = s2
drawPlot(s, totaldays, reqs, './www/stuckness.png')
sys.exit(0)
if __name__ == "__main__":
main()