OILS / test / syscall.py View on Github | oilshell.org

280 lines, 179 significant
1#!/usr/bin/env python2
2"""test/syscall.py
3
4Print a results table.
5
6Input looks like
7
801-dash
901-dash
1001-osh
1101-osh
1201-bash-4.4
13...
14"""
15from __future__ import print_function
16
17import collections
18import optparse
19import os
20import re
21import sys
22
23
24def log(msg, *args):
25 if args:
26 msg = msg % args
27 print(msg, file=sys.stderr)
28
29
30def Cell(i):
31 """Visually show number of processes.
32
33 ^ ^^ ^^^ etc.
34 """
35 s = '^' * i
36 return '%6s' % s
37
38
39# lines look like this:
40#
41# 554 01-osh.1234
42# 553 01-osh.1235
43
44WC_LINE = re.compile(
45 r'''
46\s*
47(\d+) # number of lines
48\s+
49([a-z0-9.-]+) # shell name, could be bash-4.4
50__
51(\d{2}) # case ID
52''', re.VERBOSE)
53
54assert WC_LINE.match(' 68 osh-cpp__01.19610')
55# This is unfortunate
56assert WC_LINE.match(' 68 bash-4.4__01.19610')
57
58
59def WriteHeader(f, shells, more_cols=None):
60 f.write("ID\t",)
61 for sh in shells:
62 # abbreviate
63 if sh.startswith('bash-4'):
64 sh = 'bash-4'
65 elif sh.startswith('bash-5'):
66 sh = 'bash-5'
67 f.write("%6s\t" % sh)
68 if more_cols:
69 for col in more_cols:
70 f.write('%s\t' % col)
71 f.write('Description')
72 f.write("\n")
73
74
75def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs,
76 procs_by_shell):
77 not_minimum = 0
78 more_than_bash = 0
79 fewer_than_bash = 0
80 osh_not_equal_ysh = 0
81
82 minimum = {} # case -> number of procses
83 for case_id in sorted(cases):
84 min_procs = 20
85 for sh in proc_sh:
86 n = num_procs[case_id, sh]
87 min_procs = min(n, min_procs)
88 minimum[case_id] = min_procs
89
90 osh_count = num_procs[case_id, 'osh']
91 ysh_count = num_procs[case_id, 'ysh']
92
93 if osh_count != min_procs:
94 not_minimum += 1
95
96 if ysh_count != osh_count:
97 osh_not_equal_ysh += 1
98
99 bash_count = num_procs[case_id, 'bash-5.2.21']
100 if osh_count > bash_count:
101 more_than_bash += 1
102 if osh_count < bash_count:
103 fewer_than_bash += 1
104
105 f.write('Number of Processes Started, by shell and test case\n')
106 f.write('\n')
107 f.write("Cases where ...\n")
108 f.write(" OSH isn't the minimum: %d\n" % not_minimum)
109 f.write(" OSH starts more than bash 5: %d\n" % more_than_bash)
110 f.write(" OSH starts fewer than bash 5: %d\n\n" % fewer_than_bash)
111 f.write(" YSH not equal to OSH: %d\n\n" % osh_not_equal_ysh)
112 f.write('\n')
113 WriteHeader(f, proc_sh, more_cols=['osh>min', 'ysh!osh'])
114 f.write('\n')
115
116 f.write("TOTAL\t")
117 for sh in proc_sh:
118 f.write('%6d\t' % procs_by_shell[sh])
119 f.write('\n')
120 f.write('\n')
121
122 for case_id in sorted(cases):
123 f.write(case_id + "\t")
124 for sh in proc_sh:
125 n = num_procs[case_id, sh]
126 f.write(Cell(n) + "\t")
127
128 osh_count = num_procs[case_id, 'osh']
129 ysh_count = num_procs[case_id, 'ysh']
130 min_procs = minimum[case_id]
131
132 if osh_count != min_procs:
133 f.write('%d>%d' % (osh_count, min_procs))
134 f.write('\t')
135
136 if ysh_count != osh_count:
137 f.write('%d!=%d' % (ysh_count, osh_count))
138 f.write('\t')
139
140 f.write(code_strs[case_id])
141 f.write("\n")
142
143 return not_minimum, more_than_bash, fewer_than_bash
144
145
146def WriteSyscallReport(f, cases, code_strs, syscall_sh, num_syscalls,
147 syscalls_by_shell):
148 f.write('Number of Syscalls\n\n')
149
150 WriteHeader(f, syscall_sh)
151
152 for case_id in sorted(cases):
153 f.write(case_id + "\t")
154 #min_procs = 20
155 for sh in syscall_sh:
156 n = num_syscalls[case_id, sh]
157 f.write('%6d\t' % n)
158 #min_procs = min(n, min_procs)
159
160 f.write(code_strs[case_id])
161 f.write("\n")
162
163 f.write("TOTAL\t")
164 for sh in syscall_sh:
165 f.write('%6d\t' % syscalls_by_shell[sh])
166 f.write('\n\n')
167
168
169def Options():
170 """Returns an option parser instance."""
171 p = optparse.OptionParser()
172 p.add_option('--suite',
173 dest='suite',
174 default='SUITE',
175 help='Test suite name')
176 p.add_option(
177 '--not-minimum',
178 dest='not_minimum',
179 type=int,
180 default=0,
181 help=
182 "Expected number of cases where OSH doesn't start the minimum number of"
183 "processes")
184 p.add_option(
185 '--more-than-bash',
186 dest='more_than_bash',
187 type=int,
188 default=0,
189 help=
190 'Expected number of cases where OSH starts more processes than bash')
191 return p
192
193
194def main(argv):
195 o = Options()
196 opts, argv = o.parse_args(argv[1:])
197
198 cases_path = argv[0]
199 out_dir = argv[1]
200
201 code_strs = {}
202 with open(cases_path) as f:
203 for line in f:
204 case_id, code_str = line.split(None, 1) # whitespace
205 code_strs[case_id] = code_str
206
207 cases = set()
208 shells = set()
209
210 num_procs = collections.defaultdict(int)
211 procs_by_shell = collections.defaultdict(int)
212
213 num_syscalls = collections.defaultdict(int)
214 syscalls_by_shell = collections.defaultdict(int)
215
216 #
217 # Summarize Data
218 #
219
220 for line in sys.stdin:
221 m = WC_LINE.match(line)
222 if not m:
223 raise RuntimeError('Invalid line %r' % line)
224 num_sys, sh, case = m.groups()
225 num_sys = int(num_sys)
226
227 cases.add(case)
228 shells.add(sh)
229
230 num_procs[case, sh] += 1
231 num_syscalls[case, sh] += num_sys
232
233 procs_by_shell[sh] += 1
234 syscalls_by_shell[sh] += num_sys
235
236 # Orders columns by how good the results are, then shell name.
237 proc_sh = sorted(procs_by_shell, key=lambda sh: (procs_by_shell[sh], sh))
238 syscall_sh = sorted(syscalls_by_shell,
239 key=lambda sh: (syscalls_by_shell[sh], sh))
240
241 #
242 # Print Tables
243 #
244
245 out_path = os.path.join(out_dir, 'processes.%s.txt' % opts.suite)
246 with open(out_path, 'w') as f:
247 not_minimum, more_than_bash, fewer_than_bash = WriteProcessReport(
248 f, cases, code_strs, proc_sh, num_procs, procs_by_shell)
249 log('Wrote %s', out_path)
250
251 #
252 # Print Table of Syscall Counts
253 #
254
255 out_path = os.path.join(out_dir, 'syscalls.%s.txt' % opts.suite)
256 with open(out_path, 'w') as f:
257 WriteSyscallReport(f, cases, code_strs, syscall_sh, num_syscalls,
258 syscalls_by_shell)
259 log('Wrote %s', out_path)
260
261 ok = True
262 if more_than_bash != opts.more_than_bash:
263 log('Expected %d more than bash, got %d', opts.more_than_bash,
264 more_than_bash)
265 ok = False
266
267 if not_minimum != opts.not_minimum:
268 log('Expected %d that are not minimal, got %d', opts.not_minimum,
269 not_minimum)
270 ok = False
271
272 return 0 if ok else 1
273
274
275if __name__ == '__main__':
276 try:
277 sys.exit(main(sys.argv))
278 except RuntimeError as e:
279 print('FATAL: %s' % e, file=sys.stderr)
280 sys.exit(1)