OILS / core / process_test.py View on Github | oils.pub

720 lines, 402 significant
1#!/usr/bin/env python2
2"""process_test.py: Tests for process.py."""
3
4import os
5import unittest
6
7from _devbuild.gen.id_kind_asdl import Id
8from _devbuild.gen.runtime_asdl import (RedirValue, redirect_arg, cmd_value,
9 trace)
10from _devbuild.gen.syntax_asdl import loc, redir_loc
11from asdl import runtime
12from builtin import read_osh
13from builtin import process_osh
14from builtin import trap_osh
15from core import dev
16from core import process # module under test
17from core import pyos
18from core import sh_init
19from core import state
20from core import test_lib
21from core import util
22from display import ui
23from frontend import flag_def # side effect: flags are defined, for wait builtin
24from mycpp import iolib
25from mycpp import mylib
26from mycpp.mylib import log
27from osh import cmd_parse_test
28
29import posix_ as posix
30
31_ = flag_def
32
33Process = process.Process
34ExternalThunk = process.ExternalThunk
35assertParsePipeline = cmd_parse_test.assertParsePipeline
36
37
38def Banner(msg):
39 print('-' * 60)
40 print(msg)
41
42
43class _FakeJobControl(object):
44
45 def __init__(self, enabled):
46 self.enabled = enabled
47
48 def Enabled(self):
49 return self.enabled
50
51
52class _FakeCommandEvaluator(object):
53
54 def RunPendingTraps(self):
55 pass
56
57
58def _SetupTest(self):
59 self.arena = test_lib.MakeArena('process_test.py')
60
61 self.mem = test_lib.MakeMem(self.arena)
62 parse_opts, exec_opts, mutable_opts = state.MakeOpts(self.mem, {}, None)
63 self.mem.exec_opts = exec_opts
64 self.exec_opts = exec_opts
65
66 #state.InitMem(mem, {}, '0.1')
67 sh_init.InitDefaultVars(self.mem, [])
68
69 self.job_control = process.JobControl()
70 self.job_list = process.JobList()
71
72 signal_safe = iolib.InitSignalSafe()
73 self.trap_state = trap_osh.TrapState(signal_safe)
74
75 fd_state = None
76 self.multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
77 self.tracer = dev.Tracer(None, exec_opts, mutable_opts, self.mem,
78 mylib.Stderr(), self.multi_trace)
79 self.waiter = process.Waiter(self.job_list, exec_opts, self.trap_state,
80 self.tracer)
81 self.errfmt = ui.ErrorFormatter()
82 self.fd_state = process.FdState(self.errfmt, self.job_control,
83 self.job_list, None, self.tracer, None,
84 exec_opts)
85 self.ext_prog = process.ExternalProgram('', self.fd_state, self.errfmt,
86 util.NullDebugFile())
87 self.cmd_ev = test_lib.InitCommandEvaluator(arena=self.arena,
88 ext_prog=self.ext_prog)
89
90
91def _SetupWait(self):
92 self.wait_builtin = process_osh.Wait(self.waiter, self.job_list, self.mem,
93 self.tracer, self.errfmt)
94
95
96def _MakeThunk(argv, ext_prog):
97 arg_vec = cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None,
98 None)
99 argv0_path = None
100 for path_entry in ['/bin', '/usr/bin']:
101 full_path = os.path.join(path_entry, argv[0])
102 if os.path.exists(full_path):
103 argv0_path = full_path
104 break
105 if not argv0_path:
106 argv0_path = argv[0] # fallback that tests failure case
107 return ExternalThunk(ext_prog, argv0_path, arg_vec, {})
108
109
110def _CommandNode(code_str, arena):
111 c_parser = test_lib.InitCommandParser(code_str, arena=arena)
112 return c_parser.ParseLogicalLine()
113
114
115class _Common(unittest.TestCase):
116 """Common functionality for tests below."""
117
118 def _ExtProc(self, argv):
119 thunk = _MakeThunk(argv, self.ext_prog)
120 return Process(thunk, self.job_control, self.job_list, self.tracer)
121
122 def _MakeForegroundPipeline(self, argv_list, last_str=''):
123 """
124 Foreground pipelines have self.last_thunk, from pi.AddLast().
125 Background pipelines don't
126 """
127 assert len(last_str), last_str # required
128
129 pi = process.Pipeline(False, self.job_control, self.job_list,
130 self.tracer)
131 for argv in argv_list:
132 pi.Add(self._ExtProc(argv))
133 node = _CommandNode(last_str, self.arena)
134 pi.AddLast((self.cmd_ev, node))
135 return pi
136
137 def _MakeProcess(self, node):
138 thunk = process.SubProgramThunk(self.cmd_ev, node, self.trap_state,
139 self.multi_trace, True,
140 self.exec_opts.errtrace())
141 p = process.Process(thunk, self.job_control, self.job_list,
142 self.tracer)
143 return p
144
145 def _MakeBackgroundPipeline(self, code_str):
146 node = assertParsePipeline(self, code_str)
147
148 pi = process.Pipeline(self.exec_opts.sigpipe_status_ok(),
149 self.job_control, self.job_list, self.tracer)
150 for child in node.children:
151 p = self._MakeProcess(child)
152 p.Init_ParentPipeline(pi)
153 pi.Add(p)
154 return pi
155
156
157class ProcessTest(_Common):
158
159 def setUp(self):
160 _SetupTest(self)
161
162 def testStdinRedirect(self):
163 PATH = '_tmp/one-two.txt'
164 # Write two lines
165 with open(PATH, 'w') as f:
166 f.write('one\ntwo\n')
167
168 # Should get the first line twice, because Pop() closes it!
169
170 r = RedirValue(Id.Redir_Less, runtime.NO_SPID, redir_loc.Fd(0),
171 redirect_arg.Path(PATH))
172
173 cmd_ev = _FakeCommandEvaluator()
174
175 err_out = []
176 self.fd_state.Push([r], err_out)
177 line1, _ = read_osh._ReadPortion(pyos.NEWLINE_CH, -1, cmd_ev)
178 self.fd_state.Pop(err_out)
179
180 self.fd_state.Push([r], err_out)
181 line2, _ = read_osh._ReadPortion(pyos.NEWLINE_CH, -1, cmd_ev)
182 self.fd_state.Pop(err_out)
183
184 # sys.stdin.readline() would erroneously return 'two' because of buffering.
185 self.assertEqual('one', line1)
186 self.assertEqual('one', line2)
187
188 def testProcess(self):
189 # 3 fds. Does Python open it? Shell seems to have it too. Maybe it
190 # inherits from the shell.
191 print('FDS BEFORE', os.listdir('/dev/fd'))
192
193 Banner('date')
194 argv = ['date']
195 p = self._ExtProc(argv)
196 why = trace.External(argv)
197 status = p.RunProcess(self.waiter, why)
198 log('date returned %d', status)
199 self.assertEqual(0, status)
200
201 Banner('does-not-exist')
202 p = self._ExtProc(['does-not-exist'])
203 print(p.RunProcess(self.waiter, why))
204
205 # 12 file descriptors open!
206 print('FDS AFTER', os.listdir('/dev/fd'))
207
208 def testPipeline(self):
209 print('BEFORE', os.listdir('/dev/fd'))
210
211 p = self._MakeForegroundPipeline(
212 [['ls'], ['cut', '-d', '.', '-f', '2'], ['sort']],
213 last_str='uniq -c')
214
215 p.StartPipeline(self.waiter)
216 pipe_status = p.RunLastPart(self.waiter, self.fd_state)
217 log('pipe_status: %s', pipe_status)
218
219 print('AFTER', os.listdir('/dev/fd'))
220
221 def testPipeline2(self):
222 Banner('ls | cut -d . -f 1 | head')
223 p = self._MakeForegroundPipeline(
224 [['ls'], ['cut', '-d', '.', '-f', '1']], last_str='head')
225
226 p.StartPipeline(self.waiter)
227 print(p.RunLastPart(self.waiter, self.fd_state))
228
229 def testPipeline3(self):
230 # Simulating subshell for each command
231 node1 = _CommandNode('ls', self.arena)
232 node2 = _CommandNode('head', self.arena)
233 node3 = _CommandNode('sort --reverse', self.arena)
234
235 thunk1 = process.SubProgramThunk(self.cmd_ev, node1, self.trap_state,
236 self.multi_trace, True, False)
237 thunk2 = process.SubProgramThunk(self.cmd_ev, node2, self.trap_state,
238 self.multi_trace, True, False)
239 thunk3 = process.SubProgramThunk(self.cmd_ev, node3, self.trap_state,
240 self.multi_trace, True, False)
241
242 p = process.Pipeline(False, self.job_control, self.job_list,
243 self.tracer)
244 p.Add(Process(thunk1, self.job_control, self.job_list, self.tracer))
245 p.Add(Process(thunk2, self.job_control, self.job_list, self.tracer))
246 p.Add(Process(thunk3, self.job_control, self.job_list, self.tracer))
247
248 last_thunk = (self.cmd_ev, _CommandNode('cat', self.arena))
249 p.AddLast(last_thunk)
250
251 p.StartPipeline(self.waiter)
252 print(p.RunLastPart(self.waiter, self.fd_state))
253
254 # TODO: Combine pipelines for other things:
255
256 # echo foo 1>&2 | tee stdout.txt
257 #
258 # foo=$(ls | head)
259 #
260 # foo=$(<<EOF ls | head)
261 # stdin
262 # EOF
263 #
264 # ls | head &
265
266 # Or technically we could fork the whole interpreter for foo|bar|baz and
267 # capture stdout of that interpreter.
268
269 def _MakePipeline2(self, jc):
270 pi = process.Pipeline(False, jc, self.job_list, self.tracer)
271
272 node1 = _CommandNode('/bin/echo testpipeline', self.arena)
273 node2 = _CommandNode('cat', self.arena)
274
275 thunk1 = process.SubProgramThunk(self.cmd_ev, node1, self.trap_state,
276 self.multi_trace, True, False)
277 thunk2 = process.SubProgramThunk(self.cmd_ev, node2, self.trap_state,
278 self.multi_trace, True, False)
279
280 pi.Add(Process(thunk1, jc, self.job_list, self.tracer))
281 pi.Add(Process(thunk2, jc, self.job_list, self.tracer))
282
283 return pi
284
285 def testPipelinePgidField(self):
286 jc = _FakeJobControl(False)
287
288 pi = self._MakePipeline2(jc)
289 self.assertEqual(process.INVALID_PGID, pi.ProcessGroupId())
290
291 pi.StartPipeline(self.waiter)
292 # No pgid
293 self.assertEqual(process.INVALID_PGID, pi.ProcessGroupId())
294
295 jc = _FakeJobControl(True)
296
297 pi = self._MakePipeline2(jc)
298 self.assertEqual(process.INVALID_PGID, pi.ProcessGroupId())
299
300 pi.StartPipeline(self.waiter)
301 # first process is the process group leader
302 self.assertEqual(pi.pids[0], pi.ProcessGroupId())
303
304 def testOpen(self):
305 # Disabled because mycpp translation can't handle it. We do this at a
306 # higher layer.
307 return
308
309 # This function used to raise BOTH OSError and IOError because Python 2 is
310 # inconsistent.
311 # We follow Python 3 in preferring OSError.
312 # https://stackoverflow.com/questions/29347790/difference-between-ioerror-and-oserror
313 self.assertRaises(OSError, self.fd_state.Open, '_nonexistent_')
314 self.assertRaises(OSError, self.fd_state.Open, 'metrics/')
315
316
317class JobListTest(_Common):
318 """
319 Test invariant that the 'wait' builtin removes the (pid -> status)
320 mappings (NOT the Waiter)
321
322 There are 4 styles of invoking it:
323
324 wait # for all
325 wait -n # for next
326 wait $pid1 $pid2 # for specific jobs -- problem: are pipelines included?
327 wait %j1 %j2 # job specs -- jobs are either pielines or processes
328
329 Bonus:
330
331 jobs -l can show exit status
332 """
333
334 def setUp(self):
335 _SetupTest(self)
336 _SetupWait(self)
337
338 def _RunBackgroundJob(self, argv):
339 p = self._ExtProc(argv)
340
341 # Similar to Executor::RunBackgroundJob()
342 p.SetBackground()
343 pid = p.StartProcess(trace.Fork)
344
345 #self.mem.last_bg_pid = pid # for $!
346
347 job_id = self.job_list.RegisterJob(p) # show in 'jobs' list
348 return pid, job_id
349
350 def _StartProcesses(self, n):
351 pids = []
352 job_ids = []
353
354 assert n < 10, n
355 for i in xrange(1, n + 1):
356 j = 10 - i # count down
357 argv = ['sh', '-c', 'sleep 0.0%d; echo i=%d; exit %d' % (j, j, j)]
358 pid, job_id = self._RunBackgroundJob(argv)
359 pids.append(pid)
360 job_ids.append(job_id)
361
362 log('pids %s', pids)
363 log('job_ids %s', job_ids)
364
365 return pids, job_ids
366
367 def assertJobListLength(self, length):
368 self.assertEqual(length, len(self.job_list.child_procs))
369 self.assertEqual(length, len(self.job_list.jobs))
370 self.assertEqual(length, len(self.job_list.pid_to_job))
371
372 def testWaitAll(self):
373 """ wait """
374 # Jobs list starts out empty
375 self.assertJobListLength(0)
376
377 # Fork 2 processes with &
378 pids, job_ids = self._StartProcesses(2)
379
380 # Now we have 2 jobs
381 self.assertJobListLength(2)
382
383 # Invoke the 'wait' builtin
384
385 cmd_val = test_lib.MakeBuiltinArgv(['wait'])
386 status = self.wait_builtin.Run(cmd_val)
387 self.assertEqual(0, status)
388
389 # Jobs list is now empty
390 self.assertJobListLength(0)
391
392 def testWaitNext(self):
393 """ wait -n """
394 # Jobs list starts out empty
395 self.assertJobListLength(0)
396
397 # Fork 2 processes with &
398 pids, job_ids = self._StartProcesses(2)
399
400 # Now we have 2 jobs
401 self.assertJobListLength(2)
402
403 ### 'wait -n'
404 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
405 status = self.wait_builtin.Run(cmd_val)
406 self.assertEqual(8, status)
407
408 # Jobs list now has 1 fewer job
409 self.assertJobListLength(1)
410
411 ### 'wait -n' again
412 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
413 status = self.wait_builtin.Run(cmd_val)
414 self.assertEqual(9, status)
415
416 # Now zero
417 self.assertJobListLength(0)
418
419 ### 'wait -n' again
420 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
421 status = self.wait_builtin.Run(cmd_val)
422 self.assertEqual(127, status)
423
424 # Still zero
425 self.assertJobListLength(0)
426
427 def testWaitPid(self):
428 """ wait $pid2 """
429 # Jobs list starts out empty
430 self.assertJobListLength(0)
431
432 # Fork 3 processes with &
433 pids, job_ids = self._StartProcesses(3)
434
435 # Now we have 3 jobs
436 self.assertJobListLength(3)
437
438 # wait $pid2
439 cmd_val = test_lib.MakeBuiltinArgv(['wait', str(pids[1])])
440 status = self.wait_builtin.Run(cmd_val)
441 self.assertEqual(8, status)
442
443 # Jobs list now has 1 fewer job
444 self.assertJobListLength(2)
445
446 # wait $pid3
447 cmd_val = test_lib.MakeBuiltinArgv(['wait', str(pids[2])])
448 status = self.wait_builtin.Run(cmd_val)
449 self.assertEqual(7, status)
450
451 self.assertJobListLength(1)
452
453 # wait $pid1
454 cmd_val = test_lib.MakeBuiltinArgv(['wait', str(pids[0])])
455 status = self.wait_builtin.Run(cmd_val)
456 self.assertEqual(9, status)
457
458 self.assertJobListLength(0)
459
460 def testWaitJob(self):
461 """ wait %j2 """
462
463 # Jobs list starts out empty
464 self.assertJobListLength(0)
465
466 # Fork 3 processes with &
467 pids, job_ids = self._StartProcesses(3)
468
469 # Now we have 3 jobs
470 self.assertJobListLength(3)
471
472 # wait %j2
473 cmd_val = test_lib.MakeBuiltinArgv(['wait', '%' + str(job_ids[1])])
474 status = self.wait_builtin.Run(cmd_val)
475 self.assertEqual(8, status)
476
477 self.assertJobListLength(2)
478
479 # wait %j3
480 cmd_val = test_lib.MakeBuiltinArgv(['wait', '%' + str(job_ids[2])])
481
482 status = self.wait_builtin.Run(cmd_val)
483 self.assertEqual(7, status)
484
485 self.assertJobListLength(1)
486
487 # wait %j1
488 cmd_val = test_lib.MakeBuiltinArgv(['wait', '%' + str(job_ids[0])])
489 status = self.wait_builtin.Run(cmd_val)
490 self.assertEqual(9, status)
491
492 self.assertJobListLength(0)
493
494 def testForegroundProcessCleansUpChildProcessDict(self):
495 self.assertJobListLength(0)
496
497 argv = ['sleep', '0.01']
498 p = self._ExtProc(argv)
499 why = trace.External(argv)
500 p.RunProcess(self.waiter, why)
501
502 self.assertJobListLength(0)
503
504 def testGrandchildOutlivesChild(self):
505 """ The new parent is the init process """
506
507 # Jobs list starts out empty
508 self.assertEqual(0, len(self.job_list.child_procs))
509
510 # the sleep process should outlive the sh process
511 argv = ['sh', '-c', 'sleep 0.1 & exit 99']
512 pid, job_id = self._RunBackgroundJob(argv)
513
514 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
515 status = self.wait_builtin.Run(cmd_val)
516 log('status = %d', status)
517 self.assertEqual(99, status)
518
519 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
520 status = self.wait_builtin.Run(cmd_val)
521 log('status = %d', status)
522 self.assertEqual(127, status)
523
524 # More tests:
525 #
526 # wait $pipeline_pid - with pipeline leader, and other PID
527 # wait %pipeline_job
528 # wait -n on pipeline? Does it return PIPESTATUS?
529 # wait with pipeline - should be OK
530 #
531 # Stopped jobs: does it print something interactively?
532
533
534class PipelineJobListTest(_Common):
535 """
536 Like the JobListTest above, but starts pipelines instead of individual
537 processes.
538 """
539
540 def setUp(self):
541 _SetupTest(self)
542 _SetupWait(self)
543
544 def _RunBackgroundPipeline(self, code_str):
545 # Like Executor::RunBackgroundJob()
546 pi = self._MakeBackgroundPipeline(code_str)
547 pi.StartPipeline(self.waiter)
548 pi.SetBackground()
549 #self.mem.last_bg_pid = pid # for $!
550 job_id = self.job_list.RegisterJob(pi) # show in 'jobs' list
551 return pi, job_id
552
553 def _StartPipelines(self, n):
554 pipelines = []
555 job_ids = []
556
557 assert n < 10, n
558 for i in xrange(1, n + 1):
559 j = 10 - i # count down
560 code_str = 'sleep 0.0%d | cat | (exit %d)' % (j, j)
561 #code_str = 'sleep 0.0%d | exit %d | cat' % (j, j)
562 pi, job_id = self._RunBackgroundPipeline(code_str)
563 pipelines.append(pi)
564 job_ids.append(job_id)
565
566 log('pipelines %s', pipelines)
567 log('job_ids %s', job_ids)
568
569 return pipelines, job_ids
570
571 def assertJobListLength(self, length):
572 # 3 processes per pipeline in this test
573 self.assertEqual(length * 3, len(self.job_list.child_procs))
574 self.assertEqual(length, len(self.job_list.jobs))
575 self.assertEqual(length, len(self.job_list.pid_to_job))
576
577 def testWaitAll(self):
578 """ wait """
579 # Jobs list starts out empty
580 self.assertJobListLength(0)
581
582 # Fork 2 processes with &
583 pids, job_ids = self._StartPipelines(2)
584
585 # Now we have 2 jobs
586 self.assertJobListLength(2)
587
588 # Invoke the 'wait' builtin
589
590 cmd_val = test_lib.MakeBuiltinArgv(['wait'])
591 status = self.wait_builtin.Run(cmd_val)
592 self.assertEqual(0, status)
593
594 return
595 # Jobs list is now empty
596 self.assertJobListLength(0)
597
598 def testWaitNext(self):
599 """ wait -n """
600 # Jobs list starts out empty
601 self.assertJobListLength(0)
602
603 # Fork 2 pipelines with &
604 pids, job_ids = self._StartPipelines(2)
605
606 # Now we have 2 jobs
607 self.assertJobListLength(2)
608
609 ### 'wait -n'
610 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
611 status = self.wait_builtin.Run(cmd_val)
612 return
613 self.assertEqual(8, status)
614
615 # Jobs list now has 1 fewer job
616 self.assertJobListLength(1)
617
618 ### 'wait -n' again
619 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
620 status = self.wait_builtin.Run(cmd_val)
621 self.assertEqual(9, status)
622
623 # Now zero
624 self.assertJobListLength(0)
625
626 ### 'wait -n' again
627 cmd_val = test_lib.MakeBuiltinArgv(['wait', '-n'])
628 status = self.wait_builtin.Run(cmd_val)
629 self.assertEqual(127, status)
630
631 # Still zero
632 self.assertJobListLength(0)
633
634 def testWaitPid(self):
635 """ wait $pid2 """
636 # Jobs list starts out empty
637 self.assertJobListLength(0)
638
639 # Fork 3 processes with &
640 pids, job_ids = self._StartPipelines(3)
641
642 # Now we have 3 jobs
643 self.assertJobListLength(3)
644
645 # wait $pid2
646 cmd_val = test_lib.MakeBuiltinArgv(['wait', str(pids[1])])
647 return
648 status = self.wait_builtin.Run(cmd_val)
649 self.assertEqual(8, status)
650
651 # Jobs list now has 1 fewer job
652 self.assertJobListLength(2)
653
654 # wait $pid3
655 cmd_val = test_lib.MakeBuiltinArgv(['wait', str(pids[2])])
656 status = self.wait_builtin.Run(cmd_val)
657 self.assertEqual(7, status)
658
659 self.assertJobListLength(1)
660
661 # wait $pid1
662 cmd_val = test_lib.MakeBuiltinArgv(['wait', str(pids[0])])
663 status = self.wait_builtin.Run(cmd_val)
664 self.assertEqual(9, status)
665
666 self.assertJobListLength(0)
667
668 def testWaitJob(self):
669 """ wait %j2 """
670
671 # Jobs list starts out empty
672 self.assertJobListLength(0)
673
674 # Fork 3 processes with &
675 pids, job_ids = self._StartPipelines(3)
676
677 # Now we have 3 jobs
678 self.assertJobListLength(3)
679
680 # wait %j2
681 cmd_val = test_lib.MakeBuiltinArgv(['wait', '%' + str(job_ids[1])])
682 return
683 status = self.wait_builtin.Run(cmd_val)
684 self.assertEqual(8, status)
685
686 self.assertJobListLength(2)
687
688 # wait %j3
689 cmd_val = test_lib.MakeBuiltinArgv(['wait', '%' + str(job_ids[2])])
690
691 status = self.wait_builtin.Run(cmd_val)
692 self.assertEqual(7, status)
693
694 self.assertJobListLength(1)
695
696 # wait %j1
697 cmd_val = test_lib.MakeBuiltinArgv(['wait', '%' + str(job_ids[0])])
698 status = self.wait_builtin.Run(cmd_val)
699 self.assertEqual(9, status)
700
701 self.assertJobListLength(0)
702
703 def testForegroundPipelineCleansUpChildProcessDict(self):
704 self.assertJobListLength(0)
705
706 # TODO
707 return
708
709 argv = ['sleep', '0.01']
710 p = self._ExtProc(argv)
711 why = trace.External(argv)
712 p.RunProcess(self.waiter, why)
713
714 self.assertJobListLength(0)
715
716
717if __name__ == '__main__':
718 unittest.main()
719
720# vim: sw=4