tests: cleanup background processes upon interruption

Reap background processes so that:
- Stray processes aren't left on the system
- Files aren't held open causing deletion issues on NFS
- Partitions used to run the tests from can be unmounted

* tests/tail-2/F-vs-missing.sh: Add the `kill && wait` of the
background $pid(s) to cleanup_().
* tests/tail-2/F-vs-rename.sh: Likewise.
* tests/tail-2/f-vs-rename.sh: Likewise.
* tests/tail-2/append-only.sh: Likewise.
* tests/tail-2/assert-2.sh: Likewise.
* tests/tail-2/assert.sh: Likewise.
* tests/tail-2/flush-initial.sh: Likewise.
* tests/tail-2/inotify-hash-abuse.sh: Likewise.
* tests/tail-2/inotify-hash-abuse2.sh: Likewise.
* tests/tail-2/inotify-race.sh: Likewise.
* tests/tail-2/inotify-rotate-resources.sh: Likewise.
* tests/tail-2/inotify-rotate.sh: Likewise.
* tests/tail-2/pid.sh: Likewise.
* tests/tail-2/pipe-f2.sh: Likewise.
* tests/tail-2/retry.sh: Likewise.
* tests/tail-2/symlink.sh: Likewise.
* tests/tail-2/tail-n0f.sh: Likewise.
* tests/tail-2/wait.sh: Likewise.
* tests/cp/existing-perm-race.sh: Likewise.
* tests/cp/file-perm-race.sh: Likewise.
* tests/cp/parent-perm-race.sh: Likewise.
* tests/cp/sparse-to-pipe.sh: Likewise.
* tests/dd/stats.sh: Likewise.
* tests/du/move-dir-while-traversing.sh: Likewise.
* tests/misc/cat-buf.sh: Likewise.
* tests/misc/help-version.sh: Likewise.
* tests/misc/printf-surprise.sh: Likewise.
* tests/misc/sort-compress-proc.sh: Likewise.
* tests/misc/sort-spinlock-abuse.sh: Likewise.
* tests/misc/stdbuf.sh: Likewise.
* tests/misc/tac-continue.sh: Likewise.
* tests/misc/timeout-group.sh: Likewise.
* tests/mv/i-3.sh: Likewise.
* tests/rm/dangling-symlink.sh: Likewise.
* tests/rm/isatty.sh: Likewise.
* cfg.mk (sc_prohibit_test_background_without_cleanup_):
A new syntax-check to ensure cleanup_() is defined
when background tasks are created in a test.

18 files changed, 95 insertions, 59 deletions

diff --git a/tests/tail-2/F-vs-missing.sh b/tests/tail-2/F-vs-missing.sh
index 6622f7e42..be20ee3b8 100755
--- a/tests/tail-2/F-vs-missing.sh
+++ b/tests/tail-2/F-vs-missing.sh
@@ -28,6 +28,9 @@ check_tail_output()
     { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -49,9 +52,7 @@ for mode in '' '---disable-inotify'; do
   tail_re='has appeared' retry_delay_ check_tail_output .1 7 ||
     { echo "$0: file: unexpected delay?"; cat out; fail=1; }
 
-  kill $pid
-
-  wait $pid
+  cleanup_
 done
 
 
diff --git a/tests/tail-2/F-vs-rename.sh b/tests/tail-2/F-vs-rename.sh
index e087ec2e8..ee61a548b 100755
--- a/tests/tail-2/F-vs-rename.sh
+++ b/tests/tail-2/F-vs-rename.sh
@@ -28,6 +28,9 @@ check_tail_output()
     { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -74,9 +77,7 @@ for mode in '' '---disable-inotify'; do
   retry_delay_ tail_f_vs_rename_3 .1 7 ||
     { echo "$0: a: unexpected delay?"; cat out; fail=1; }
 
-  kill $pid
-
-  wait $pid
+  cleanup_
 done
 
 Exit $fail
diff --git a/tests/tail-2/append-only.sh b/tests/tail-2/append-only.sh
index 9da10ae9c..2ea5df1fe 100755
--- a/tests/tail-2/append-only.sh
+++ b/tests/tail-2/append-only.sh
@@ -21,6 +21,9 @@
 print_ver_ tail
 require_root_
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 chattr_a_works=1
 touch f
 chattr +a f 2>/dev/null || chattr_a_works=0
@@ -35,6 +38,7 @@ fi
 for mode in '' '---disable-inotify'; do
   sleep 1 & pid=$!
   tail --pid=$pid -f $mode f || fail=1
+  cleanup_
 done
 
 chattr -a f 2>/dev/null
diff --git a/tests/tail-2/assert-2.sh b/tests/tail-2/assert-2.sh
index aaf8785ae..67804feaf 100755
--- a/tests/tail-2/assert-2.sh
+++ b/tests/tail-2/assert-2.sh
@@ -28,10 +28,12 @@ check_tail_output()
     { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
-
 for mode in '' '---disable-inotify'; do
   rm -f a foo out
   touch a || framework_failure_
@@ -41,16 +43,15 @@ for mode in '' '---disable-inotify'; do
   # Wait up to 12.7s for tail to start.
   echo x > a || framework_failure_
   tail_re='^x$' retry_delay_ check_tail_output .1 7 ||
-    { cat out; fail=1; }
+    { cat out; fail=1; break; }
 
   # Wait up to 12.7s for tail to notice new foo file
   ok='ok ok ok'
   echo "$ok" > foo || framework_failure_
   tail_re="^$ok$" retry_delay_ check_tail_output .1 7 ||
-    { echo "$0: foo: unexpected delay?"; cat out; fail=1; }
+    { echo "$0: foo: unexpected delay?"; cat out; fail=1; break; }
 
-  kill $pid
-  wait $pid
+  cleanup_
 done
 
 Exit $fail
diff --git a/tests/tail-2/assert.sh b/tests/tail-2/assert.sh
index 906e364b5..43bd1e146 100755
--- a/tests/tail-2/assert.sh
+++ b/tests/tail-2/assert.sh
@@ -33,10 +33,12 @@ check_tail_output()
     { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
-
 for mode in '' '---disable-inotify'; do
   rm -f a foo out
   touch a foo || framework_failure_
@@ -46,22 +48,21 @@ for mode in '' '---disable-inotify'; do
   # Wait up to 12.7s for tail to start.
   echo x > a || framework_failure_
   tail_re='^x$' retry_delay_ check_tail_output .1 7 ||
-    { cat out; fail=1; }
+    { cat out; fail=1; break; }
 
   # Wait 12.7s for this diagnostic:
   # tail: foo: No such file or directory
   rm foo || framework_failure_
   tail_re='No such file' retry_delay_ check_tail_output .1 7 ||
-    { cat out; fail=1; }
+    { cat out; fail=1; break; }
 
   # Wait up to 12.7s for tail to notice new foo file
   ok='ok ok ok'
   echo "$ok" > foo || framework_failure_
   tail_re="^$ok$" retry_delay_ check_tail_output .1 7 ||
-    { echo "$0: foo: unexpected delay?"; cat out; fail=1; }
+    { echo "$0: foo: unexpected delay?"; cat out; fail=1; break; }
 
-  kill $pid
-  wait $pid
+  cleanup_
 done
 
 Exit $fail
diff --git a/tests/tail-2/f-vs-rename.sh b/tests/tail-2/f-vs-rename.sh
index bfd0949b4..083d71449 100755
--- a/tests/tail-2/f-vs-rename.sh
+++ b/tests/tail-2/f-vs-rename.sh
@@ -28,6 +28,9 @@ check_tail_output()
     { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -47,9 +50,7 @@ for mode in '' '---disable-inotify'; do
   # Wait up to 12.7s for "y" to appear in the output:
   tail_re='^y$' retry_delay_ check_tail_output .1 7 || { cat out; fail=1; }
 
-  kill $pid
-
-  wait $pid
+  cleanup_
 done
 
 Exit $fail
diff --git a/tests/tail-2/flush-initial.sh b/tests/tail-2/flush-initial.sh
index aaeb878f5..a25bbeb1b 100755
--- a/tests/tail-2/flush-initial.sh
+++ b/tests/tail-2/flush-initial.sh
@@ -22,10 +22,13 @@ print_ver_ tail
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 echo line > in || framework_failure_
 # Output should be buffered since we're writing to file
 # so we're depending on the flush to write out
-tail $fastpoll -f in > out & tail_pid=$!
+tail $fastpoll -f in > out & pid=$!
 
 # Wait for 3.1s for the file to be flushed.
 tail_flush()
@@ -37,8 +40,6 @@ tail_flush()
 }
 retry_delay_ tail_flush .1 5 || fail=1
 
-kill $tail_pid
-
-wait $tail_pid
+cleanup_
 
 Exit $fail
diff --git a/tests/tail-2/inotify-hash-abuse.sh b/tests/tail-2/inotify-hash-abuse.sh
index d8c14c1c5..6333699c1 100755
--- a/tests/tail-2/inotify-hash-abuse.sh
+++ b/tests/tail-2/inotify-hash-abuse.sh
@@ -30,6 +30,9 @@ check_tail_output()
     { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -58,10 +61,10 @@ for mode in '' '---disable-inotify'; do
   tail_re='has appeared' retry_delay_ check_tail_output .1 6 ||
     { cat out; fail=1; }
 
-  # Kill the working tail, or fail if it has already aborted
-  kill $pid || fail=1
+  # Double check that tail hasn't aborted
+  kill -0 $pid || fail=1
 
-  wait $pid
+  cleanup_
 done
 
 
diff --git a/tests/tail-2/inotify-hash-abuse2.sh b/tests/tail-2/inotify-hash-abuse2.sh
index b05423795..d641c241b 100755
--- a/tests/tail-2/inotify-hash-abuse2.sh
+++ b/tests/tail-2/inotify-hash-abuse2.sh
@@ -20,6 +20,9 @@
 . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
 print_ver_ tail
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -34,10 +37,10 @@ for mode in '' '---disable-inotify'; do
     touch f
   done
 
-  # Kill the working tail, or fail if it has already aborted
-  kill $pid || fail=1
+  # Ensure tail hasn't aborted
+  kill -0 $pid || fail=1
 
-  wait $pid
+  cleanup_
 done
 
 Exit $fail
diff --git a/tests/tail-2/inotify-race.sh b/tests/tail-2/inotify-race.sh
index d28f898b6..6e1a7fa40 100755
--- a/tests/tail-2/inotify-race.sh
+++ b/tests/tail-2/inotify-race.sh
@@ -23,6 +23,12 @@
 . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
 print_ver_ tail
 
+# Terminate any background gdb/tail process
+cleanup_() {
+  kill $pid 2>/dev/null && wait $pid
+  kill $sleep 2>/dev/null && wait $sleep
+}
+
 touch file || framework_failure_
 touch tail.out || framework_failure_
 
@@ -75,8 +81,7 @@ gdb -nx --batch-silent \
     --eval-command='shell echo never-seen-with-tail-7.5 >> file' \
     --eval-command='continue'                      \
     --eval-command='quit'                          \
-    tail < /dev/null > /dev/null 2>&1 &
-pid=$!
+    tail < /dev/null > /dev/null 2>&1 & pid=$!
 
 tail --pid=$pid -f tail.out | (read; kill $pid)
 
diff --git a/tests/tail-2/inotify-rotate-resources.sh b/tests/tail-2/inotify-rotate-resources.sh
index b65ceb7c5..792236516 100755
--- a/tests/tail-2/inotify-rotate-resources.sh
+++ b/tests/tail-2/inotify-rotate-resources.sh
@@ -48,13 +48,17 @@ cleanup_fail()
   fail=1
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 fastpoll='-s.1 --max-unchanged-stats=1'
 
-# Normally less than a second is required here, but with heavy load
-# and a lot of disk activity, even 20 seconds per grep_timeout is insufficient,
-# which leads to this timeout (used as a safety net for cleanup)
-# killing tail before processing is completed.
 touch k || framework_failure_
+
+# Note the timeout guard isn't strictly necessary here,
+# however without it strace will ignore SIGTERM.
+# strace does always honor SIGTERM with the -I2 option,
+# though that's not available on RHEL6 for example.
 timeout 180 strace -e inotify_rm_watch -o strace.out \
   tail -F $fastpoll k >> out 2>&1 & pid=$!
 
@@ -92,7 +96,7 @@ for i in $(seq 2); do
 done
 
 test "$reverted_to_polling_" = 1 && skip_ 'inotify resources already depleted'
-kill $pid
-wait $pid
+
+cleanup_
 
 Exit $fail
diff --git a/tests/tail-2/inotify-rotate.sh b/tests/tail-2/inotify-rotate.sh
index be1f07e5b..638a3bbb6 100755
--- a/tests/tail-2/inotify-rotate.sh
+++ b/tests/tail-2/inotify-rotate.sh
@@ -32,11 +32,14 @@ check_tail_output()
 # Wait up to 25.5 seconds for grep REGEXP 'out' to succeed.
 grep_timeout() { tail_re="$1" retry_delay_ check_tail_output .1 8; }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 cleanup_fail()
 {
   cat out
   warn_ $1
-  kill $pid
+  cleanup_
   fail=1
 }
 
@@ -68,8 +71,7 @@ for i in $(seq 50); do
     # wait for "ok" to appear in 'out'
     grep_timeout 'ok' || { cleanup_fail 'failed to detect echoed ok'; break; }
 
-    kill $pid
+    cleanup_
 done
 
-wait
 Exit $fail
diff --git a/tests/tail-2/pid.sh b/tests/tail-2/pid.sh
index 7a4986763..b410478e5 100755
--- a/tests/tail-2/pid.sh
+++ b/tests/tail-2/pid.sh
@@ -22,6 +22,8 @@ getlimits_
 
 touch empty here || framework_failure_
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
 
 for mode in '' '---disable-inotify'; do
   # Use tail itself to create a background process to monitor,
@@ -32,9 +34,7 @@ for mode in '' '---disable-inotify'; do
   timeout 1 tail -f -s.1 --pid=$pid $mode here
   test $? = 124 || fail=1
 
-  # Cleanup background process
-  kill $pid
-  wait $pid
+  cleanup_
 
   # Ensure that tail --pid=PID exits with success status when PID is dead.
   # Use an unlikely-to-be-live PID
diff --git a/tests/tail-2/pipe-f2.sh b/tests/tail-2/pipe-f2.sh
index 4d5fe5102..71a081c30 100755
--- a/tests/tail-2/pipe-f2.sh
+++ b/tests/tail-2/pipe-f2.sh
@@ -24,6 +24,9 @@ mkfifo_or_skip_ fifo
 echo 1 > fifo &
 echo 1 > exp || framework_failure_
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -40,9 +43,9 @@ retry_delay_ check_tail_output .1 7 || fail=1
 
 compare exp out || fail=1
 
-# Kill the still-running tail, or fail if it's gone.
-kill $pid || fail=1
+# Ensure tail is still running
+kill -0 $pid || fail=1
 
-wait $pid
+cleanup_
 
 Exit $fail
diff --git a/tests/tail-2/retry.sh b/tests/tail-2/retry.sh
index 6aee9967b..a1497d90f 100755
--- a/tests/tail-2/retry.sh
+++ b/tests/tail-2/retry.sh
@@ -36,6 +36,9 @@ wait4lines_ ()
   [ "$(countlines_)" -ge "$elc" ] || { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # Speedup the non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -62,8 +65,7 @@ retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; }
 echo "X" > missing              || framework_failure_
 # Wait for the expected output.
 retry_delay_ wait4lines_ .1 6 3 || { cat out; fail=1; }
-kill $pid
-wait $pid
+cleanup_
 # Expect 3 lines in the output file.
 [ "$(countlines_)" = 3 ]   || { fail=1; cat out; }
 grep -F 'cannot open' out  || { fail=1; cat out; }
@@ -80,8 +82,7 @@ retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; }
 echo "X" > missing              || framework_failure_
 # Wait for the expected output.
 retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; }
-kill $pid
-wait $pid
+cleanup_
 # Expect 4 lines in the output file.
 [ "$(countlines_)" = 4 ]   || { fail=1; cat out; }
 grep -F 'retry only effective for the initial open' out \
diff --git a/tests/tail-2/symlink.sh b/tests/tail-2/symlink.sh
index 9f1a2010f..b5dcbbd68 100755
--- a/tests/tail-2/symlink.sh
+++ b/tests/tail-2/symlink.sh
@@ -36,6 +36,9 @@ wait4lines_ ()
   [ "$(countlines_)" -ge "$elc" ] || { sleep $delay; return 1; }
 }
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # speedup non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -50,8 +53,7 @@ retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; }
 echo "X" > target               || framework_failure_
 # Wait for the expected output.
 retry_delay_ wait4lines_ .1 6 3 || { cat out; fail=1; }
-kill $pid
-wait $pid
+cleanup_
 # Expect 3 lines in the output file.
 [ "$(countlines_)" = 3 ]   || { fail=1; cat out; }
 grep -F 'cannot open' out  || { fail=1; cat out; }
@@ -74,8 +76,7 @@ retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; }
 echo "X2" > target2             || framework_failure_
 # Wait for the expected output.
 retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; }
-kill $pid
-wait $pid
+cleanup_
 # Expect 4 lines in the output file.
 [ "$(countlines_)" = 4 ]    || { fail=1; cat out; }
 grep -F 'become inacce' out || { fail=1; cat out; }
diff --git a/tests/tail-2/tail-n0f.sh b/tests/tail-2/tail-n0f.sh
index f713a1257..ba9f40718 100755
--- a/tests/tail-2/tail-n0f.sh
+++ b/tests/tail-2/tail-n0f.sh
@@ -35,6 +35,9 @@ chmod 0 unreadable || framework_failure_
 tail -c0 unreadable || fail=1
 tail -n0 unreadable || fail=1
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 for mode in '' '---disable-inotify'; do
   for file in empty nonempty; do
     for c_or_n in c n; do
@@ -51,8 +54,7 @@ for mode in '' '---disable-inotify'; do
       # Wait up to 1.5s for tail to sleep
       retry_delay_ tail_sleeping .1 4 ||
         { echo $0: process in unexpected state: $state >&2; fail=1; }
-      kill $pid
-      wait $pid
+      cleanup_
     done
   done
 done
diff --git a/tests/tail-2/wait.sh b/tests/tail-2/wait.sh
index ebae62bd2..071d889d6 100755
--- a/tests/tail-2/wait.sh
+++ b/tests/tail-2/wait.sh
@@ -23,6 +23,9 @@ print_ver_ tail
 touch here || framework_failure_
 { touch unreadable && chmod a-r unreadable; } || framework_failure_
 
+# Terminate any background tail process
+cleanup_() { kill $pid 2>/dev/null && wait $pid; }
+
 # speedup non inotify case
 fastpoll='-s.1 --max-unchanged-stats=1'
 
@@ -72,8 +75,7 @@ for mode in '' '---disable-inotify'; do
     sleep $delay
     echo NO >> l
     sleep $delay
-    kill $pid
-    wait $pid
+    cleanup_
     rm -f k l
 
     test -s tail.out