timeout: handle signals more transparently

This was originally attempted in commit v8.12-117-g5a647a0,
but reverted before release because of the unreliability
of disabling core dumps using setrlimit() on Linux kernels.
This new version instead uses prctl() where available to
more reliably disable core dumps for the timeout process.

* m4/jm-macros.m4: Define HAVE_SETRLIMIT and HAVE_PRCTL.
* src/timeout.c (disable_core_dumps): A new function
that disables coredumps using prctl or setrlimit if available.
(main): If the child exited with a signal and we can
disable core dumps, then raise that signal to the timeout
process itself, so that callers may also see the signal status.
Also print a message indicating when the monitored command
dumped core, as that information is lost in the signal
propagation through timeout.

2 files changed, 33 insertions, 14 deletions

diff --git a/m4/jm-macros.m4 b/m4/jm-macros.m4
index 016172f04..ff89aa31b 100644
--- a/m4/jm-macros.m4
+++ b/m4/jm-macros.m4
@@ -64,7 +64,7 @@ AC_DEFUN([coreutils_MACROS],
   # Used by sort.c.
   AC_CHECK_FUNCS_ONCE([nl_langinfo])
   # Used by timeout.c
-  AC_CHECK_FUNCS_ONCE([setrlimit])
+  AC_CHECK_FUNCS_ONCE([setrlimit prctl])
 
   # Used by tail.c.
   AC_CHECK_FUNCS([inotify_init],
diff --git a/src/timeout.c b/src/timeout.c
index c0a252748..4ce18adc6 100644
--- a/src/timeout.c
+++ b/src/timeout.c
@@ -49,6 +49,9 @@
 #include <stdio.h>
 #include <sys/types.h>
 #include <signal.h>
+#if HAVE_PRCTL
+# include <sys/prctl.h>
+#endif
 #include <sys/wait.h>
 
 #include "system.h"
@@ -316,6 +319,29 @@ install_signal_handlers (int sigterm)
   sigaction (sigterm, &sa, NULL); /* user specified termination signal.  */
 }
 
+/* Try to disable core dumps for this process.
+   Return TRUE if successful, FALSE otherwise.  */
+static bool
+disable_core_dumps (void)
+{
+#if HAVE_PRCTL && defined PR_SET_DUMPABLE
+  if (prctl (PR_SET_DUMPABLE, 0) == 0)
+    return true;
+
+#elif HAVE_SETRLIMIT && defined RLIMIT_CORE
+  /* Note this doesn't disable processing by a filter in
+     /proc/sys/kernel/core_pattern on Linux.  */
+  if (setrlimit (RLIMIT_CORE, &(struct rlimit) {0,0}) == 0)
+    return true;
+
+#else
+  return false;
+#endif
+
+  error (0, errno, _("warning: disabling core dumps failed"));
+  return false;
+}
+
 int
 main (int argc, char **argv)
 {
@@ -426,21 +452,14 @@ main (int argc, char **argv)
           else if (WIFSIGNALED (status))
             {
               int sig = WTERMSIG (status);
-/* The following is not used as one cannot disable processing
-   by a filter in /proc/sys/kernel/core_pattern on Linux.  */
-#if 0 && HAVE_SETRLIMIT && defined RLIMIT_CORE
-              if (!timed_out)
+              if (WCOREDUMP (status))
+                error (0, 0, _("the monitored command dumped core"));
+              if (!timed_out && disable_core_dumps ())
                 {
-                  /* exit with the signal flag set, but avoid core files.  */
-                  if (setrlimit (RLIMIT_CORE, &(struct rlimit) {0,0}) == 0)
-                    {
-                      signal (sig, SIG_DFL);
-                      raise (sig);
-                    }
-                  else
-                    error (0, errno, _("warning: disabling core dumps failed"));
+                  /* exit with the signal flag set.  */
+                  signal (sig, SIG_DFL);
+                  raise (sig);
                 }
-#endif
               status = sig + 128; /* what sh returns for signaled processes.  */
             }
           else