stdbuf: A new program to run a command with modified stdio buffering

* AUTHORS: Register as the author.
* NEWS: Mention this change.
* README: Add stdbuf command to list.
* configure.ac: Only enable on ELF systems with GCC.
* cfg.mk (sc_system_h_headers): Use VC_LIST_EXCEPT rather than
VC_LIST, so we can add an exception, if needed.
* .x-sc_system_h_headers: New file.  Exempt libstdbuf.c.
* Makefile.am (syntax_check_exceptions): Add .x-sc_system_h_headers.
* doc/coreutils.texi (stdbuf invocation): Add stdbuf info.
* man/.gitignore: Ignore generated manpage.
* src/.gitignore: Ignore stdbuf and libstdbuf.so binaries.
* man/Makefile.am (stdbuf.1): Add dependency.
* man/stdbuf.x: New file with example usage.
* po/POTFILES.in: Reference new command and shared library sources.
* src/Makefile.am (build_if_possible__progs): Add stdbuf and libstdbuf,
(pkglib_PROGRAMS): Reference optional shared lib,
(libstdbuf_so_LDADD): Ensure we don't link with non PIC libcoreutils.a.
(libstdbuf_so_LDFLAGS): Add -shared GCC option,
(libstdbuf_so_CFLAGS): Add -fPIC GCC option.
(check-README): Exclude libstbuf.
(check-AUTHORS): ditto.
(sc_tight_scope): Exclude functions starting with __.
* src/libstdbuf.c: The LD_PRELOAD shared library to control buffering.
* src/stdbuf.c: New file to setup env variables before execing command.
* tests/Makefile.am: Reference new test file.
* tests/misc/help-version: Set expected exit codes.
* tests/misc/invalid-opt: ditto.
* tests/misc/stdbuf: Add 9 tests.

4 files changed, 550 insertions, 3 deletions

diff --git a/src/.gitignore b/src/.gitignore
index bc1452390..f2886dea2 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -40,6 +40,7 @@ hostname
 id
 join
 kill
+libstdbuf.so
 libver.a
 link
 ln
@@ -81,6 +82,7 @@ sleep
 sort
 split
 stat
+stdbuf
 stty
 su
 sum
diff --git a/src/Makefile.am b/src/Makefile.am
index 3bed7b1af..4f21c8633 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -24,7 +24,7 @@ no_install__progs = \
   arch hostname su
 
 build_if_possible__progs = \
-  chroot df hostid nice pinky stty su uname uptime users who
+  chroot df hostid nice pinky stdbuf libstdbuf.so stty su uname uptime users who
 
 AM_CFLAGS = $(WARN_CFLAGS) $(WERROR_CFLAGS)
 
@@ -48,6 +48,8 @@ bin_PROGRAMS = $(OPTIONAL_BIN_PROGS)
 
 noinst_PROGRAMS = setuidgid getlimits
 
+pkglib_PROGRAMS = $(OPTIONAL_PKGLIB_PROGS)
+
 noinst_HEADERS = \
   chown-core.h \
   copy.h \
@@ -91,6 +93,7 @@ du_LDADD = $(LDADD)
 getlimits_LDADD = $(LDADD)
 ptx_LDADD = $(LDADD)
 split_LDADD = $(LDADD)
+stdbuf_LDADD = $(LDADD)
 timeout_LDADD = $(LDADD)
 truncate_LDADD = $(LDADD)
 
@@ -170,6 +173,7 @@ du_LDADD += $(LIBICONV)
 getlimits_LDADD += $(LIBICONV)
 ptx_LDADD += $(LIBICONV)
 split_LDADD += $(LIBICONV)
+stdbuf_LDADD += $(LIBICONV)
 timeout_LDADD += $(LIBICONV)
 truncate_LDADD += $(LIBICONV)
 
@@ -286,6 +290,16 @@ sha512sum_CPPFLAGS = -DHASH_ALGO_SHA512=1 $(AM_CPPFLAGS)
 
 ginstall_CPPFLAGS = -DENABLE_MATCHPATHCON=1 $(AM_CPPFLAGS)
 
+# Ensure we don't link against libcoreutils.a as that lib is
+# not compiled with -fPIC which causes issues on 64 bit at least
+libstdbuf_so_LDADD =
+
+# Note libstdbuf is only compiled if GCC is available
+# (as per the check in configure.ac), so these flags should be available.
+# libtool is probably required to relax this dependency.
+libstdbuf_so_LDFLAGS = -shared
+libstdbuf_so_CFLAGS = -fPIC $(AM_CFLAGS)
+
 editpl = sed -e 's,@''PERL''@,$(PERL),g'
 
 BUILT_SOURCES += dircolors.h
@@ -369,6 +383,7 @@ check-README:
 	rm -rf $(pr) $(pm)
 	echo $(all_programs) \
 	 | tr -s ' ' '\n' | sed -e 's,$(EXEEXT)$$,,;s/ginstall/install/' \
+	 | sed /libstdbuf/d \
 	 | $(ASSORT) -u > $(pm) && \
 	sed -n '/^The programs .* are:/,/^[a-zA-Z]/p' $(top_srcdir)/README \
 	  | sed -n '/^   */s///p' | tr -s ' ' '\n' > $(pr)
@@ -394,6 +409,7 @@ check-AUTHORS: $(all_programs)
 	  && { echo "$@: skipping this check"; exit 0; }; \
 	rm -f $(au_actual) $(au_dotdot);		\
 	for i in `ls $(all_programs) | sed -e 's,$(EXEEXT)$$,,' \
+	    | sed /libstdbuf/d				\
 	    | $(ASSORT) -u`; do				\
 	  test "$$i" = '[' && continue;			\
 	  exe=$$i;					\
@@ -416,7 +432,9 @@ check-AUTHORS: $(all_programs)
 # Most functions in src/*.c should have static scope.
 # Any that don't must be marked with `extern', but `main'
 # and `usage' are exceptions.  They're always extern, but
-# don't need to be marked.
+# don't need to be marked. Also functions starting with __
+# are exempted due to possibly being added by the compiler
+# (when compiled as a shared library for example).
 #
 # The second nm|grep checks for file-scope variables with `extern' scope.
 .PHONY: sc_tight_scope
@@ -427,7 +445,7 @@ sc_tight_scope: $(bin_PROGRAMS)
 	       test -f $$f && d= || d=$(srcdir)/; echo $$d$$f; done`;	\
 	hdr=`for f in $(noinst_HEADERS); do				\
 	       test -f $$f && d= || d=$(srcdir)/; echo $$d$$f; done`;	\
-	( printf 'main\nusage\n';					\
+	( printf 'main\nusage\n_.*\n';					\
 	  grep -h -A1 '^extern .*[^;]$$' $$src				\
 	    | grep -vE '^(extern |--)' | sed 's/ .*//';			\
 	  perl -ne '/^extern \S+ (\S*) \(/ and print "$$1\n"' $$hdr;	\
diff --git a/src/libstdbuf.c b/src/libstdbuf.c
new file mode 100644
index 000000000..8eec0960a
--- /dev/null
+++ b/src/libstdbuf.c
@@ -0,0 +1,141 @@
+/* libstdbuf -- a shared lib to preload to setup stdio buffering for a command
+   Copyright (C) 2009 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* Written by Pádraig Brady.  LD_PRELOAD idea from Brian Dessent.  */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "system.h"
+#include "verify.h"
+
+/* Note currently for glibc (2.3.5) the following call does not change the
+   the buffer size, and more problematically does not give any indication
+   that the new size request was ignored:
+
+       setvbuf (stdout, (char*)NULL, _IOFBF, 8192);
+
+   The ISO C99 standard section 7.19.5.6 on the setvbuf function says:
+
+   ... If buf is not a null pointer, the array it points to _may_ be used
+   instead of a buffer allocated by the setvbuf function and the argument
+   size specifies the size of the array; otherwise, size _may_ determine
+   the size of a buffer allocated by the setvbuf function. ...
+
+   Obviously some interpret the above to mean setvbuf(....,size)
+   is only a hint from the application which I don't agree with.
+
+   FreeBSD's libc seems more sensible in this regard. From the man page:
+
+   The size argument may be given as zero to obtain deferred optimal-size
+   buffer allocation as usual.  If it is not zero, then except for
+   unbuffered files, the buf argument should point to a buffer at least size
+   bytes long; this buffer will be used instead of the current buffer.  (If
+   the size argument is not zero but buf is NULL, a buffer of the given size
+   will be allocated immediately, and released on close.  This is an extension
+   to ANSI C; portable code should use a size of 0 with any NULL buffer.)
+   --------------------
+   Another issue is that on glibc-2.7 the following doesn't buffer
+   the first write if it's greater than 1 byte.
+
+       setvbuf(stdout,buf,_IOFBF,127);
+
+   Now the POSIX standard says that "allocating a buffer of size bytes does
+   not necessarily imply that all of size bytes are used for the buffer area".
+   However I think it's just a buggy implementation due to the various
+   inconsistencies with write sizes and subsequent writes.  */
+
+static const char *
+fileno_to_name (const int fd)
+{
+  const char *ret = NULL;
+
+  switch (fd)
+    {
+    case 0:
+      ret = "stdin";
+      break;
+    case 1:
+      ret = "stdout";
+      break;
+    case 2:
+      ret = "stderr";
+      break;
+    default:
+      ret = "unknown";
+      break;
+    }
+
+  return ret;
+}
+
+static void
+apply_mode (FILE *stream, const char *mode)
+{
+  char *buf = NULL;
+  int setvbuf_mode;
+  size_t size = 0;
+
+  if (*mode == '0')
+    setvbuf_mode = _IONBF;
+  else if (*mode == 'L')
+    setvbuf_mode = _IOLBF;      /* FIXME: should we allow 1ML  */
+  else
+    {
+      setvbuf_mode = _IOFBF;
+      verify (SIZE_MAX <= ULONG_MAX);
+      size = strtoul (mode, NULL, 10);
+      if (size > 0)
+        {
+          if (!(buf = malloc (size)))   /* will be freed by fclose()  */
+            {
+              /* We could defer the allocation to libc, however since
+                 glibc currently ignores the combination of NULL buffer
+                 with non zero size, we'll fail here.  */
+              fprintf (stderr,
+                       _("failed to allocate a %" PRIuMAX
+                         " byte stdio buffer\n"), (uintmax_t) size);
+              return;
+            }
+        }
+      else
+        {
+          fprintf (stderr, _("invalid buffering mode %s for %s\n"),
+                   mode, fileno_to_name (fileno (stream)));
+          return;
+        }
+    }
+
+  if (setvbuf (stream, buf, setvbuf_mode, size) != 0)
+    {
+      fprintf (stderr, _("could not set buffering of %s to mode %s\n"),
+               fileno_to_name (fileno (stream)), mode);
+    }
+}
+
+__attribute__ ((constructor)) static void
+stdbuf (void)
+{
+  char *e_mode = getenv ("_STDBUF_E");
+  char *i_mode = getenv ("_STDBUF_I");
+  char *o_mode = getenv ("_STDBUF_O");
+  if (e_mode) /* Do first so can write errors to stderr  */
+    apply_mode (stderr, e_mode);
+  if (i_mode)
+    apply_mode (stdin, i_mode);
+  if (o_mode)
+    apply_mode (stdout, o_mode);
+}
diff --git a/src/stdbuf.c b/src/stdbuf.c
new file mode 100644
index 000000000..89f2242de
--- /dev/null
+++ b/src/stdbuf.c
@@ -0,0 +1,386 @@
+/* stdbuf -- setup the standard streams for a command
+   Copyright (C) 2009 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* Written by Pádraig Brady.  */
+
+#include <config.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <assert.h>
+
+#include "system.h"
+#include "error.h"
+#include "posixver.h"
+#include "quote.h"
+#include "xstrtol.h"
+#include "c-ctype.h"
+
+/* The official name of this program (e.g., no `g' prefix).  */
+#define PROGRAM_NAME "stdbuf"
+#define LIB_NAME "libstdbuf.so" /* FIXME: don't hardcode  */
+
+#define AUTHORS proper_name_utf8 ("Padraig Brady", "P\303\241draig Brady")
+
+/* Internal error  */
+enum { EXIT_CANCELED = 125 };
+
+static char *program_path;
+
+extern char **environ;
+
+static struct
+{
+  size_t size;
+  int optc;
+  char *optarg;
+} stdbuf[3];
+
+static struct option const longopts[] =
+{
+  {"input", required_argument, NULL, 'i'},
+  {"output", required_argument, NULL, 'o'},
+  {"error", required_argument, NULL, 'e'},
+  {GETOPT_HELP_OPTION_DECL},
+  {GETOPT_VERSION_OPTION_DECL},
+  {NULL, 0, NULL, 0}
+};
+
+/* Set size to the value of STR, interpreted as a decimal integer,
+   optionally multiplied by various values.
+   Return -1 on error, 0 on success.
+
+   This supports dd BLOCK size suffixes.
+   Note we don't support dd's b=512, c=1, w=2 or 21x512MiB formats.  */
+static int
+parse_size (char const *str, size_t *size)
+{
+  uintmax_t tmp_size;
+  enum strtol_error e = xstrtoumax (str, NULL, 10, &tmp_size, "EGkKMPTYZ0");
+  if (e == LONGINT_OK && tmp_size > SIZE_MAX)
+    e = LONGINT_OVERFLOW;
+
+  if (e == LONGINT_OK)
+    {
+      errno = 0;
+      *size = tmp_size;
+      return 0;
+    }
+
+  errno = (e == LONGINT_OVERFLOW ? EOVERFLOW : 0);
+  return -1;
+}
+
+void
+usage (int status)
+{
+  if (status != EXIT_SUCCESS)
+    fprintf (stderr, _("Try `%s --help' for more information.\n"),
+             program_name);
+  else
+    {
+      printf (_("Usage: %s OPTION... COMMAND\n"), program_name);
+      fputs (_("\
+Run COMMAND, with modified buffering operations for its standard streams.\n\
+\n\
+"), stdout);
+      fputs (_("\
+Mandatory arguments to long options are mandatory for short options too.\n\
+"), stdout);
+      fputs (_("\
+  -i, --input=MODE   Adjust standard input stream buffering\n\
+  -o, --output=MODE  Adjust standard output stream buffering\n\
+  -e, --error=MODE   Adjust standard error stream buffering\n\
+"), stdout);
+      fputs (HELP_OPTION_DESCRIPTION, stdout);
+      fputs (VERSION_OPTION_DESCRIPTION, stdout);
+      fputs (_("\n\
+If MODE is `L' then corresponding stream will be line buffered.\n\
+This option is invalid with standard input.\n"), stdout);
+      fputs (_("\n\
+If MODE is `0' then corresponding stream will be unbuffered.\n\
+"), stdout);
+      fputs (_("\n\
+Otherwise MODE is a number which may be followed by one of the following:\n\
+KB 1000, K 1024, MB 1000*1000, M 1024*1024, and so on for G, T, P, E, Z, Y.\n\
+In this case the corresponding stream will be fully buffered with the buffer\n\
+size set to MODE bytes.\n\
+"), stdout);
+      fputs (_("\n\
+NOTE: If COMMAND adjusts the buffering of its standard streams (`tee' does\n\
+for e.g.) then that will override corresponding settings changed by `stdbuf'.\n\
+Also some filters (like `dd' and `cat' etc.) don't use streams for I/O,\n\
+and are thus unaffected by `stdbuf' settings.\n\
+"), stdout);
+      emit_bug_reporting_address ();
+    }
+  exit (status);
+}
+
+/* argv[0] can be anything really, but generally it contains
+   the path to the executable or just a name if it was executed
+   using $PATH. In the latter case to get the path we can:
+   search getenv("PATH"), readlink("/prof/self/exe"), getenv("_"),
+   dladdr(), pstat_getpathname(), etc.  */
+
+static void
+set_program_path (const char *arg)
+{
+  if (strchr (arg, '/'))        /* Use absolute or relative paths directly.  */
+    {
+      program_path = dir_name (arg);
+    }
+  else
+    {
+      char *path;
+      char tmppath[PATH_MAX + 1];
+      ssize_t len = readlink ("/proc/self/exe", tmppath, sizeof (tmppath) - 1);
+      if (len > 0)
+        {
+          tmppath[len] = '\0';
+          program_path = dir_name (tmppath);
+        }
+      else if ((path = getenv ("PATH")))
+        {
+          char *dir;
+          path = xstrdup (path);
+          for (dir = strtok (path, ":"); dir != NULL; dir = strtok (NULL, ":"))
+            {
+              int req = snprintf (tmppath, sizeof (tmppath), "%s/%s", dir, arg);
+              if (req >= sizeof (tmppath))
+                {
+                  error (0, 0, _("path truncated when looking for %s"),
+                         quote (arg));
+                }
+              else if (access (tmppath, X_OK) == 0)
+                {
+                  program_path = dir_name (tmppath);
+                  break;
+                }
+            }
+          free (path);
+        }
+    }
+}
+
+static int
+optc_to_fileno (int c)
+{
+  int ret = -1;
+
+  switch (c)
+    {
+    case 'e':
+      ret = STDERR_FILENO;
+      break;
+    case 'i':
+      ret = STDIN_FILENO;
+      break;
+    case 'o':
+      ret = STDOUT_FILENO;
+      break;
+    }
+
+  return ret;
+}
+
+static void
+set_LD_PRELOAD (void)
+{
+  int ret;
+  char *old_libs = getenv ("LD_PRELOAD");
+  char *LD_PRELOAD;
+
+  /* Note this would auto add the appropriate search path for "libstdbuf.so":
+     gcc stdbuf.c -Wl,-rpath,'$ORIGIN' -Wl,-rpath,$PKGLIBDIR
+     However we want the lookup done for the exec'd command not stdbuf.
+
+     Since we don't link against libstdbuf.so add it to LIBDIR rather than
+     LIBEXECDIR, as we'll search for it in the "sys default" case below.  */
+  char const *const search_path[] = {
+    program_path,
+    PKGLIBDIR,
+    "",                         /* sys default */
+    NULL
+  };
+
+  char const *const *path = search_path;
+  char *libstdbuf;
+
+  do
+    {
+      struct stat sb;
+
+      if (!**path)              /* system default  */
+        {
+          libstdbuf = xstrdup (LIB_NAME);
+          break;
+        }
+      ret = asprintf (&libstdbuf, "%s/%s", *path, LIB_NAME);
+      if (ret < 0)
+        xalloc_die ();
+      if (stat (libstdbuf, &sb) == 0)   /* file_exists  */
+        break;
+      free (libstdbuf);
+    }
+  while (*++path);
+
+  /* FIXME: Do we need to support libstdbuf.dll, c:, '\' separators etc?  */
+
+  if (old_libs)
+    ret = asprintf (&LD_PRELOAD, "LD_PRELOAD=%s:%s", old_libs, libstdbuf);
+  else
+    ret = asprintf (&LD_PRELOAD, "LD_PRELOAD=%s", libstdbuf);
+
+  if (ret < 0)
+    xalloc_die ();
+
+  free (libstdbuf);
+
+  ret = putenv (LD_PRELOAD);
+
+  if (ret != 0)
+    {
+      error (EXIT_CANCELED, errno,
+             _("failed to update the environment with %s"),
+             quote (LD_PRELOAD));
+    }
+}
+
+/* Populate environ with _STDBUF_I=$MODE _STDBUF_O=$MODE _STDBUF_E=$MODE  */
+
+static void
+set_libstdbuf_options (void)
+{
+  int i;
+
+  for (i = 0; i < ARRAY_CARDINALITY (stdbuf); i++)
+    {
+      if (stdbuf[i].optarg)
+        {
+          char *var;
+          int ret;
+
+          if (*stdbuf[i].optarg == 'L')
+            ret = asprintf (&var, "%s%c=L", "_STDBUF_",
+                            toupper (stdbuf[i].optc));
+          else
+            ret = asprintf (&var, "%s%c=%" PRIuMAX, "_STDBUF_",
+                            toupper (stdbuf[i].optc),
+                            (uintmax_t) stdbuf[i].size);
+          if (ret < 0)
+            xalloc_die ();
+
+          if (putenv (var) != 0)
+            {
+              error (EXIT_CANCELED, errno,
+                     _("failed to update the environment with %s"),
+                     quote (var));
+            }
+        }
+    }
+}
+
+int
+main (int argc, char **argv)
+{
+  int c;
+
+  initialize_main (&argc, &argv);
+  set_program_name (argv[0]);
+  setlocale (LC_ALL, "");
+  bindtextdomain (PACKAGE, LOCALEDIR);
+  textdomain (PACKAGE);
+
+  initialize_exit_failure (EXIT_CANCELED);
+  atexit (close_stdout);
+
+  while ((c = getopt_long (argc, argv, "+i:o:e:", longopts, NULL)) != -1)
+    {
+      int opt_fileno;
+
+      switch (c)
+        {
+        /* Old McDonald had a farm ei...  */
+        case 'e':
+        case 'i':
+        case 'o':
+          opt_fileno = optc_to_fileno (c);
+          assert (0 < opt_fileno && opt_fileno <= ARRAY_CARDINALITY (stdbuf));
+          stdbuf[opt_fileno].optc = c;
+          while (c_isspace (*optarg))
+            optarg++;
+          stdbuf[opt_fileno].optarg = optarg;
+          if (c == 'i' && *optarg == 'L')
+            {
+              /* -oL will be by far the most common use of this utility,
+                 but one could easily think -iL might have the same affect,
+                 so disallow it as it could be confusing.  */
+              error (0, 0, _("line buffering stdin is meaningless"));
+              usage (EXIT_CANCELED);
+            }
+
+          if (!STREQ (optarg, "L")
+              && parse_size (optarg, &stdbuf[opt_fileno].size) == -1)
+            error (EXIT_CANCELED, errno, _("invalid mode %s"), quote (optarg));
+
+          break;
+
+        case_GETOPT_HELP_CHAR;
+
+        case_GETOPT_VERSION_CHAR (PROGRAM_NAME, AUTHORS);
+
+        default:
+          usage (EXIT_CANCELED);
+        }
+    }
+
+  argv += optind;
+  argc -= optind;
+
+  /* must specify at least 1 command.  */
+  if (argc < 1)
+    {
+      error (0, 0, _("missing operand"));
+      usage (EXIT_CANCELED);
+    }
+
+  /* FIXME: Should we mandate at least one option?  */
+
+  set_libstdbuf_options ();
+
+  /* Try to preload libstdbuf first from the same path as
+     stdbuf is running from.  */
+  set_program_path (argv[0]);
+  if (!program_path)
+    program_path = xstrdup (PKGLIBDIR);  /* Need to init to non NULL.  */
+  set_LD_PRELOAD ();
+  free (program_path);
+
+  execvp (*argv, argv);
+
+  {
+    int exit_status = (errno == ENOENT ? EXIT_ENOENT : EXIT_CANNOT_INVOKE);
+    error (0, errno, _("failed to run command %s"), quote (argv[0]));
+    exit (exit_status);
+  }
+}
+
+/*
+ * Local variables:
+ *  indent-tabs-mode: nil
+ * End:
+ */