dd: add a flag to discard cached data

* src/dd.c (FFS_MASK): A new macro (Find First Set) refactored
from the following enum as it's now used twice.
(usage): Mention the new 'nocache' flag.
(cache_round): A new function to help ignore requests
to drop cache, that are less than page_size.
(invalidate_cache): A new function to call posix_fadvise()
with the appropriate offset and length.  Note we don't
use fdadvise() so we can detect errors when count=0.
(dd_copy): Call invalidate_cache() for the portions read.
(iwrite): Likewise for the portions written.
(main): Call invalidate_cache for page_size slop or
for full file when count=0.
* cfg.mk (sc_dd_O_FLAGS): Adjust to pass.
* doc/coreutils.texi (dd invocation): Describe the 'nocache' flag,
and give some examples of how it can be used.
* tests/dd/nocache: A new test.
* tests/Makefile.am: Reference the new test.
* NEWS: Mention the new feature.

6 files changed, 248 insertions, 8 deletions

diff --git a/NEWS b/NEWS
index 658a89af7..3157ef201 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,12 @@ GNU coreutils NEWS                                    -*- outline -*-
   wc would dereference a NULL pointer upon an early out-of-memory error
   [bug introduced in coreutils-7.1]
 
+** New features
+
+  dd now accepts the 'nocache' flag to the iflag and oflag options,
+  which will discard any cache associated with the files, or
+  processed portion thereof.
+
 
 * Noteworthy changes in release 8.10 (2011-02-04) [stable]
 
diff --git a/cfg.mk b/cfg.mk
index e4f3faaad..c897dc45f 100644
--- a/cfg.mk
+++ b/cfg.mk
@@ -39,8 +39,8 @@ _hv_file ?= $(srcdir)/tests/misc/help-version
 dd = $(srcdir)/src/dd.c
 sc_dd_O_FLAGS:
 	@rm -f $@.1 $@.2
-	@{ echo O_FULLBLOCK; perl -nle '/^ +\| (O_\w*)$$/ and print $$1' \
-	  $(dd); } | sort > $@.1
+	@{ echo O_FULLBLOCK; echo O_NOCACHE;				\
+	  perl -nle '/^ +\| (O_\w*)$$/ and print $$1' $(dd); } | sort > $@.1
 	@{ echo O_NOFOLLOW; perl -nle '/{"[a-z]+",\s*(O_\w+)},/ and print $$1' \
 	  $(dd); } | sort > $@.2
 	@diff -u $@.1 $@.2 || diff=1 || diff=;				\
diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index ae3712dba..8dd1ed8ba 100644
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -8174,6 +8174,31 @@ last-access and last-modified time) is not necessarily synchronized.
 @cindex synchronized data and metadata I/O
 Use synchronized I/O for both data and metadata.
 
+@item nocache
+@opindex nocache
+@cindex discarding file cache
+Discard the data cache for a file.
+When count=0 all cache is discarded,
+otherwise the cache is dropped for the processed
+portion of the file.  Also when count=0
+failure to discard the cache is diagnosed
+and reflected in the exit status.
+Here as some usage examples:
+
+@example
+# Advise to drop cache for whole file
+dd if=ifile iflag=nocache count=0
+
+# Ensure drop cache for the whole file
+dd of=ofile oflag=nocache conv=notrunc,fdatasync count=0
+
+# Drop cache for part of file
+dd if=ifile iflag=nocache skip=10 count=10 of=/dev/null
+
+# Stream data using just the read-ahead cache
+dd if=ifile of=ofile iflag=nocache oflag=nocache
+@end example
+
 @item nonblock
 @opindex nonblock
 @cindex nonblocking I/O
diff --git a/src/dd.c b/src/dd.c
index acbcea95f..6069671ff 100644
--- a/src/dd.c
+++ b/src/dd.c
@@ -225,6 +225,9 @@ static sig_atomic_t volatile interrupt_signal;
 /* A count of the number of pending info signals that have been received.  */
 static sig_atomic_t volatile info_signal_count;
 
+/* Whether to discard cache for input or output.  */
+static bool i_nocache, o_nocache;
+
 /* Function used for read (to handle iflag=fullblock parameter).  */
 static ssize_t (*iread_fnc) (int fd, char *buf, size_t size);
 
@@ -259,6 +262,7 @@ static struct symbol_value const conversions[] =
   {"", 0}
 };
 
+#define FFS_MASK(x) ((x) ^ ((x) & ((x) - 1)))
 enum
   {
     /* Compute a value that's bitwise disjoint from the union
@@ -278,17 +282,23 @@ enum
           | O_SYNC
           | O_TEXT
           ),
-    /* Use its lowest bit.  */
-    O_FULLBLOCK = v ^ (v & (v - 1))
+
+    /* Use its lowest bits for private flags.  */
+    O_FULLBLOCK = FFS_MASK (v),
+    v2 = v ^ O_FULLBLOCK,
+
+    O_NOCACHE = FFS_MASK (v2)
   };
 
 /* Ensure that we got something.  */
 verify (O_FULLBLOCK != 0);
+verify (O_NOCACHE != 0);
 
 #define MULTIPLE_BITS_SET(i) (((i) & ((i) - 1)) != 0)
 
 /* Ensure that this is a single-bit value.  */
 verify ( ! MULTIPLE_BITS_SET (O_FULLBLOCK));
+verify ( ! MULTIPLE_BITS_SET (O_NOCACHE));
 
 /* Flags, for iflag="..." and oflag="...".  */
 static struct symbol_value const flags[] =
@@ -300,6 +310,7 @@ static struct symbol_value const flags[] =
   {"directory",	O_DIRECTORY},
   {"dsync",	O_DSYNC},
   {"noatime",	O_NOATIME},
+  {"nocache",	O_NOCACHE},   /* Discard cache.  */
   {"noctty",	O_NOCTTY},
   {"nofollow",	HAVE_WORKING_O_NOFOLLOW ? O_NOFOLLOW : 0},
   {"nolinks",	O_NOLINKS},
@@ -534,6 +545,10 @@ Each FLAG symbol may be:\n\
         fputs (_("  nonblock  use non-blocking I/O\n"), stdout);
       if (O_NOATIME)
         fputs (_("  noatime   do not update access time\n"), stdout);
+#if HAVE_POSIX_FADVISE
+      if (O_NOCACHE)
+        fputs (_("  nocache   discard cached data\n"), stdout);
+#endif
       if (O_NOCTTY)
         fputs (_("  noctty    do not assign controlling terminal from file\n"),
                stdout);
@@ -787,6 +802,91 @@ process_signals (void)
     }
 }
 
+/* Return LEN rounded down to a multiple of PAGE_SIZE
+   while storing the remainder internally per FD.
+   Pass LEN == 0 to get the current remainder.  */
+
+static off_t
+cache_round (int fd, off_t len)
+{
+  static off_t i_pending, o_pending;
+  off_t *pending = (fd == STDIN_FILENO ? &i_pending : &o_pending);
+
+  if (len)
+    {
+      off_t c_pending = *pending + len;
+      *pending = c_pending % page_size;
+      if (c_pending > *pending)
+        len = c_pending - *pending;
+      else
+        len = 0;
+    }
+  else
+    len = *pending;
+
+  return len;
+}
+
+/* Discard the cache from the current offset of either
+   STDIN_FILENO or STDOUT_FILENO.
+   Return true on success.  */
+
+static bool
+invalidate_cache (int fd, off_t len)
+{
+  int adv_ret = -1;
+
+  /* Minimize syscalls.  */
+  off_t clen = cache_round (fd, len);
+  if (len && !clen)
+    return true; /* Don't advise this time.  */
+  if (!len && !clen && max_records)
+    return true; /* Nothing pending.  */
+  off_t pending = len ? cache_round (fd, 0) : 0;
+
+  if (fd == STDIN_FILENO)
+    {
+      if (input_seekable)
+        {
+          /* Note we're being careful here to only invalidate what
+             we've read, so as not to dump any read ahead cache.  */
+#if HAVE_POSIX_FADVISE
+            adv_ret = posix_fadvise (fd, input_offset - clen - pending, clen,
+                                     POSIX_FADV_DONTNEED);
+#else
+            errno = ENOTSUP;
+#endif
+        }
+      else
+        errno = ESPIPE;
+    }
+  else if (fd == STDOUT_FILENO)
+    {
+      static off_t output_offset = -2;
+
+      if (output_offset != -1)
+        {
+          if (0 > output_offset)
+            {
+              output_offset = lseek (fd, 0, SEEK_CUR);
+              output_offset -= clen + pending;
+            }
+          if (0 <= output_offset)
+            {
+#if HAVE_POSIX_FADVISE
+              adv_ret = posix_fadvise (fd, output_offset, clen,
+                                       POSIX_FADV_DONTNEED);
+#else
+              errno = ENOTSUP;
+#endif
+              output_offset += clen + pending;
+            }
+        }
+    }
+
+  return adv_ret != -1 ? true : false;
+}
+
 /* Read from FD into the buffer BUF of size SIZE, processing any
    signals that arrive before bytes are read.  Return the number of
    bytes read if successful, -1 (setting errno) on failure.  */
@@ -853,9 +953,7 @@ iwrite (int fd, char const *buf, size_t size)
          posix_fadvise to tell the system not to pollute the buffer
          cache with this data.  Don't bother to diagnose lseek or
          posix_fadvise failure. */
-      off_t off = lseek (STDOUT_FILENO, 0, SEEK_CUR);
-      if (0 <= off)
-        fdadvise (STDOUT_FILENO, off, 0, FADVISE_DONTNEED);
+      invalidate_cache (STDOUT_FILENO, 0);
 
       /* Attempt to ensure that that final block is committed
          to disk as quickly as possible.  */
@@ -884,6 +982,9 @@ iwrite (int fd, char const *buf, size_t size)
         total_written += nwritten;
     }
 
+  if (o_nocache && total_written)
+    invalidate_cache (fd, total_written);
+
   return total_written;
 }
 
@@ -1107,6 +1208,20 @@ scanargs (int argc, char *const *argv)
     error (EXIT_FAILURE, 0, _("cannot combine lcase and ucase"));
   if (multiple_bits_set (conversions_mask & (C_EXCL | C_NOCREAT)))
     error (EXIT_FAILURE, 0, _("cannot combine excl and nocreat"));
+  if (multiple_bits_set (input_flags & (O_DIRECT | O_NOCACHE))
+      || multiple_bits_set (output_flags & (O_DIRECT | O_NOCACHE)))
+    error (EXIT_FAILURE, 0, _("cannot combine direct and nocache"));
+
+  if (input_flags & O_NOCACHE)
+    {
+      i_nocache = true;
+      input_flags &= ~O_NOCACHE;
+    }
+  if (output_flags & O_NOCACHE)
+    {
+      o_nocache = true;
+      output_flags &= ~O_NOCACHE;
+    }
 }
 
 /* Fix up translation table. */
@@ -1689,6 +1804,9 @@ dd_copy (void)
 
       nread = iread_fnc (STDIN_FILENO, ibuf, input_blocksize);
 
+      if (nread >= 0 && i_nocache)
+        invalidate_cache (STDIN_FILENO, nread);
+
       if (nread == 0)
         break;			/* EOF.  */
 
@@ -1698,8 +1816,14 @@ dd_copy (void)
           if (conversions_mask & C_NOERROR)
             {
               print_stats ();
+              size_t bad_portion = input_blocksize - partread;
+
+              /* We already know this data is not cached,
+                 but call this so that correct offsets are maintained.  */
+              invalidate_cache (STDIN_FILENO, bad_portion);
+
               /* Seek past the bad block if possible. */
-              if (!advance_input_after_read_error (input_blocksize - partread))
+              if (!advance_input_after_read_error (bad_portion))
                 {
                   exit_status = EXIT_FAILURE;
 
@@ -1954,5 +2078,31 @@ main (int argc, char **argv)
 
   exit_status = dd_copy ();
 
+  if (max_records == 0)
+    {
+      /* Special case to invalidate cache to end of file.  */
+      if (i_nocache && !invalidate_cache (STDIN_FILENO, 0))
+        {
+          error (0, errno, _("failed to discard cache for: %s"),
+                 quote (input_file));
+          exit_status = EXIT_FAILURE;
+        }
+      if (o_nocache && !invalidate_cache (STDOUT_FILENO, 0))
+        {
+          error (0, errno, _("failed to discard cache for: %s"),
+                 quote (output_file));
+          exit_status = EXIT_FAILURE;
+        }
+    }
+  else if (max_records != (uintmax_t) -1)
+    {
+      /* Invalidate any pending region less that page size,
+         in case the kernel might round up.  */
+      if (i_nocache)
+        invalidate_cache (STDIN_FILENO, 0);
+      if (o_nocache)
+        invalidate_cache (STDOUT_FILENO, 0);
+    }
+
   quit (exit_status);
 }
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 28eafe8ae..99b46e8e8 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -352,6 +352,7 @@ TESTS =						\
   df/unreadable					\
   dd/direct					\
   dd/misc					\
+  dd/nocache					\
   dd/not-rewound				\
   dd/reblock					\
   dd/skip-seek					\
diff --git a/tests/dd/nocache b/tests/dd/nocache
new file mode 100755
index 000000000..b0d4cc107
--- /dev/null
+++ b/tests/dd/nocache
@@ -0,0 +1,58 @@
+#!/bin/sh
+# Ensure dd handles the 'nocache' flag
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ dd
+
+# This should not call posix_fadvise
+dd iflag=nocache oflag=nocache if=/dev/null of=/dev/null || fail=1
+
+# We should get an error for trying to process a pipe
+dd iflag=nocache count=0 && fail=1
+
+# O_DIRECT is orthogonal to drop cache so mutually exclusive
+dd iflag=nocache,direct if=/dev/null && fail=1
+
+# The rest ensure that the documented uses cases
+# proceed without error
+for f in ifile ofile; do
+  dd if=/dev/zero of=$f conv=fdatasync count=100 || framework_failure
+done
+
+# Advise to drop cache for whole file
+if ! dd if=ifile iflag=nocache count=0 2>err; then
+  if grep -F 'Operation not supported' err >/dev/null; then
+     warn_ 'skipping part; this file system lacks support for posix_fadvise()'
+     skip=1
+  else
+    fail=1
+  fi
+fi
+
+if test "$skip" != 1; then
+  # Ensure drop cache for whole file
+  dd of=ofile oflag=nocache conv=notrunc,fdatasync count=0 || fail=1
+
+  # Drop cache for part of file
+  dd if=ifile iflag=nocache skip=10 count=10 of=/dev/null || fail=1
+
+  # Stream data just using readahead cache
+  dd if=ifile of=ofile iflag=nocache oflag=nocache || fail=1
+fi
+
+Exit $fail