md5sum -c: ignore a line with a NUL byte among checksum hex digits

* src/md5sum.c (hex_digits): Require that all "digest_hex_bytes"
be hexadecimal digits, not just those before the first NUL byte.
This bug dates back to the original version:
3763a4f24eb21be40674d13ff7b04e078f473e85
* tests/misc/md5sum (nul-in-cksum): Test for the above.
* NEWS [Bug fixes]: Mention this.
Prompted by a report from Flóki Pálsson in
http://bugzilla.redhat.com/439531

3 files changed, 20 insertions, 2 deletions

diff --git a/NEWS b/NEWS
index 3a584e9cc..3cc7151d4 100644
--- a/NEWS
+++ b/NEWS
@@ -25,6 +25,13 @@ GNU coreutils NEWS                                    -*- outline -*-
   sha1sum, sha224sum, sha384sum, and sha512sum are affected, too.
   [bug introduced in coreutils-5.1.0]
 
+  md5sum -c would accept a NUL-containing checksum string like "abcd\0..."
+  and would unnecessarily read and compute the checksum of the named file,
+  and then compare that checksum to the invalid one: guaranteed to fail.
+  Now, it recognizes that the line is not valid and skips it.
+  sha1sum, sha224sum, sha384sum, and sha512sum are affected, too.
+  [bug present in the original version, in coreutils-4.5.1, 1995]
+
   "mkdir -Z x dir" no longer segfaults when diagnosing invalid context "x"
   mkfifo and mknod would fail similarly.  Now they're fixed.
 
diff --git a/src/md5sum.c b/src/md5sum.c
index ba762d149..f83a7b115 100644
--- a/src/md5sum.c
+++ b/src/md5sum.c
@@ -343,16 +343,19 @@ split_3 (char *s, size_t s_len,
   return true;
 }
 
+/* Return true if S is a NUL-terminated string of DIGEST_HEX_BYTES hex digits.
+   Otherwise, return false.  */
 static bool
 hex_digits (unsigned char const *s)
 {
-  while (*s)
+  unsigned int i;
+  for (i = 0; i < digest_hex_bytes; i++)
     {
       if (!isxdigit (*s))
         return false;
       ++s;
     }
-  return true;
+  return *s == '\0';
 }
 
 /* An interface to the function, DIGEST_STREAM.
diff --git a/tests/misc/md5sum b/tests/misc/md5sum
index 25069fd15..474656f24 100755
--- a/tests/misc/md5sum
+++ b/tests/misc/md5sum
@@ -66,6 +66,14 @@ my @Tests =
 				{AUX=> {f=> 'bar'}}, {EXIT=> 1}],
      ['bsd-segv', '--check', {IN=> {'z' => "MD5 ("}}, {EXIT=> 1},
       {ERR=> "$prog: z: no properly formatted MD5 checksum lines found\n"}],
+
+     # Ensure that when there's a NUL byte among the checksum hex digits
+     # we detect the invalid formatting and don't even open the file.
+     # Up to coreutils-6.10, this would report:
+     #   h: FAILED
+     #   md5sum: WARNING: 1 of 1 computed checksum did NOT match
+     ['nul-in-cksum', '--check', {IN=> {'h'=>("\0"x32)."  h\n"}}, {EXIT=> 1},
+      {ERR=> "$prog: h: no properly formatted MD5 checksum lines found\n"}],
     );
 
 # Insert the `--text' argument for each test.