sort: avoid a NaN-induced infloop

These commands would fail to terminate:
    yes -- -nan | head -156903 | sort -g > /dev/null
    echo nan > F; sort -m -g F F
That can happen with any strtold implementation that includes
uninitialized data in its return value.  The problem arises in the
mergefps function when bubble-sorting the two or more lines, each
from one of the input streams being merged: compare(a,b) returns 64,
yet compare(b,a) also returns a positive value.  With a broken
comparison function like that, the bubble sort never terminates.
Why do the long-double bit strings corresponding to two identical
"nan" strings not compare equal?  Because some parts of the result
are uninitialized and thus depend on the state of the stack.
For more details, see http://bugs.gnu.org/9612.
* src/sort.c (nan_compare): New function.
(general_numcompare): Use it rather than bare memcmp.
Reported by Aaron Denney in http://bugs.debian.org/642557.
* NEWS (Bug fixes): Mention it.
* tests/misc/sort-NaN-infloop: New file.
* tests/Makefile.am (TESTS): Add it.

4 files changed, 52 insertions, 1 deletions

diff --git a/NEWS b/NEWS
index 140e6faca..f05a08841 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,10 @@ GNU coreutils NEWS                                    -*- outline -*-
 
 * Noteworthy changes in release ?.? (????-??-??) [?]
 
+** Bug fixes
+
+  sort -g no longer infloops for certain inputs containing NaNs
+
 ** Improvements
 
   md5sum --check now supports the -r format from the corresponding BSD tool.
diff --git a/src/sort.c b/src/sort.c
index 3d3119d90..3e94a6e79 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -1910,6 +1910,24 @@ numcompare (char const *a, char const *b)
   return strnumcmp (a, b, decimal_point, thousands_sep);
 }
 
+/* Work around a problem whereby the long double value returned by glibc's
+   strtold ("NaN", ...) contains uninitialized bits: clear all bytes of
+   A and B before calling strtold.  FIXME: remove this function once
+   gnulib guarantees that strtold's result is always well defined.  */
+static int
+nan_compare (char const *sa, char const *sb)
+{
+  long_double a;
+  memset (&a, 0, sizeof a);
+  a = strtold (sa, NULL);
+
+  long_double b;
+  memset (&b, 0, sizeof b);
+  b = strtold (sb, NULL);
+
+  return memcmp (&a, &b, sizeof a);
+}
+
 static int
 general_numcompare (char const *sa, char const *sb)
 {
@@ -1935,7 +1953,7 @@ general_numcompare (char const *sa, char const *sb)
           : a == b ? 0
           : b == b ? -1
           : a == a ? 1
-          : memcmp (&a, &b, sizeof a));
+          : nan_compare (sa, sb));
 }
 
 /* Return an integer in 1..12 of the month name MONTH.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index eeb4cab2d..2cf409ae9 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -250,6 +250,7 @@ TESTS =						\
   misc/sort-unique				\
   misc/sort-unique-segv				\
   misc/sort-version				\
+  misc/sort-NaN-infloop				\
   split/filter					\
   split/suffix-length				\
   split/b-chunk					\
diff --git a/tests/misc/sort-NaN-infloop b/tests/misc/sort-NaN-infloop
new file mode 100755
index 000000000..ead871e43
--- /dev/null
+++ b/tests/misc/sort-NaN-infloop
@@ -0,0 +1,28 @@
+#!/bin/sh
+# exercise the NaN-infloop bug in coreutils-8.13
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ sort
+
+echo nan > F || fail=1
+printf 'nan\nnan\n' > exp || fail=1
+timeout 10 sort -g -m F F > out || fail=1
+
+compare out exp || fail=1
+
+Exit $fail