factor: fix integer validation and GMP fallback

In the recent factor rewrite, the GMP code
wasn't actually used; just an error was printed
on integer overflow.  While fixing that it was noticed
that correct input validation wasn't done in all cases
when falling back to the GMP code.

* src/factor.c (print_factors) Fallback to GMP on overflow.
(strto2uintmax): Scan the string for invalid characters,
so that case can be detected independently of overflow.
Return an error when an empty string is passed.
Also allow leading spaces and '+' in input numbers.
* tests/misc/factor.pl: Ensure the GMP code is exercised
when compiled in. Also add a test to verify leading
spaces and '+' are allowed.

1 files changed, 33 insertions, 5 deletions

diff --git a/src/factor.c b/src/factor.c
index d5f248a6a..73c59e9b9 100644
--- a/src/factor.c
+++ b/src/factor.c
@@ -2222,14 +2222,30 @@ static strtol_error
 strto2uintmax (uintmax_t *hip, uintmax_t *lop, const char *s)
 {
   unsigned int lo_carry;
-  uintmax_t hi, lo;
+  uintmax_t hi = 0, lo = 0;
 
-  strtol_error err;
+  strtol_error err = LONGINT_INVALID;
 
-  hi = lo = 0;
+  /* Skip initial spaces and '+'.  */
   for (;;)
     {
-      unsigned int c = *s++;
+      char c = *s;
+      if (c == ' ')
+        s++;
+      else if (c == '+')
+        {
+          s++;
+          break;
+        }
+      else
+        break;
+    }
+
+  /* Initial scan for invalid digits.  */
+  const char *p = s;
+  for (;;)
+    {
+      unsigned int c = *p++;
       if (c == 0)
         break;
 
@@ -2238,9 +2254,17 @@ strto2uintmax (uintmax_t *hip, uintmax_t *lop, const char *s)
           err = LONGINT_INVALID;
           break;
         }
-      c -= '0';
 
       err = LONGINT_OK;           /* we've seen at least one valid digit */
+    }
+
+  for (;err == LONGINT_OK;)
+    {
+      unsigned int c = *s++;
+      if (c == 0)
+        break;
+
+      c -= '0';
 
       if (UNLIKELY (hi > ~(uintmax_t)0 / 10))
         {
@@ -2343,6 +2367,10 @@ print_factors (const char *input)
         }
       break;
 
+    case LONGINT_OVERFLOW:
+      /* Try GMP.  */
+      break;
+
     default:
       error (0, 0, _("%s is not a valid positive integer"), quote (input));
       return false;