* new http option for debug. This is mostly useful to debug XOAUTH2

     autentication and reveals sensitive login information. Use with care.
     Remove your .pine-debug file after using this option.

12 files changed, 73 insertions, 16 deletions

diff --git a/alpine/alpine.c b/alpine/alpine.c
index 7c63e77..3831b89 100644
--- a/alpine/alpine.c
+++ b/alpine/alpine.c
@@ -388,6 +388,11 @@ main(int argc, char **argv)
 #endif
       mail_parameters(NULL, SET_TCPDEBUG, (void *) TRUE);
 
+#ifndef DEBUGJOURNAL
+    if(ps_global->debug_http)
+#endif
+      mail_parameters(NULL, SET_HTTPDEBUG, (void *) TRUE);
+
 #ifdef	_WINDOWS
     mswin_setdebug(debug, debugfile);
     mswin_setdebugoncallback (imap_telemetry_on);
@@ -821,7 +826,7 @@ main(int argc, char **argv)
 				pine_pico_puts, pine_pico_seek, NULL, NULL);
 
 #ifdef	DEBUG
-    if(ps_global->debug_imap > 4 || debug > 9){
+    if(ps_global->debug_imap > 4 || debug > 9 || ps_global->debug_http > 0){
 	q_status_message(SM_ORDER | SM_DING, 5, 9,
 	      _("Warning: sensitive authentication data included in debug file"));
 	flush_status_messages(0);
@@ -1808,6 +1813,11 @@ main_menu_screen(struct pine *pine_state)
 	    else if(debug <= 7 && olddebug > 7 && !ps_global->debugmem)
 	      mail_parameters(NULL, SET_TCPDEBUG, (void *) FALSE);
 
+	    if(debug > 7 && olddebug <= 7)
+	      mail_parameters(NULL, SET_HTTPDEBUG, (void *) TRUE);
+	    else if(debug <= 7 && olddebug > 7 && !ps_global->debugmem)
+	      mail_parameters(NULL, SET_HTTPDEBUG, (void *) FALSE);
+
 	    dprint((1, "*** Debug level set to %d ***\n", debug));
 	    if(debugfile)
 	      fflush(debugfile);
diff --git a/alpine/arg.c b/alpine/arg.c
index 8e00a21..72ab064 100644
--- a/alpine/arg.c
+++ b/alpine/arg.c
@@ -107,7 +107,7 @@ N_("\t\toption on the command line"),
 N_(" -bail\t\tExit if pinerc file doesn't already exist"),
 #ifdef	DEBUG
 N_(" -d n\t\tDebug - set debug level to 'n', or use the following:"),
-N_(" -d keywords...\tflush,timestamp,imap=0..4,tcp,numfiles=0..31,verbose=0..9"),
+N_(" -d keywords...\tflush,timestamp,imap=0..4,tcp,http,numfiles=0..31,verbose=0..9"),
 #endif
 N_(" -f <folder>\tFolder - give folder name to open"),
 N_(" -c <number>\tContext - which context to apply to -f arg"),
@@ -1144,6 +1144,10 @@ process_debug_str(char *debug_str)
 			|| struncmp(*p, "tcpdebug", 8) == 0){
 		    ps_global->debug_tcp = 1;
 		}
+		else if(struncmp(*p, "http", 4) == 0 
+			|| struncmp(*p, "httpdebug", 9) == 0){
+		    ps_global->debug_http = 1;
+		}
 		else if(struncmp(*p, "verbose", 7) == 0){
 		    q = *p + 7;
 		    if(!*q || !*(q+1) || *q != '=' ||
@@ -1213,8 +1217,10 @@ process_debug_str(char *debug_str)
 	}
 	else{
 	    debug = atoi(debug_str);
-	    if(debug > 9)
+	    if(debug > 9){
 	      ps_global->debug_imap = 5;
+	      ps_global->debug_http = 1;
+	    }
 	    else if(debug > 7)
 	      ps_global->debug_imap = 4;
 	    else if(debug > 6)
diff --git a/alpine/imap.c b/alpine/imap.c
index 00b5c99..2e02ef6 100644
--- a/alpine/imap.c
+++ b/alpine/imap.c
@@ -1265,7 +1265,7 @@ mm_notify(MAILSTREAM *stream, char *string, long int errflg)
     /* be sure to log the message... */
 #ifdef DEBUG
     if(ps_global->debug_imap || ps_global->debugmem)
-      dprint((errflg == TCPDEBUG ? 7 : 2,
+      dprint((errflg == TCPDEBUG || errflg == HTTPDEBUG ? 7 : 2,
 	      "IMAP %2.2d:%2.2d:%2.2d %d/%d mm_notify %s: %s: %s\n",
 	      tm_now->tm_hour, tm_now->tm_min, tm_now->tm_sec,
 	      tm_now->tm_mon+1, tm_now->tm_mday,
@@ -1274,7 +1274,8 @@ mm_notify(MAILSTREAM *stream, char *string, long int errflg)
 	        (errflg == WARN)     ? "warning" :
 	         (errflg == PARSE)    ? "parse"   :
 		  (errflg == TCPDEBUG) ? "tcp"     :
-		   (errflg == BYE)      ? "bye"     : "unknown",
+		   (errflg == HTTPDEBUG) ? "http"   :
+		    (errflg == BYE)      ? "bye"     : "unknown",
 	      (stream && stream->mailbox) ? stream->mailbox : "-no folder-",
 	      string ? string : "?"));
 #endif
@@ -1357,7 +1358,9 @@ mm_log(char *string, long int errflg)
     tm_now = localtime(&now);
 
     dprint((((errflg == TCPDEBUG) && ps_global->debug_tcp) ? 1 :
-           (errflg == TCPDEBUG) ? 10 : 2,
+           (errflg == TCPDEBUG) ? 10 : 
+	   ((errflg == HTTPDEBUG) && ps_global->debug_http) ? 1 :
+	   (errflg == HTTPDEBUG) ? 10 :  2,
 	    "IMAP %2.2d:%2.2d:%2.2d %d/%d mm_log %s: %s\n",
 	    tm_now->tm_hour, tm_now->tm_min, tm_now->tm_sec,
 	    tm_now->tm_mon+1, tm_now->tm_mday,
@@ -1366,7 +1369,8 @@ mm_log(char *string, long int errflg)
 	      (errflg == WARN)     ? "warning" :
 	       (errflg == PARSE)    ? "parse"   :
 		(errflg == TCPDEBUG) ? "tcp"     :
-		 (errflg == BYE)      ? "bye"     : "unknown",
+		 (errflg == HTTPDEBUG) ? "http"   :
+		  (errflg == BYE)       ? "bye"    : "unknown",
 	    string ? string : "?"));
 
     if(errflg == ERROR && !strncmp(string, "[TRYCREATE]", 11)){
diff --git a/alpine/osdep/debuging.c b/alpine/osdep/debuging.c
index f7a2971..f7f83ff 100644
--- a/alpine/osdep/debuging.c
+++ b/alpine/osdep/debuging.c
@@ -74,7 +74,7 @@ init_debug(void)
     char newfname[MAXPATH+1], filename[MAXPATH+1], *dfile = NULL;
     int i, fd;
 
-    if(!((debug || ps_global->debug_imap || ps_global->debug_tcp) &&  ps_global->write_debug_file))
+    if(!((debug || ps_global->debug_imap || ps_global->debug_tcp || ps_global->debug_http) &&  ps_global->write_debug_file))
       return;
 
     for(i = ps_global->debug_nfiles - 1; i > 0; i--){
@@ -130,7 +130,8 @@ init_debug(void)
 	dprint((0, "Starting after the reading_pinerc calls, the data in this file should\nbe encoded as UTF-8. Before that it will be in the user's native charset.\n"));
 	if(dfile && (debug > DEFAULT_DEBUG ||
 		     ps_global->debug_imap > 0 ||
-		     ps_global->debug_tcp > 0)){
+		     ps_global->debug_tcp > 0 ||
+		     ps_global->debug_http > 0)){
 	    snprintf(newfname, sizeof(newfname), "Debug file: %s (level=%d imap=%d)", dfile,
 		     debug, ps_global->debug_imap);
 	    init_error(ps_global, SM_ORDER, 3, 5, newfname);
@@ -258,6 +259,7 @@ do_debug(FILE *debug_fp)
     if(debug <= DEFAULT_DEBUG
        && !ps_global->debug_flush
        && !ps_global->debug_tcp
+       && !ps_global->debug_http
        && !ps_global->debug_timestamp
        && !ps_global->debug_imap
        && ok
diff --git a/imap/src/c-client/c-client.h b/imap/src/c-client/c-client.h
index b279bff..b190db8 100644
--- a/imap/src/c-client/c-client.h
+++ b/imap/src/c-client/c-client.h
@@ -43,6 +43,7 @@ extern "C" {
 #include "rfc822.h"		/* RFC822 and MIME routines */
 #include "smtp.h"		/* SMTP sending routines */
 #include "nntp.h"		/* NNTP sending routines */
+#include "http.h"		/* HTTP management routines */
 #include "utf8.h"		/* Unicode and charset routines */
 #include "utf8aux.h"		/* Unicode auxiliary routines */
 #include "misc.h"		/* miscellaneous utility routines */
diff --git a/imap/src/c-client/http.c b/imap/src/c-client/http.c
index fef3c7f..cb34f89 100644
--- a/imap/src/c-client/http.c
+++ b/imap/src/c-client/http.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2020 Eduardo Chappa
+ * Copyright 2018-2021 Eduardo Chappa
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -11,10 +11,11 @@
 #include <ctype.h>
 #include <stdio.h>
 #include <time.h>
-#include "c-client.h"
+#include "c-client.h"	/* this includes http.h */
 #include "flstring.h"
 #include "netmsg.h"
-#include "http.h"
+
+unsigned long http_debug;
 
 //char t[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%&'*+-.^_`|~";
 static char http_notok[] = "\1\2\3\4\5\6\7\10\11\12\13\14\15\16\17\20\21\22\23\24\25\26\27\30\31\32\33\34\35\36\37\40\42\50\51\54\57\72\73\74\75\76\77\100\133\134\135\173\175\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377";
@@ -107,7 +108,6 @@ unsigned char *http_response_from_reply(HTTPSTREAM *);
 
 /* HTTP function prototypes */
 int http_valid_net_parse (unsigned char *, NETMBX *);
-void *http_parameters (long function,void *value);
 
 long http_send (HTTPSTREAM *, HTTP_REQUEST_S *);
 long http_reply (HTTPSTREAM *);
@@ -131,6 +131,19 @@ PARAMETER *http_parse_parameter(unsigned char *, int);
 
 void http_parse_headers(HTTPSTREAM *);
 
+void *
+http_parameters (long function,void *value)
+{
+   void *ret = NIL;
+   switch((int) function){
+	case SET_HTTPDEBUG: http_debug = (long) value;
+	case GET_HTTPDEBUG: ret = (void *) http_debug;
+			    break;
+   }
+   return ret;
+}
+
+
 unsigned char *
 http_response_from_reply(HTTPSTREAM *stream)
 {
@@ -941,6 +954,7 @@ http_open (unsigned char *url)
       http_close(stream);
       stream = NIL;
   }
+  stream->debug = http_debug;
   return stream;
 }
 
@@ -1104,7 +1118,9 @@ http_send (HTTPSTREAM *stream, HTTP_REQUEST_S *req)
     buffer_add(&s, req->request); buffer_add(&s, "\015\012");
     buffer_add(&s, req->header); buffer_add(&s, "\015\012");
     buffer_add(&s, req->body); buffer_add(&s, "\015\012");
-    mm_log(s, TCPDEBUG);
+
+    if(stream->debug) mm_log(s, HTTPDEBUG);
+
     ret = net_soutr (stream->netstream,s)
 	  ? http_reply (stream)
 	  : http_fake (stream,"http connection broken in command");
@@ -1159,6 +1175,8 @@ http_reply (HTTPSTREAM *stream)
   if (stream->response) fs_give ((void **) &stream->response);
   stream->response = (unsigned char *) net_getline(stream->netstream);
 
+  if(stream->debug) mm_log(stream->response, HTTPDEBUG);
+
   if(stream->response){
      buffer_add(&stream->reply, stream->response);
      buffer_add(&stream->reply, "\015\012");
@@ -1178,6 +1196,7 @@ http_reply (HTTPSTREAM *stream)
     if(stream->response){
        buffer_add(&stream->reply, stream->response);
        http_add_header_data(stream, stream->response);
+       if(stream->debug) mm_log(stream->response, HTTPDEBUG);
     }
      buffer_add(&stream->reply, "\015\012");
 //    save_header(stream->headers, stream->response);
@@ -1190,7 +1209,10 @@ http_reply (HTTPSTREAM *stream)
      size = atol(stream->header->content_length->p->vp->value);
      if (stream->response) fs_give ((void **) &stream->response);
      stream->response = (unsigned char *) net_getsize (stream->netstream, size);
-     if(stream->response) buffer_add(&stream->reply, stream->response);
+     if(stream->response){
+	buffer_add(&stream->reply, stream->response);
+	if(stream->debug) mm_log(stream->response, HTTPDEBUG);
+     }
   }
   else if (stream->header->transfer_encoding){
      HTTP_PARAM_LIST_S *p = stream->header->transfer_encoding->p;
@@ -1211,6 +1233,7 @@ http_reply (HTTPSTREAM *stream)
 	     fs_give ((void **) &stream->response);
 	     stream->response = (unsigned char *) net_getsize (stream->netstream, size);
 	     buffer_add(&stream->reply, stream->response);
+	     if(stream->debug) mm_log(stream->response, HTTPDEBUG);
 	  }
 	  if(size == 0L) done++;
 	}
diff --git a/imap/src/c-client/http.h b/imap/src/c-client/http.h
index 41b1684..9c7464e 100644
--- a/imap/src/c-client/http.h
+++ b/imap/src/c-client/http.h
@@ -90,6 +90,7 @@ typedef struct http_status_s {
 typedef struct http_stream {
   NETSTREAM *netstream;
   HTTP_HEADER_DATA_S *header;	/* headers sent by the server */
+  unsigned int debug : 1;	/* send debug information */
   char *url;		/* original url */
   char *urlhost;	/* get original host */
   char *urltail;	/* the part of the URL after the original host */
@@ -115,5 +116,7 @@ void http_close (HTTPSTREAM *stream);
 HTTP_PARAM_S *http_param_get(int);
 void http_param_free(HTTP_PARAM_S **);
 
+void *http_parameters (long,void *);
+
 /* Ugghh.... just construct the URL for a get request */
 unsigned char *http_get_param_url(unsigned char *, HTTP_PARAM_S *);
diff --git a/imap/src/c-client/mail.c b/imap/src/c-client/mail.c
index 981025c..f3e14fe 100644
--- a/imap/src/c-client/mail.c
+++ b/imap/src/c-client/mail.c
@@ -701,6 +701,7 @@ void *mail_parameters (MAILSTREAM *stream,long function,void *value)
     if ((r = smtp_parameters (function,value)) != NULL) ret = r;
     if ((r = env_parameters (function,value)) != NULL) ret = r;
     if ((r = tcp_parameters (function,value)) != NULL) ret = r;
+    if ((r = http_parameters (function,value)) != NULL) ret = r;
     if ((r = utf8_parameters (function,value)) != NULL) ret = r;
     if (stream && stream->dtb) {/* if have stream, do for its driver only */
       if ((r = (*stream->dtb->parameters) (function,value)) != NULL) ret = r;
diff --git a/imap/src/c-client/mail.h b/imap/src/c-client/mail.h
index 79f0b3f..ce5e284 100644
--- a/imap/src/c-client/mail.h
+++ b/imap/src/c-client/mail.h
@@ -160,6 +160,8 @@
 #define SET_OA2CLIENTINFO (long) 172
 #define GET_OA2DEVICEINFO (long) 173
 #define SET_OA2DEVICEINFO (long) 174
+#define GET_HTTPDEBUG	  (long) 175
+#define SET_HTTPDEBUG	  (long) 176
 
 	/* 2xx: environment */
 #define GET_USERNAME (long) 201
@@ -573,6 +575,7 @@
 #define PARSE (long) 3		/* mm_log parse error type */
 #define BYE (long) 4		/* mm_notify stream dying */
 #define TCPDEBUG (long) 5	/* mm_log TCP debug babble */
+#define HTTPDEBUG (long) 6	/* mm_log HTTP debug babble */
 
 
 /* Bits from mail_parse_flags().  Don't change these, since the header format
diff --git a/imap/src/c-client/oauth2_aux.c b/imap/src/c-client/oauth2_aux.c
index 420f3be..b1af30b 100644
--- a/imap/src/c-client/oauth2_aux.c
+++ b/imap/src/c-client/oauth2_aux.c
@@ -20,7 +20,7 @@
  *    the deviceinfo method.
  */
 
-#include "http.h"
+/* http.h is supposed to be included, typically by including c-client.h */
 #include "json.h"
 #include "oauth2_aux.h"
 
diff --git a/pith/pine.hlp b/pith/pine.hlp
index fc6ed96..3904fe6 100644
--- a/pith/pine.hlp
+++ b/pith/pine.hlp
@@ -194,6 +194,9 @@ New features include:
      old Alpine behavior. See 
      <A href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=981781">this</A>
      report for more background information.
+<LI> Add the capability to record http debug. This is necessary to debug XOAUTH2
+     authemtication, and records sensitive login information. Do not share your
+     debug file if you use this form of debug.
 </UL>
 
 <P>
diff --git a/pith/state.h b/pith/state.h
index 24bc890..0ce781d 100644
--- a/pith/state.h
+++ b/pith/state.h
@@ -237,6 +237,7 @@ struct pine {
     unsigned 	 debug_timestamp:1;
     unsigned 	 debug_flush:1;
     unsigned 	 debug_tcp:1;
+    unsigned 	 debug_http:1;
     unsigned 	 debug_imap:3;
     unsigned 	 debug_nfiles:5;
     unsigned     debugmem:1;