Changed HTTP API.

common/ChangeLog

@@ -1,3 +1,21 @@
+2006-08-14  Werner Koch  <wk@g10code.com>
+
+	* http.h (struct http_context_s): Moved to implementation.
+	* http.c (http_open): Changed call to return a context.
+	(http_open_document): Ditto.
+	(http_get_read_ptr, http_get_read_ptr, http_get_status_code): New.
+	(do_parse_uri): Replaced strlwr by straight code to ease
+	standalone use of this file.
+	(http_wait_response): Removed arg STATUS_CODE as it is available
+	through an accessor function. Adjusted caller.
+	(http_escape_string): New.
+
+	* estream.c (es_read_line): Renamed to ..
+	(doreadline): .. this.  Changed all callers.
+	(es_read_line): New.  This is theusual limited getline variabnt as
+	used at several places.  Here taken and adjusted from xreadline.c
+	(es_free): New.
+
 2006-08-11  Werner Koch  <wk@g10code.com>
 
 	* http.c: Major internal changes to optionallly support GNUTLS and

common/estream.c

@@ -1,5 +1,5 @@
-/* estream.c - Extended stream I/O/ Library
+/* estream.c - Extended Stream I/O Library
- * Copyright (C) 2004 g10 Code GmbH
+ * Copyright (C) 2004, 2006 g10 Code GmbH
  *
  * This file is part of Libestream.
  *
@@ -1501,9 +1501,9 @@ es_skip (estream_t stream, size_t size)
 
 
 static int
-es_read_line (estream_t ES__RESTRICT stream, size_t max_length,
+doreadline (estream_t ES__RESTRICT stream, size_t max_length,
-	      char *ES__RESTRICT *ES__RESTRICT line,
+             char *ES__RESTRICT *ES__RESTRICT line,
-	      size_t *ES__RESTRICT line_length)
+             size_t *ES__RESTRICT line_length)
 {
   size_t space_left;
   size_t line_size;
@@ -2386,7 +2386,7 @@ es_fgets (char *ES__RESTRICT s, int n, estream_t ES__RESTRICT stream)
       int err;
       
       ESTREAM_LOCK (stream);
-      err = es_read_line (stream, n, &s, NULL);
+      err = doreadline (stream, n, &s, NULL);
       ESTREAM_UNLOCK (stream);
       if (! err)
 	ret = s;
@@ -2420,7 +2420,7 @@ es_getline (char *ES__RESTRICT *ES__RESTRICT lineptr, size_t *ES__RESTRICT n,
   int err;
 
   ESTREAM_LOCK (stream);
-  err = es_read_line (stream, 0, &line, &line_n);
+  err = doreadline (stream, 0, &line, &line_n);
   ESTREAM_UNLOCK (stream);
   if (err)
     goto out;
@@ -2466,6 +2466,129 @@ es_getline (char *ES__RESTRICT *ES__RESTRICT lineptr, size_t *ES__RESTRICT n,
 }
 
 
+
+/* Same as fgets() but if the provided buffer is too short a larger
+   one will be allocated.  This is similar to getline. A line is
+   considered a byte stream ending in a LF.
+
+   If MAX_LENGTH is not NULL, it shall point to a value with the
+   maximum allowed allocation.  
+
+   Returns the length of the line. EOF is indicated by a line of
+   length zero. A truncated line is indicated my setting the value at
+   MAX_LENGTH to 0.  If the returned value is less then 0 not enough
+   memory was enable or another error occurred; ERRNO is then set
+   accordingly.
+
+   If a line has been truncated, the file pointer is moved forward to
+   the end of the line so that the next read starts with the next
+   line.  Note that MAX_LENGTH must be re-initialzied in this case.
+
+   The caller initially needs to provide the address of a variable,
+   initialized to NULL, at ADDR_OF_BUFFER and don't change this value
+   anymore with the following invocations.  LENGTH_OF_BUFFER should be
+   the address of a variable, initialized to 0, which is also
+   maintained by this function.  Thus, both paramaters should be
+   considered the state of this function.
+
+   Note: The returned buffer is allocated with enough extra space to
+   allow the caller to append a CR,LF,Nul.  The buffer should be
+   released using es_free.
+ */
+ssize_t
+es_read_line (estream_t stream, 
+              char **addr_of_buffer, size_t *length_of_buffer,
+              size_t *max_length)
+{
+  int c;
+  char  *buffer = *addr_of_buffer;
+  size_t length = *length_of_buffer;
+  size_t nbytes = 0;
+  size_t maxlen = max_length? *max_length : 0;
+  char *p;
+
+  if (!buffer)
+    { 
+      /* No buffer given - allocate a new one. */
+      length = 256;
+      buffer = MEM_ALLOC (length);
+      *addr_of_buffer = buffer;
+      if (!buffer)
+        {
+          *length_of_buffer = 0;
+          if (max_length)
+            *max_length = 0;
+          return -1;
+        }
+      *length_of_buffer = length;
+    }
+
+  if (length < 4)
+    {
+      /* This should never happen. If it does, the fucntion has been
+         called with wrong arguments. */
+      errno = EINVAL;
+      return -1;
+    }
+  length -= 3; /* Reserve 3 bytes for CR,LF,EOL. */
+
+  ESTREAM_LOCK (stream);
+  p = buffer;
+  while  ((c = es_getc_unlocked (stream)) != EOF)
+    {
+      if (nbytes == length)
+        { 
+          /* Enlarge the buffer. */
+          if (maxlen && length > maxlen) 
+            {
+              /* We are beyond our limit: Skip the rest of the line. */
+              while (c != '\n' && (c=es_getc_unlocked (stream)) != EOF)
+                ;
+              *p++ = '\n'; /* Always append a LF (we reserved some space). */
+              nbytes++;
+              if (max_length)
+                *max_length = 0; /* Indicate truncation. */
+              break; /* the while loop. */
+            }
+          length += 3; /* Adjust for the reserved bytes. */
+          length += length < 1024? 256 : 1024;
+          *addr_of_buffer = MEM_REALLOC (buffer, length);
+          if (!*addr_of_buffer)
+            {
+              int save_errno = errno;
+              MEM_FREE (buffer); 
+              *length_of_buffer = *max_length = 0;
+              ESTREAM_UNLOCK (stream);
+              errno = save_errno;
+              return -1;
+            }
+          buffer = *addr_of_buffer;
+          *length_of_buffer = length;
+          length -= 3; 
+          p = buffer + nbytes;
+	}
+      *p++ = c;
+      nbytes++;
+      if (c == '\n')
+        break;
+    }
+  *p = 0; /* Make sure the line is a string. */
+  ESTREAM_UNLOCK (stream);
+
+  return nbytes;
+}
+
+/* Wrapper around free() to match the memory allocation system used
+   by estream.  Should be used for all buffers returned to the caller
+   by libestream. */
+void
+es_free (void *a)
+{
+  if (a)
+    MEM_FREE (a);
+}
+
+
 int
 es_vfprintf (estream_t ES__RESTRICT stream, const char *ES__RESTRICT format,
 	     va_list ap)
@@ -2616,3 +2739,4 @@ es_opaque_get (estream_t stream)
 
   return opaque;
 }
+

common/estream.h

@@ -184,6 +184,10 @@ int es_fputs (const char *ES__RESTRICT s, estream_t ES__RESTRICT stream);
 ssize_t es_getline (char *ES__RESTRICT *ES__RESTRICT lineptr,
 		    size_t *ES__RESTRICT n,
 		    estream_t stream);
+ssize_t es_read_line (estream_t stream, 
+                      char **addr_of_buffer, size_t *length_of_buffer,
+                      size_t *max_length);
+void es_free (void *a);
 
 int es_fprintf (estream_t ES__RESTRICT stream,
 		const char *ES__RESTRICT format, ...);

common/http.c

@@ -156,8 +156,32 @@ typedef struct cookie_s *cookie_t;
 static gpg_error_t (*tls_callback) (http_t, gnutls_session_t, int);
 #endif /*HTTP_USE_GNUTLS*/
 
+/* Our handle context. */
+struct http_context_s 
+{
+  unsigned int status_code;
+  int sock;
+  int in_data;
+#ifdef HTTP_USE_ESTREAM
+  estream_t fp_read;
+  estream_t fp_write;
+  void *write_cookie;
+#else /*!HTTP_USE_ESTREAM*/
+  FILE *fp_read;
+  FILE *fp_write;
+#endif /*!HTTP_USE_ESTREAM*/
+  void *tls_context;
+  int is_http_0_9;
+  parsed_uri_t uri;
+  http_req_t req_type;
+  char *buffer;          /* Line buffer. */
+  size_t buffer_size;
+  unsigned int flags;
+};
 
 
+
+
 #ifdef HAVE_W32_SYSTEM
 static void
 deinit_sockets (void)
@@ -253,20 +277,27 @@ http_register_tls_callback ( gpg_error_t (*cb) (http_t, void *, int) )
 
 
 
+/* Start a HTTP retrieval and return on success in R_HD a context
+   pointer for completing the the request and to wait for the
+   response. */
 gpg_error_t
-http_open (http_t hd, http_req_t reqtype, const char *url, 
+http_open (http_t *r_hd, http_req_t reqtype, const char *url, 
            const char *auth, unsigned int flags, const char *proxy,
            void *tls_context)
 {
   gpg_error_t err;
+  http_t hd;
+  
+  *r_hd = NULL;
 
   if (!(reqtype == HTTP_REQ_GET || reqtype == HTTP_REQ_POST))
     return gpg_error (GPG_ERR_INV_ARG);
 
-  /* Initialize the handle. */
+  /* Create the handle. */
-  memset (hd, 0, sizeof *hd);
+  hd = xtrycalloc (1, sizeof *hd);
+  if (!hd)
+    return gpg_error_from_errno (errno);
   hd->sock = -1;
-  hd->initialized = 1;
   hd->req_type = reqtype;
   hd->flags = flags;
   hd->tls_context = tls_context;
@@ -284,8 +315,10 @@ http_open (http_t hd, http_req_t reqtype, const char *url,
       if (hd->fp_write)
         P_ES(fclose) (hd->fp_write);
       http_release_parsed_uri (hd->uri);
-      hd->initialized = 0;
+      xfree (hd);
     }
+  else
+    *r_hd = hd;
   return err;
 }
 
@@ -310,7 +343,7 @@ http_start_data (http_t hd)
 
 
 gpg_error_t
-http_wait_response (http_t hd, unsigned int *ret_status)
+http_wait_response (http_t hd)
 {
   gpg_error_t err;
 
@@ -370,9 +403,6 @@ http_wait_response (http_t hd, unsigned int *ret_status)
 #endif /*!HTTP_USE_ESTREAM*/
 
   err = parse_response (hd);
-  if (!err && ret_status)
-    *ret_status = hd->status_code;
-
   return err;
 }
 
@@ -382,19 +412,20 @@ http_wait_response (http_t hd, unsigned int *ret_status)
    be used as an HTTP proxy and any enabled $http_proxy gets
    ignored. */
 gpg_error_t
-http_open_document (http_t hd, const char *document, 
+http_open_document (http_t *r_hd, const char *document, 
                     const char *auth, unsigned int flags, const char *proxy,
                     void *tls_context)
 {
   gpg_error_t err;
 
-  err = http_open (hd, HTTP_REQ_GET, document, auth, flags, proxy,tls_context);
+  err = http_open (r_hd, HTTP_REQ_GET, document, auth, flags,
+                   proxy, tls_context);
   if (err)
     return err;
 
-  err = http_wait_response (hd, NULL);
+  err = http_wait_response (*r_hd);
   if (err)
-    http_close (hd, 0);
+    http_close (*r_hd, 0);
 
   return err;
 }
@@ -403,7 +434,7 @@ http_open_document (http_t hd, const char *document,
 void
 http_close (http_t hd, int keep_read_stream)
 {
-  if (!hd || !hd->initialized)
+  if (!hd)
     return;
   if (!hd->fp_read && !hd->fp_write && hd->sock != -1)
     sock_close (hd->sock);
@@ -413,11 +444,41 @@ http_close (http_t hd, int keep_read_stream)
     P_ES(fclose) (hd->fp_write);
   http_release_parsed_uri (hd->uri);
   xfree (hd->buffer);
-  hd->initialized = 0;
+  xfree (hd);
 }
 
 
+#ifdef HTTP_USE_ESTREAM
+estream_t
+http_get_read_ptr (http_t hd)
+{
+  return hd?hd->fp_read:NULL;
+}
+estream_t
+http_get_write_ptr (http_t hd)
+{
+  return hd?hd->fp_write:NULL;
+}
+#else /*!HTTP_USE_ESTREAM*/
+FILE *
+http_get_read_ptr (http_t hd)
+{
+  return hd?hd->fp_read:NULL;
+}
+FILE *
+http_get_write_ptr (http_t hd)
+{
+  return hd?hd->fp_write:NULL;
+}
+#endif /*!HTTP_USE_ESTREAM*/
+unsigned int
+http_get_status_code (http_t hd)
+{
+  return hd?hd->status_code:0;
+}
 
+
+
 /*
  * Parse an URI and put the result into the newly allocated RET_URI.
  * The caller must always use release_parsed_uri() to releases the
@@ -452,7 +513,7 @@ static gpg_error_t
 do_parse_uri (parsed_uri_t uri, int only_local_part)
 {
   uri_tuple_t *tail;
-  char *p, *p2, *p3;
+  char *p, *p2, *p3, *pp;
   int n;
 
   p = uri->buffer;
@@ -474,7 +535,8 @@ do_parse_uri (parsed_uri_t uri, int only_local_part)
       if (!(p2 = strchr (p, ':')) || p2 == p)
 	return gpg_error (GPG_ERR_BAD_URI); /* No scheme. */
       *p2++ = 0;
-      strlwr (p);
+      for (pp=p; *pp; pp++)
+       *pp = tolower (*(unsigned char*)pp);
       uri->scheme = p;
       if (!strcmp (uri->scheme, "http"))
         uri->port = 80;
@@ -511,7 +573,8 @@ do_parse_uri (parsed_uri_t uri, int only_local_part)
               p = p3;
             }
 
-	  strlwr (p);
+          for (pp=p; *pp; pp++)
+            *pp = tolower (*(unsigned char*)pp);
 	  uri->host = p;
 	  if ((p3 = strchr (p, ':')))
 	    {
@@ -648,6 +711,29 @@ insert_escapes (char *buffer, const char *string,
 }
 
 
+/* Allocate a new string from STRING using standard HTTP escaping as
+   well as escaping of characters given in SPECIALS.  A common pattern
+   for SPECIALS is "%;?&=". However it depends on the needs, for
+   example "+" and "/: often needs to be escaped too.  Returns NULL on
+   failure and sets ERRNO. */
+char *
+http_escape_string (const char *string, const char *specials)
+{
+  int n;
+  char *buf;
+
+  n = insert_escapes (NULL, string, specials);
+  buf = xtrymalloc (n+1);
+  if (buf)
+    {
+      insert_escapes (buf, string, specials);
+      buf[n] = 0;
+    }
+  return buf;
+}
+
+
+
 static uri_tuple_t
 parse_tuple (char *string)
 {
@@ -1095,6 +1181,9 @@ parse_response (http_t hd)
 	return gpg_error (GPG_ERR_TRUNCATED); /* Line has been truncated. */
       if (!len)
 	return gpg_error (GPG_ERR_EOF);
+      if ( (hd->flags & HTTP_FLAG_LOG_RESP) )
+        log_info ("RESP: `%.*s'\n",
+                  (int)strlen(line)-(*line&&line[1]?2:0),line);
     }
   while (!*line);
 
@@ -1138,6 +1227,9 @@ parse_response (http_t hd)
       /* Trim line endings of empty lines. */
       if ((*line == '\r' && line[1] == '\n') || *line == '\n')
 	*line = 0;
+      if ( (hd->flags & HTTP_FLAG_LOG_RESP) )
+        log_info ("RESP: `%.*s'\n",
+                  (int)strlen(line)-(*line&&line[1]?2:0),line);
     }
   while (len && *line);
 
@@ -1603,7 +1695,7 @@ main (int argc, char **argv)
   int rc;
   parsed_uri_t uri;
   uri_tuple_t r;
-  struct http_context_s hd;
+  http_t hd;
   int c;
   gnutls_session_t tls_session = NULL;
 #ifdef HTTP_USE_GNUTLS
@@ -1706,10 +1798,11 @@ main (int argc, char **argv)
       log_error ("can't get `%s': %s\n", *argv, gpg_strerror (rc));
       return 1;
     }
-  log_info ("open_http_document succeeded; status=%u\n", hd.status_code);
+  log_info ("open_http_document succeeded; status=%u\n",
-  while ((c = P_ES(getc) (hd.fp_read)) != EOF)
+            http_get_status_code (hd));
+  while ((c = P_ES(getc) (http_get_read_ptr (hd))) != EOF)
     putchar (c);
-  http_close (&hd, 0);
+  http_close (hd, 0);
 
 #ifdef HTTP_USE_GNUTLS
   gnutls_deinit (tls_session);

common/http.h

@@ -65,31 +65,11 @@ enum
   { 
     HTTP_FLAG_TRY_PROXY = 1,
     HTTP_FLAG_NO_SHUTDOWN = 2,
-    HTTP_FLAG_TRY_SRV = 4
+    HTTP_FLAG_TRY_SRV = 4,
+    HTTP_FLAG_LOG_RESP = 8
   };
 
-struct http_context_s 
+struct http_context_s;
-{
-  int initialized;
-  unsigned int status_code;
-  int sock;
-  int in_data;
-#ifdef HTTP_USE_ESTREAM
-  estream_t fp_read;
-  estream_t fp_write;
-  void *write_cookie;
-#else /*!HTTP_USE_ESTREAM*/
-  FILE *fp_read;
-  FILE *fp_write;
-#endif /*!HTTP_USE_ESTREAM*/
-  void *tls_context;
-  int is_http_0_9;
-  parsed_uri_t uri;
-  http_req_t req_type;
-  char *buffer;          /* Line buffer. */
-  size_t buffer_size;
-  unsigned int flags;
-};
 typedef struct http_context_s *http_t;
 
 void http_register_tls_callback (gpg_error_t (*cb) (http_t, void *, int));
@@ -98,7 +78,7 @@ gpg_error_t http_parse_uri (parsed_uri_t *ret_uri, const char *uri);
 
 void http_release_parsed_uri (parsed_uri_t uri);
 
-gpg_error_t http_open (http_t hd, http_req_t reqtype,
+gpg_error_t http_open (http_t *r_hd, http_req_t reqtype,
                        const char *url,
                        const char *auth,
                        unsigned int flags,
@@ -107,15 +87,27 @@ gpg_error_t http_open (http_t hd, http_req_t reqtype,
 
 void http_start_data (http_t hd);
 
-gpg_error_t http_wait_response (http_t hd, unsigned int *ret_status);
+gpg_error_t http_wait_response (http_t hd);
 
 void http_close (http_t hd, int keep_read_stream);
 
-gpg_error_t http_open_document (http_t hd,
+gpg_error_t http_open_document (http_t *r_hd,
                                 const char *document,
                                 const char *auth,
                                 unsigned int flags,
                                 const char *proxy,
                                 void *tls_context);
 
+#ifdef HTTP_USE_ESTREAM
+estream_t http_get_read_ptr (http_t hd);
+estream_t http_get_write_ptr (http_t hd);
+#else /*!HTTP_USE_ESTREAM*/
+FILE *http_get_read_ptr (http_t hd);
+FILE *http_get_write_ptr (http_t hd);
+#endif /*!HTTP_USE_ESTREAM*/
+unsigned int http_get_status_code (http_t hd);
+
+char *http_escape_string (const char *string, const char *specials);
+
+
 #endif /*GNUPG_COMMON_HTTP_H*/

common/xreadline.c

@@ -40,8 +40,8 @@
    memory was enable and ERRNO is set accordingly.
 
    If a line has been truncated, the file pointer is moved forward to
-   the end of the line so that the next read start with the next
+   the end of the line so that the next read starts with the next
-   line.  Note that MAX_LENGTH must be re-initialzied in this case..
+   line.  Note that MAX_LENGTH must be re-initialzied in this case.
 
    Note: The returned buffer is allocated with enough extra space to
    append a CR,LF,Nul