Send an IMAP command, but don't wait for responses.

	* providers/imap/camel-imap-command.c (camel_imap_command_start):
	Send an IMAP command, but don't wait for responses.
	(camel_imap_command_response): Read a single line of response from
	the server.
	(camel_imap_command, etc): Reimplement in terms of the new code.

	* providers/imap/camel-imap-folder.c (imap_rescan): Use
	camel_imap_command_start and camel_imap_command_response, and
	call camel_operation_progress after each line read from the
	server.
	(imap_update_summary): Likewise, although with more fudging on the
	precentages... Also, fix this so that if none of the new messages
	are cached, it only does a single FETCH, and if some of them are
	cached, it does two FETCHes (one to get the UIDs, FLAGS, and
	SIZEs, and another to get the bodies of all of the messages that
	aren't cached now that it knows the relevant UIDs). This should
	speed up startup a bunch (especially if you have high bandwidth
	but also high latency to the IMAP server).

svn path=/trunk/; revision=11430

4 files changed, 483 insertions, 233 deletions

diff --git a/camel/providers/imap/camel-imap-command.c b/camel/providers/imap/camel-imap-command.c
index db113e0fd7..428ecf565d 100644
--- a/camel/providers/imap/camel-imap-command.c
+++ b/camel/providers/imap/camel-imap-command.c
@@ -40,36 +40,34 @@
 #include "camel-imap-private.h"
 #include <camel/camel-exception.h>
 
+static gboolean imap_command_start (CamelImapStore *store, CamelFolder *folder,
+				    const char *cmd, CamelException *ex);
+CamelImapResponse *imap_read_response (CamelImapStore *store,
+				       CamelException *ex);
 static char *imap_read_untagged (CamelImapStore *store, char *line,
 				 CamelException *ex);
-static CamelImapResponse *imap_read_response (CamelImapStore *store,
-					      CamelException *ex);
 static char *imap_command_strdup_vprintf (CamelImapStore *store,
 					  const char *fmt, va_list ap);
+static char *imap_command_strdup_printf (CamelImapStore *store,
+					 const char *fmt, ...);
 
 /**
- * camel_imap_command: Send a command to a IMAP server and get a response
+ * camel_imap_command:
  * @store: the IMAP store
  * @folder: The folder to perform the operation in (or %NULL if not
  * relevant).
  * @ex: a CamelException
- * @fmt: an sort of printf-style format string, followed by arguments
+ * @fmt: a sort of printf-style format string, followed by arguments
  *
- * This function makes sure that @folder (if non-%NULL) is the
- * currently-selected folder on @store and then sends the IMAP command
- * specified by @fmt and the following arguments. It then reads the
- * server's response(s) and parses the final result.
+ * This function calls camel_imap_command_start() to send the
+ * command, then reads the complete response to it using
+ * camel_imap_command_response() and returns a CamelImapResponse
+ * structure.
  *
  * As a special case, if @fmt is %NULL, it will just select @folder
  * and return the response from doing so.
- * 
- * @fmt can include the following %-escapes ONLY:
- *	%s, %d, %%: as with printf
- *	%S: an IMAP "string" (quoted string or literal)
  *
- * %S strings will be passed as literals if the server supports LITERAL+
- * and quoted strings otherwise. (%S does not support strings that
- * contain newlines.)
+ * See camel_imap_command_start() for details on @fmt.
  *
  * On success, the store's command_lock will be locked. It will be freed
  * when you call camel_imap_response_free. (The lock is recursive, so
@@ -84,73 +82,123 @@ CamelImapResponse *
 camel_imap_command (CamelImapStore *store, CamelFolder *folder,
 		    CamelException *ex, const char *fmt, ...)
 {
-	gchar *cmdbuf;
 	va_list ap;
-	CamelException internal_ex;
+	char *cmd;
 
 	CAMEL_IMAP_STORE_LOCK (store, command_lock);
 
-	/* Check for current folder */
-	if (folder && (!fmt || folder != store->current_folder)) {
-		CamelImapResponse *response;
-
+	if (fmt) {
+		va_start (ap, fmt);
+		cmd = imap_command_strdup_vprintf (store, fmt, ap);
+		va_end (ap);
+	} else {
 		if (store->current_folder) {
 			camel_object_unref (CAMEL_OBJECT (store->current_folder));
 			store->current_folder = NULL;
 		}
-		response = camel_imap_command (store, NULL, ex, "SELECT %S",
-					       folder->full_name);
-		if (!response) {
-			CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
-			return NULL;
-		}
 		store->current_folder = folder;
 		camel_object_ref (CAMEL_OBJECT (folder));
+		cmd = imap_command_strdup_printf (store, "SELECT %S",
+						  folder->full_name);
+	}
 
-		camel_imap_folder_selected (folder, response, ex);
-		if (!fmt) {
-			/* This undoes the level of locking we did,
-			 * but not the level of locking associated with
-			 * "response".
-			 */
-			CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
-			return response;
-		}
-
-		/* Contrariwise, this undoes "response"s lock,
-		 * but not our own.
-		 */
-		camel_imap_response_free (store, response);
+	if (!imap_command_start (store, folder, cmd, ex)) {
+		g_free (cmd);
+		CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
+		return NULL;
 	}
+	g_free (cmd);
+
+	return imap_read_response (store, ex);
+}
+
+/**
+ * camel_imap_command_start:
+ * @store: the IMAP store
+ * @folder: The folder to perform the operation in (or %NULL if not
+ * relevant).
+ * @ex: a CamelException
+ * @fmt: a sort of printf-style format string, followed by arguments
+ *
+ * This function makes sure that @folder (if non-%NULL) is the
+ * currently-selected folder on @store and then sends the IMAP command
+ * specified by @fmt and the following arguments.
+ *
+ * @fmt can include the following %-escapes ONLY:
+ *	%s, %d, %%: as with printf
+ *	%S: an IMAP "string" (quoted string or literal)
+ *
+ * %S strings will be passed as literals if the server supports LITERAL+
+ * and quoted strings otherwise. (%S does not support strings that
+ * contain newlines.)
+ *
+ * On success, the store's command_lock will be locked. It will be
+ * freed when %CAMEL_IMAP_RESPONSE_TAGGED or %CAMEL_IMAP_RESPONSE_ERROR
+ * is returned from camel_imap_command_response(). (The lock is
+ * recursive, so callers can grab and release it themselves if they
+ * need to run multiple commands atomically.)
+ *
+ * Return value: %TRUE if the command was sent successfully, %FALSE if
+ * an error occurred (in which case @ex will be set).
+ **/
+gboolean
+camel_imap_command_start (CamelImapStore *store, CamelFolder *folder,
+			  CamelException *ex, const char *fmt, ...)
+{
+	va_list ap;
+	char *cmd;
+	gboolean ok;
 
-	/* Send the command */
 	va_start (ap, fmt);
-	cmdbuf = imap_command_strdup_vprintf (store, fmt, ap);
+	cmd = imap_command_strdup_vprintf (store, fmt, ap);
 	va_end (ap);
 
-	camel_exception_init (&internal_ex);
-	camel_remote_store_send_string (CAMEL_REMOTE_STORE (store), &internal_ex,
-					"%c%.5d %s\r\n", store->tag_prefix,
-					store->command++, cmdbuf);
-	g_free (cmdbuf);
-	if (camel_exception_is_set (&internal_ex)) {
-		camel_exception_xfer (ex, &internal_ex);
+	CAMEL_IMAP_STORE_LOCK (store, command_lock);
+	ok = imap_command_start (store, folder, cmd, ex);
+	g_free (cmd);
+
+	if (!ok)
 		CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
-		return NULL;
+	return ok;
+}
+
+static gboolean
+imap_command_start (CamelImapStore *store, CamelFolder *folder,
+		    const char *cmd, CamelException *ex)
+{
+	/* Check for current folder */
+	if (folder && folder != store->current_folder) {
+		CamelImapResponse *response;
+		CamelException internal_ex;
+
+		response = camel_imap_command (store, folder, ex, NULL);
+		if (!response)
+			return NULL;
+		camel_exception_init (&internal_ex);
+		camel_imap_folder_selected (folder, response, &internal_ex);
+		camel_imap_response_free (store, response);
+		if (camel_exception_is_set (&internal_ex)) {
+			camel_exception_xfer (ex, &internal_ex);
+			return FALSE;
+		}
 	}
 
-	/* Read the response. */
-	return imap_read_response (store, ex);
+	/* Send the command */
+	return camel_remote_store_send_string (CAMEL_REMOTE_STORE (store), ex,
+					       "%c%.5d %s\r\n",
+					       store->tag_prefix,
+					       store->command++, cmd) != -1;
 }
 
 /**
- * camel_imap_command_continuation: Send more command data to the IMAP server
+ * camel_imap_command_continuation:
  * @store: the IMAP store
+ * @cmd: buffer containing the response/request data
  * @ex: a CamelException
- * @cmdbuf: buffer containing the response/request data
  *
  * This method is for sending continuing responses to the IMAP server
- * after camel_imap_command returns a CAMEL_IMAP_PLUS response.
+ * after camel_imap_command() or camel_imap_command_response() returns
+ * a continuation response.
  * 
  * This function assumes you have an exclusive lock on the remote stream.
  *
@@ -158,11 +206,11 @@ camel_imap_command (CamelImapStore *store, CamelFolder *folder,
  * command_lock will be released.
  **/
 CamelImapResponse *
-camel_imap_command_continuation (CamelImapStore *store, CamelException *ex,
-				 const char *cmdbuf)
+camel_imap_command_continuation (CamelImapStore *store, const char *cmd,
+				 CamelException *ex)
 {
 	if (camel_remote_store_send_string (CAMEL_REMOTE_STORE (store), ex,
-					    "%s\r\n", cmdbuf) < 0) {
+					    "%s\r\n", cmd) < 0) {
 		CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
 		return NULL;
 	}
@@ -170,54 +218,90 @@ camel_imap_command_continuation (CamelImapStore *store, CamelException *ex,
 	return imap_read_response (store, ex);
 }
 
-/* Read the response to an IMAP command. */
-static CamelImapResponse *
-imap_read_response (CamelImapStore *store, CamelException *ex)
+/**
+ * camel_imap_command_response:
+ * @store: the IMAP store
+ * @response: a pointer to pass back the response data in
+ * @ex: a CamelException
+ *
+ * This reads a single tagged, untagged, or continuation response from
+ * @store into *@response. The caller must free the string when it is
+ * done with it.
+ *
+ * Return value: One of %CAMEL_IMAP_RESPONSE_CONTINUATION,
+ * %CAMEL_IMAP_RESPONSE_UNTAGGED, %CAMEL_IMAP_RESPONSE_TAGGED, or
+ * %CAMEL_IMAP_RESPONSE_ERROR. If either of the last two, @store's
+ * command lock will be unlocked.
+ **/
+CamelImapResponseType
+camel_imap_command_response (CamelImapStore *store, char **response,
+			     CamelException *ex)
 {
-	CamelImapResponse *response;
-	CamelException internal_ex;
-	char *respbuf, *retcode;
+	CamelImapResponseType type;
+	char *respbuf;
 
-	/* Read first line */
 	if (camel_remote_store_recv_line (CAMEL_REMOTE_STORE (store),
 					  &respbuf, ex) < 0) {
 		CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
-		return NULL;
+		return CAMEL_IMAP_RESPONSE_ERROR;
 	}
 
-	response = g_new0 (CamelImapResponse, 1);
-	if (camel_disco_store_status (CAMEL_DISCO_STORE (store)) != CAMEL_DISCO_STORE_RESYNCING) {
-		response->folder = store->current_folder;
-		if (response->folder)
-			camel_object_ref (CAMEL_OBJECT (response->folder));
-	}
-	response->untagged = g_ptr_array_new ();
+	switch (*respbuf) {
+	case '*':
+		type = CAMEL_IMAP_RESPONSE_UNTAGGED;
 
-	camel_exception_init (&internal_ex);
-
-	/* Check for untagged data */
-	while (!strncmp (respbuf, "* ", 2)) {
 		/* Read the rest of the response if it is multi-line. */
-		respbuf = imap_read_untagged (store, respbuf, &internal_ex);
-		if (camel_exception_is_set (&internal_ex))
-			break;
-
-		if (!g_strncasecmp (respbuf, "* BYE", 5)) {
+		respbuf = imap_read_untagged (store, respbuf, ex);
+		if (!respbuf)
+			type = CAMEL_IMAP_RESPONSE_ERROR;
+		else if (!g_strncasecmp (respbuf, "* BYE", 5)) {
 			/* Connection was lost, no more data to fetch */
 			store->connected = FALSE;
 			g_free (respbuf);
-			respbuf = NULL;
-			break;
+			type = CAMEL_IMAP_RESPONSE_ERROR;
 		}
+		break;
+	case '+':
+		type = CAMEL_IMAP_RESPONSE_CONTINUATION;
+		break;
+	default:
+		type = CAMEL_IMAP_RESPONSE_TAGGED;
+		break;
+	}
+	*response = respbuf;
 
-		g_ptr_array_add (response->untagged, respbuf);
-		if (camel_remote_store_recv_line (
-			CAMEL_REMOTE_STORE (store), &respbuf, ex) < 0)
-			break;
+	if (type == CAMEL_IMAP_RESPONSE_ERROR ||
+	    type == CAMEL_IMAP_RESPONSE_TAGGED)
+		CAMEL_IMAP_STORE_UNLOCK (store, command_lock);
+	return type;
+}
+
+CamelImapResponse *
+imap_read_response (CamelImapStore *store, CamelException *ex)
+{
+	CamelImapResponse *response;
+	CamelImapResponseType type;
+	char *respbuf, *p;
+
+	/* Get another lock so that when we reach the tagged
+	 * response and camel_imap_command_response unlocks,
+	 * we're still locked. This lock is owned by response
+	 * and gets unlocked when response is freed.
+	 */
+	CAMEL_IMAP_STORE_LOCK (store, command_lock);
+
+	response = g_new0 (CamelImapResponse, 1);
+	if (store->current_folder && camel_disco_store_status (CAMEL_DISCO_STORE (store)) != CAMEL_DISCO_STORE_RESYNCING) {
+		response->folder = store->current_folder;
+		camel_object_ref (CAMEL_OBJECT (response->folder));
 	}
 
-	if (!respbuf || camel_exception_is_set (&internal_ex)) {
-		camel_exception_xfer (ex, &internal_ex);
+	response->untagged = g_ptr_array_new ();
+	while ((type = camel_imap_command_response (store, &respbuf, ex))
+	       == CAMEL_IMAP_RESPONSE_UNTAGGED)
+		g_ptr_array_add (response->untagged, respbuf);
+
+	if (type == CAMEL_IMAP_RESPONSE_ERROR) {
 		camel_imap_response_free_without_processing (store, response);
 		return NULL;
 	}
@@ -227,14 +311,14 @@ imap_read_response (CamelImapStore *store, CamelException *ex)
 	/* Check for OK or continuation response. */
 	if (*respbuf == '+')
 		return response;
-	retcode = imap_next_word (respbuf);
-	if (!strncmp (retcode, "OK", 2))
+	p = strchr (respbuf, ' ');
+	if (p && !g_strncasecmp (p, " OK", 3))
 		return response;
 
 	/* We should never get BAD, or anything else but +, OK, or NO
 	 * for that matter.
 	 */
-	if (strncmp (retcode, "NO", 2) != 0) {
+	if (!p || g_strncasecmp (p, " NO", 3) != 0) {
 		g_warning ("Unexpected response from IMAP server: %s",
 			   respbuf);
 		camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
@@ -244,10 +328,12 @@ imap_read_response (CamelImapStore *store, CamelException *ex)
 		return NULL;
 	}
 
-	retcode = imap_next_word (retcode);
+	p += 3;
+	if (!*p++)
+		p = NULL;
 	camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
 			      _("IMAP command failed: %s"),
-			      retcode ? retcode : _("Unknown error"));
+			      p ? p : _("Unknown error"));
 	camel_imap_response_free_without_processing (store, response);
 	return NULL;
 }
@@ -293,9 +379,9 @@ imap_read_untagged (CamelImapStore *store, char *line, CamelException *ex)
 					   str->str + 1, length);
 		if (nread == -1) {
 			if (errno == EINTR)
-				camel_exception_set (ex, CAMEL_EXCEPTION_USER_CANCEL, _("Operation cancelled"));
+				camel_exception_set(ex, CAMEL_EXCEPTION_USER_CANCEL, _("Operation cancelled"));
 			else
-				camel_exception_set (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE, strerror(errno));
+				camel_exception_set(ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE, strerror(errno));
 			camel_service_disconnect (CAMEL_SERVICE (store), FALSE, NULL);
 			goto lose;
 		}
@@ -409,13 +495,6 @@ camel_imap_response_free (CamelImapStore *store, CamelImapResponse *response)
 								sizeof (int));
 				}
 				g_array_append_val (expunged, number);
-
-				/* camel_imap_folder_changed expects
-				 * "exists" to be the value after all
-				 * expunges.
-				 */
-				if (exists)
-					exists--;
 			}
 		}
 		g_free (resp);
@@ -427,12 +506,8 @@ camel_imap_response_free (CamelImapStore *store, CamelImapResponse *response)
 	if (response->folder) {
 		if (exists > 0 || expunged) {
 			/* Update the summary */
-			CamelException ex;
-			
-			camel_exception_init (&ex);
-			camel_imap_folder_changed (response->folder, exists, expunged, &ex);
-			camel_exception_clear (&ex);
-			
+			camel_imap_folder_changed (response->folder,
+						   exists, expunged, NULL);
 			if (expunged)
 				g_array_free (expunged, TRUE);
 		}
@@ -535,7 +610,6 @@ camel_imap_response_extract_continuation (CamelImapStore *store,
 	if (response->status && *response->status == '+') {
 		status = response->status;
 		response->status = NULL;
-		CAMEL_IMAP_STORE_LOCK (store, command_lock);
 		camel_imap_response_free (store, response);
 		return status;
 	}
@@ -651,3 +725,16 @@ imap_command_strdup_vprintf (CamelImapStore *store, const char *fmt,
 
 	return out;
 }
+
+static char *
+imap_command_strdup_printf (CamelImapStore *store, const char *fmt, ...)
+{
+	va_list ap;
+	char *result;
+
+	va_start (ap, fmt);
+	result = imap_command_strdup_vprintf (store, fmt, ap);
+	va_end (ap);
+
+	return result;
+}
diff --git a/camel/providers/imap/camel-imap-command.h b/camel/providers/imap/camel-imap-command.h
index ba9ca7010a..fbcf82ee7d 100644
--- a/camel/providers/imap/camel-imap-command.h
+++ b/camel/providers/imap/camel-imap-command.h
@@ -36,6 +36,13 @@ extern "C" {
 #include <glib.h>
 #include "camel-imap-types.h"
 
+typedef enum {
+	CAMEL_IMAP_RESPONSE_ERROR,
+	CAMEL_IMAP_RESPONSE_CONTINUATION,
+	CAMEL_IMAP_RESPONSE_UNTAGGED,
+	CAMEL_IMAP_RESPONSE_TAGGED
+} CamelImapResponseType;
+
 struct _CamelImapResponse {
 	CamelFolder *folder;
 	GPtrArray *untagged;
@@ -47,19 +54,27 @@ CamelImapResponse *camel_imap_command              (CamelImapStore *store,
 						    CamelException *ex,
 						    const char *fmt, ...);
 CamelImapResponse *camel_imap_command_continuation (CamelImapStore *store,
-						    CamelException *ex,
-						    const char *cmdbuf);
+						    const char *cmd,
+						    CamelException *ex);
+
+void  camel_imap_response_free                     (CamelImapStore *store,
+						    CamelImapResponse *response);
+void  camel_imap_response_free_without_processing  (CamelImapStore *store,
+						    CamelImapResponse *response);
+char *camel_imap_response_extract                  (CamelImapStore *store,
+						    CamelImapResponse *response,
+						    const char *type,
+						    CamelException *ex);
+char *camel_imap_response_extract_continuation     (CamelImapStore *store,
+						    CamelImapResponse *response,
+						    CamelException *ex);
 
-void  camel_imap_response_free                 (CamelImapStore *store,
-						CamelImapResponse *response);
-void  camel_imap_response_free_without_processing(CamelImapStore *store,
-						  CamelImapResponse *response);
-char *camel_imap_response_extract              (CamelImapStore *store,
-						CamelImapResponse *response,
-						const char *type,
-						CamelException *ex);
-char *camel_imap_response_extract_continuation (CamelImapStore *store,
-						CamelImapResponse *response,
-						CamelException *ex);
+gboolean           camel_imap_command_start        (CamelImapStore *store,
+						    CamelFolder *folder,
+						    CamelException *ex,
+						    const char *fmt, ...);
+CamelImapResponseType camel_imap_command_response  (CamelImapStore *store,
+						    char **respbuf,
+						    CamelException *ex);
 
 #endif /* CAMEL_IMAP_COMMAND_H */
diff --git a/camel/providers/imap/camel-imap-folder.c b/camel/providers/imap/camel-imap-folder.c
index 88b8e5f1b5..3984db4bd9 100644
--- a/camel/providers/imap/camel-imap-folder.c
+++ b/camel/providers/imap/camel-imap-folder.c
@@ -411,59 +411,77 @@ imap_rescan (CamelFolder *folder, int exists, CamelException *ex)
 {
 	CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 	CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
-	CamelImapResponse *response;
 	struct {
 		char *uid;
 		guint32 flags;
-	} *new = NULL;
+	} *new;
 	char *resp;
-	int i, seq, summary_len;
+	CamelImapResponseType type;
+	int i, seq, summary_len, summary_got;
 	CamelMessageInfo *info;
 	CamelImapMessageInfo *iinfo;
 	GArray *removed;
-	GData *fetch_data;
-	gpointer data;
+	gboolean ok;
 
 	CAMEL_IMAP_STORE_ASSERT_LOCKED (store, command_lock);
 	imap_folder->need_rescan = FALSE;
 
-	camel_operation_start (NULL, _("Scanning IMAP folder"));
-
 	summary_len = camel_folder_summary_count (folder->summary);
-	if (summary_len) {
-		/* Check UIDs and flags of all messages we already know of. */
-		info = camel_folder_summary_index (folder->summary, summary_len - 1);
-		response = camel_imap_command (store, folder, ex,
-					       "UID FETCH 1:%s (FLAGS)",
-					       camel_message_info_uid (info));
-		camel_folder_summary_info_free (folder->summary, info);
-		if (!response) {
-			camel_operation_end (NULL);
-			return;
-		}
+	if (summary_len == 0) {
+		if (exists)
+			camel_imap_folder_changed (folder, exists, NULL, ex);
+		return;
+	}
 
-		new = g_malloc0 (summary_len * sizeof (*new));
-		for (i = 0; i < response->untagged->len; i++) {
-			resp = response->untagged->pdata[i];
+	/* Check UIDs and flags of all messages we already know of. */
+	camel_operation_start (NULL, _("Scanning for changed messages"));
+	info = camel_folder_summary_index (folder->summary, summary_len - 1);
+	ok = camel_imap_command_start (store, folder, ex,
+				       "UID FETCH 1:%s (FLAGS)",
+				       camel_message_info_uid (info));
+	camel_folder_summary_info_free (folder->summary, info);
+	if (!ok) {
+		camel_operation_end (NULL);
+		return;
+	}
 
-			seq = strtoul (resp + 2, &resp, 10);
-			if (g_strncasecmp (resp, " FETCH (", 8) != 0)
-				continue;
-			if (seq >= summary_len)
-				continue;
+	new = g_malloc0 (summary_len * sizeof (*new));
+	summary_got = 0;
+	while ((type = camel_imap_command_response (store, &resp, ex)) == CAMEL_IMAP_RESPONSE_UNTAGGED) {
+		GData *data;
+		char *uid;
+		guint32 flags;
 
-			fetch_data = parse_fetch_response (imap_folder, resp + 7);
-			data = g_datalist_get_data (&fetch_data, "UID");
-			if (data && !new[seq - 1].uid)
-				new[seq - 1].uid = g_strdup (data);
-			data = g_datalist_get_data (&fetch_data, "FLAGS");
-			if (data)
-				new[seq - 1].flags = GPOINTER_TO_UINT (data);
-			g_datalist_clear (&fetch_data);
+		data = parse_fetch_response (imap_folder, resp);
+		g_free (resp);
+		if (!data)
+			continue;
+
+		seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+		uid = g_datalist_get_data (&data, "UID");
+		flags = GPOINTER_TO_UINT (g_datalist_get_data (&data, "FLAGS"));
+
+		if (!uid || !seq || seq >= summary_len) {
+			g_datalist_clear (&data);
+			continue;
 		}
-		camel_imap_response_free_without_processing (store, response);
+
+		camel_operation_progress (NULL, ++summary_got * 100 / summary_len);
+		new[seq - 1].uid = g_strdup (uid);
+		new[seq - 1].flags = flags;
+		g_datalist_clear (&data);
 	}
 
+	camel_operation_end (NULL);
+	if (type == CAMEL_IMAP_RESPONSE_ERROR) {
+		for (i = 0; i < summary_len && new[i].uid; i++)
+			g_free (new[i].uid);
+		g_free (new);
+		return;
+	}
+	/* Free the final tagged response */
+	g_free (resp);
+
 	/* If we find a UID in the summary that doesn't correspond to
 	 * the UID in the folder, then either: (a) it's a real UID,
 	 * but the message was deleted on the server, or (b) it's a
@@ -521,8 +539,6 @@ imap_rescan (CamelFolder *folder, int exists, CamelException *ex)
 	/* And finally update the summary. */
 	camel_imap_folder_changed (folder, exists, removed, ex);
 	g_array_free (removed, TRUE);
-
-	camel_operation_end (NULL);
 }
 
 /* Find all messages in @folder with flags matching @flags and @mask.
@@ -946,7 +962,7 @@ do_append (CamelFolder *folder, CamelMimeMessage *message,
 
 	/* send the rest of our data - the mime message */
 	g_byte_array_append (ba, "\0", 3);
-	response = camel_imap_command_continuation (store, ex, ba->data);
+	response = camel_imap_command_continuation (store, ba->data, ex);
 	g_byte_array_free (ba, TRUE);
 	if (!response)
 		return response;
@@ -1536,110 +1552,226 @@ imap_cache_message (CamelDiscoFolder *disco_folder, const char *uid,
 		camel_object_unref (CAMEL_OBJECT (stream));
 }
 
+/* We pretend that a FLAGS or RFC822.SIZE response is always exactly
+ * 20 bytes long, and a BODY[HEADERS] response is always 2000 bytes
+ * long. Since we know how many of each kind of response we're
+ * expecting, we can find the total (pretend) amount of server traffic
+ * to expect and then count off the responses as we read them to update
+ * the progress bar.
+ */
+#define IMAP_PRETEND_SIZEOF_FLAGS	  20
+#define IMAP_PRETEND_SIZEOF_SIZE	  20
+#define IMAP_PRETEND_SIZEOF_HEADERS	2000
+
+static void
+add_message_from_data (CamelFolder *folder, GPtrArray *messages,
+		       int first, GData *data)
+{
+	int seq;
+	CamelMimeMessage *msg;
+	CamelStream *stream;
+	CamelMessageInfo *mi;
+
+	seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+	if (seq < first)
+		return;
+	stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
+	if (!stream)
+		return;
+
+	if (seq - first >= messages->len)
+		g_ptr_array_set_size (messages, seq - first + 1);
+
+	msg = camel_mime_message_new ();
+	camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream);
+	mi = camel_folder_summary_info_new_from_message (folder->summary, msg);
+	camel_object_unref (CAMEL_OBJECT (msg));
+
+	messages->pdata[seq - first] = mi;
+}
+
 static void
-imap_update_summary (CamelFolder *folder,
+imap_update_summary (CamelFolder *folder, int exists,
 		     CamelFolderChangeInfo *changes,
 		     GPtrArray *recents,
 		     CamelException *ex)
 {
 	CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 	CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
-	CamelImapResponse *response;
-	GPtrArray *lines, *messages;
-	char *p, *uid;
-	int i, seq, first, exists = 0;
-	CamelMimeMessage *msg;
+	CamelImapResponseType type;
+	GPtrArray *fetch_data = NULL, *messages = NULL, *needheaders;
+	char *uid, *resp;
+	const char *header_spec;
+	int i, seq, first, size, got, uidval;
 	CamelMessageInfo *mi;
-	GData *fetch_data;
 	CamelStream *stream;
+	guint32 flags;
+	GData *data;
 
 	CAMEL_IMAP_STORE_ASSERT_LOCKED (store, command_lock);
+	if (store->server_level >= IMAP_LEVEL_IMAP4REV1)
+		header_spec = "HEADER";
+	else
+		header_spec = "0";
 
-	first = camel_folder_summary_count (folder->summary) + 1;
+	/* Figure out if any of the new messages are already cached (which
+	 * may be the case if we're re-syncing after disconnected operation).
+	 * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
+	 * and ask for the headers too at the same time.
+	 */
+	seq = camel_folder_summary_count (folder->summary);
+	first = seq + 1;
+	if (seq > 0) {
+		mi = camel_folder_summary_index (folder->summary, seq - 1);
+		uidval = atoi (camel_message_info_uid (mi));
+		camel_folder_summary_info_free (folder->summary, mi);
+	} else
+		uidval = 0;
 
-	response = camel_imap_command (store, folder, ex, "FETCH %d:* (UID FLAGS RFC822.SIZE)", first);
-	if (!response)
-		return;
+	size = (exists - seq) * (IMAP_PRETEND_SIZEOF_FLAGS + IMAP_PRETEND_SIZEOF_SIZE);
+	got = 0;
+
+	if (uidval >= camel_imap_message_cache_max_uid (imap_folder->cache)) {
+		/* None of the new messages are cached */
+		size += (exists - seq) * IMAP_PRETEND_SIZEOF_HEADERS;
+		if (!camel_imap_command_start (store, folder, ex,
+					       "UID FETCH %d:* (FLAGS RFC822.SIZE BODY.PEEK[%s])",
+					       uidval + 1, header_spec))
+			return;
+		camel_operation_start (NULL, _("Fetching summary information for new messages"));
+	} else {
+		if (!camel_imap_command_start (store, folder, ex,
+					       "UID FETCH %d:* (FLAGS RFC822.SIZE)",
+					       uidval + 1))
+			return;
+		camel_operation_start (NULL, _("Scanning for new messages"));
+	}
 
-	/* Walk through the responses, looking for UIDs, and make sure
-	 * we have those headers cached.
+	/* Parse the responses. We can't add a message to the summary
+	 * until we've gotten its headers, and there's no guarantee
+	 * the server will send the responses in a useful order...
 	 */
+	fetch_data = g_ptr_array_new ();
 	messages = g_ptr_array_new ();
-	lines = response->untagged;
-	for (i = 0; i < lines->len; i++) {
-		p = lines->pdata[i];
-		if (*p++ != '*' || *p++ != ' ') {
-			g_ptr_array_remove_index_fast (lines, i--);
+	while ((type = camel_imap_command_response (store, &resp, ex)) ==
+	       CAMEL_IMAP_RESPONSE_UNTAGGED) {
+		data = parse_fetch_response (imap_folder, resp);
+		g_free (resp);
+		if (!data)
 			continue;
-		}
-		seq = strtoul (p, &p, 10);
-		if (!g_strcasecmp (p, " EXISTS")) {
-			exists = seq;
-			g_ptr_array_remove_index_fast (lines, i--);
+
+		seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+		if (seq < first) {
+			g_datalist_clear (&data);
 			continue;
 		}
-		if (!seq || seq < first || g_strncasecmp (p, " FETCH (", 8) != 0) {
-			g_ptr_array_remove_index_fast (lines, i--);
-			continue;
+
+		if (g_datalist_get_data (&data, "FLAGS"))
+			got += IMAP_PRETEND_SIZEOF_FLAGS;
+		if (g_datalist_get_data (&data, "RFC822.SIZE"))
+			got += IMAP_PRETEND_SIZEOF_SIZE;
+		stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
+		if (stream) {
+			got += IMAP_PRETEND_SIZEOF_HEADERS;
+
+			/* Use the stream now so we don't tie up many
+			 * many fds if we're fetching many many messages.
+			 */
+			add_message_from_data (folder, messages, first, data);
+			g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
 		}
 
-		if (seq - first >= messages->len)
-			g_ptr_array_set_size (messages, seq - first + 1);
+		camel_operation_progress (NULL, got * 100 / size);
+		g_ptr_array_add (fetch_data, data);
+	}
+	camel_operation_end (NULL);
+
+	if (type == CAMEL_IMAP_RESPONSE_ERROR)
+		goto lose;
 
-		fetch_data = parse_fetch_response (imap_folder, p + 7);
-		uid = g_datalist_get_data (&fetch_data, "UID");
+	/* Figure out which headers we still need to fetch. */
+	needheaders = g_ptr_array_new ();
+	size = got = 0;
+	for (i = 0; i < fetch_data->len; i++) {
+		data = fetch_data->pdata[i];
+		if (g_datalist_get_data (&data, "BODY_PART_LEN"))
+			continue;
+
+		uid = g_datalist_get_data (&data, "UID");
 		if (uid) {
-			stream = camel_imap_folder_fetch_data (
-				imap_folder, uid,
-				store->server_level >= IMAP_LEVEL_IMAP4REV1 ?
-				"HEADER" : "0", FALSE, ex);
-			if (!stream) {
-				camel_imap_response_free_without_processing (store, response);
-				/* XXX messages */
-				return;
-			}
+			g_ptr_array_add (needheaders, uid);
+			size += IMAP_PRETEND_SIZEOF_HEADERS;
+		}
+	}
 
-			msg = camel_mime_message_new ();
-			camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream);
-			camel_object_unref (CAMEL_OBJECT (stream));
-			mi = camel_folder_summary_info_new_from_message (folder->summary, msg);
-			camel_object_unref (CAMEL_OBJECT (msg));
+	/* And fetch them */
+	if (needheaders->len) {
+		char *set;
 
-			messages->pdata[seq - first] = mi;
+		/* FIXME: sort needheaders */
+		set = imap_uid_array_to_set (folder->summary, needheaders);
+		g_ptr_array_free (needheaders, TRUE);
+		if (!camel_imap_command_start (store, folder, ex,
+					       "UID FETCH %s BODY.PEEK[%s]",
+					       set, header_spec)) {
+			g_free (set);
+			goto lose;
 		}
-		g_datalist_clear (&fetch_data);
+		g_free (set);
+
+		camel_operation_start (NULL, _("Fetching summary information for new messages"));
+		while ((type = camel_imap_command_response (store, &resp, ex))
+		       == CAMEL_IMAP_RESPONSE_UNTAGGED) {
+			data = parse_fetch_response (imap_folder, resp);
+			g_free (resp);
+			if (!data)
+				continue;
+
+			stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
+			if (stream) {
+				add_message_from_data (folder, messages, first, data);
+				got += IMAP_PRETEND_SIZEOF_HEADERS;
+				camel_operation_progress (NULL, got * 100 / size);
+			}
+			g_datalist_clear (&data);
+		}
+		camel_operation_end (NULL);
+
+		if (type == CAMEL_IMAP_RESPONSE_ERROR)
+			goto lose;
 	}
 
-	/* Now go back through and create summary items */
-	lines = response->untagged;
-	for (i = 0; i < lines->len; i++) {
-		p = lines->pdata[i];
-		seq = strtoul (p + 2, &p, 10);
-		p = strchr (p, '(');
+	/* Now finish up summary entries (fix UIDs, set flags and size) */
+	for (i = 0; i < fetch_data->len; i++) {
+		data = fetch_data->pdata[i];
 
-		mi = messages->pdata[seq - first];
-		if (!mi) /* ? */
+		seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+		if (seq >= first + messages->len) {
+			g_datalist_clear (&data);
 			continue;
-		fetch_data = parse_fetch_response (imap_folder, p);
-
-		if (g_datalist_get_data (&fetch_data, "UID"))
-			camel_message_info_set_uid (mi, g_strdup (g_datalist_get_data (&fetch_data, "UID")));
-		if (g_datalist_get_data (&fetch_data, "FLAGS")) {
-			guint32 flags = GPOINTER_TO_INT (g_datalist_get_data (&fetch_data, "FLAGS"));
+		}
+		mi = messages->pdata[seq - first];
 
+		uid = g_datalist_get_data (&data, "UID");
+		if (uid)
+			camel_message_info_set_uid (mi, g_strdup (uid));
+		flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
+		if (flags) {
 			((CamelImapMessageInfo *)mi)->server_flags = flags;
 			/* "or" them in with the existing flags that may
 			 * have been set by summary_info_new_from_message.
 			 */
 			mi->flags |= flags;
 		}
-		if (g_datalist_get_data (&fetch_data, "RFC822.SIZE"))
-			mi->size = GPOINTER_TO_INT (g_datalist_get_data (&fetch_data, "RFC822.SIZE"));
+		size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
+		if (size)
+			mi->size = size;
 
-		g_datalist_clear (&fetch_data);
+		g_datalist_clear (&data);
 	}
-	camel_imap_response_free_without_processing (store, response);
+	g_ptr_array_free (fetch_data, TRUE);
 
+	/* And add the entries to the summary, etc. */
 	for (i = 0; i < messages->len; i++) {
 		mi = messages->pdata[i];
 		if (!mi) {
@@ -1648,14 +1780,28 @@ imap_update_summary (CamelFolder *folder,
 		}
 		camel_folder_summary_add (folder->summary, mi);
 		camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
+
 		if (recents && (mi->flags & CAMEL_IMAP_MESSAGE_RECENT))
 			g_ptr_array_add (recents, (char *)camel_message_info_uid (mi));
 	}
 	g_ptr_array_free (messages, TRUE);
+	return;
 
-	/* Did more mail arrive while we were doing this? */
-	if (exists && exists > camel_folder_summary_count (folder->summary))
-		imap_update_summary (folder, changes, recents, ex);
+ lose:
+	if (fetch_data) {
+		for (i = 0; i < fetch_data->len; i++) {
+			data = fetch_data->pdata[i];
+			g_datalist_clear (&data);
+		}
+		g_ptr_array_free (fetch_data, TRUE);
+	}
+	if (messages) {
+		for (i = 0; i < messages->len; i++) {
+			if (messages->pdata[i])
+				camel_folder_summary_info_free (folder->summary, messages->pdata[i]);
+		}
+		g_ptr_array_free (fetch_data, TRUE);
+	}
 }
 
 /* Called with the store's command_lock locked */
@@ -1697,7 +1843,7 @@ camel_imap_folder_changed (CamelFolder *folder, int exists,
 	if (exists > len) {
 		if (imap_folder->do_filtering)
 			recents = g_ptr_array_new ();
-		imap_update_summary (folder, changes, recents, ex);
+		imap_update_summary (folder, exists, changes, recents, ex);
 	}
 
 	if (camel_folder_change_info_changed (changes)) {
@@ -1827,6 +1973,8 @@ parse_fetch_response (CamelImapFolder *imap_folder, char *response)
 		if (g_strncasecmp (response, " FETCH (", 8) != 0)
 			return NULL;
 		response += 7;
+
+		g_datalist_set_data (&data, "SEQUENCE", GINT_TO_POINTER (seq));
 	}
 
 	do {
diff --git a/camel/providers/imap/camel-imap-store.c b/camel/providers/imap/camel-imap-store.c
index 7589aedcab..580a889078 100644
--- a/camel/providers/imap/camel-imap-store.c
+++ b/camel/providers/imap/camel-imap-store.c
@@ -409,7 +409,7 @@ try_auth (CamelImapStore *store, const char *mech, CamelException *ex)
 		if (camel_exception_is_set (ex))
 			goto break_and_lose;
 
-		response = camel_imap_command_continuation (store, ex, sasl_resp);
+		response = camel_imap_command_continuation (store, sasl_resp, ex);
 		g_free (sasl_resp);
 		if (!response)
 			goto lose;
@@ -430,7 +430,7 @@ try_auth (CamelImapStore *store, const char *mech, CamelException *ex)
 
  break_and_lose:
 	/* Get the server out of "waiting for continuation data" mode. */
-	response = camel_imap_command_continuation (store, NULL, "*");
+	response = camel_imap_command_continuation (store, "*", NULL);
 	if (response)
 		camel_imap_response_free (store, response);