Send an IMAP command, but don't wait for responses.

	* providers/imap/camel-imap-command.c (camel_imap_command_start):
	Send an IMAP command, but don't wait for responses.
	(camel_imap_command_response): Read a single line of response from
	the server.
	(camel_imap_command, etc): Reimplement in terms of the new code.

	* providers/imap/camel-imap-folder.c (imap_rescan): Use
	camel_imap_command_start and camel_imap_command_response, and
	call camel_operation_progress after each line read from the
	server.
	(imap_update_summary): Likewise, although with more fudging on the
	precentages... Also, fix this so that if none of the new messages
	are cached, it only does a single FETCH, and if some of them are
	cached, it does two FETCHes (one to get the UIDs, FLAGS, and
	SIZEs, and another to get the bodies of all of the messages that
	aren't cached now that it knows the relevant UIDs). This should
	speed up startup a bunch (especially if you have high bandwidth
	but also high latency to the IMAP server).

svn path=/trunk/; revision=11430

1 files changed, 254 insertions, 106 deletions

diff --git a/camel/providers/imap/camel-imap-folder.c b/camel/providers/imap/camel-imap-folder.c
index 88b8e5f1b5..3984db4bd9 100644
--- a/camel/providers/imap/camel-imap-folder.c
+++ b/camel/providers/imap/camel-imap-folder.c
@@ -411,59 +411,77 @@ imap_rescan (CamelFolder *folder, int exists, CamelException *ex)
 {
 	CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 	CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
-	CamelImapResponse *response;
 	struct {
 		char *uid;
 		guint32 flags;
-	} *new = NULL;
+	} *new;
 	char *resp;
-	int i, seq, summary_len;
+	CamelImapResponseType type;
+	int i, seq, summary_len, summary_got;
 	CamelMessageInfo *info;
 	CamelImapMessageInfo *iinfo;
 	GArray *removed;
-	GData *fetch_data;
-	gpointer data;
+	gboolean ok;
 
 	CAMEL_IMAP_STORE_ASSERT_LOCKED (store, command_lock);
 	imap_folder->need_rescan = FALSE;
 
-	camel_operation_start (NULL, _("Scanning IMAP folder"));
-
 	summary_len = camel_folder_summary_count (folder->summary);
-	if (summary_len) {
-		/* Check UIDs and flags of all messages we already know of. */
-		info = camel_folder_summary_index (folder->summary, summary_len - 1);
-		response = camel_imap_command (store, folder, ex,
-					       "UID FETCH 1:%s (FLAGS)",
-					       camel_message_info_uid (info));
-		camel_folder_summary_info_free (folder->summary, info);
-		if (!response) {
-			camel_operation_end (NULL);
-			return;
-		}
+	if (summary_len == 0) {
+		if (exists)
+			camel_imap_folder_changed (folder, exists, NULL, ex);
+		return;
+	}
 
-		new = g_malloc0 (summary_len * sizeof (*new));
-		for (i = 0; i < response->untagged->len; i++) {
-			resp = response->untagged->pdata[i];
+	/* Check UIDs and flags of all messages we already know of. */
+	camel_operation_start (NULL, _("Scanning for changed messages"));
+	info = camel_folder_summary_index (folder->summary, summary_len - 1);
+	ok = camel_imap_command_start (store, folder, ex,
+				       "UID FETCH 1:%s (FLAGS)",
+				       camel_message_info_uid (info));
+	camel_folder_summary_info_free (folder->summary, info);
+	if (!ok) {
+		camel_operation_end (NULL);
+		return;
+	}
 
-			seq = strtoul (resp + 2, &resp, 10);
-			if (g_strncasecmp (resp, " FETCH (", 8) != 0)
-				continue;
-			if (seq >= summary_len)
-				continue;
+	new = g_malloc0 (summary_len * sizeof (*new));
+	summary_got = 0;
+	while ((type = camel_imap_command_response (store, &resp, ex)) == CAMEL_IMAP_RESPONSE_UNTAGGED) {
+		GData *data;
+		char *uid;
+		guint32 flags;
 
-			fetch_data = parse_fetch_response (imap_folder, resp + 7);
-			data = g_datalist_get_data (&fetch_data, "UID");
-			if (data && !new[seq - 1].uid)
-				new[seq - 1].uid = g_strdup (data);
-			data = g_datalist_get_data (&fetch_data, "FLAGS");
-			if (data)
-				new[seq - 1].flags = GPOINTER_TO_UINT (data);
-			g_datalist_clear (&fetch_data);
+		data = parse_fetch_response (imap_folder, resp);
+		g_free (resp);
+		if (!data)
+			continue;
+
+		seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+		uid = g_datalist_get_data (&data, "UID");
+		flags = GPOINTER_TO_UINT (g_datalist_get_data (&data, "FLAGS"));
+
+		if (!uid || !seq || seq >= summary_len) {
+			g_datalist_clear (&data);
+			continue;
 		}
-		camel_imap_response_free_without_processing (store, response);
+
+		camel_operation_progress (NULL, ++summary_got * 100 / summary_len);
+		new[seq - 1].uid = g_strdup (uid);
+		new[seq - 1].flags = flags;
+		g_datalist_clear (&data);
 	}
 
+	camel_operation_end (NULL);
+	if (type == CAMEL_IMAP_RESPONSE_ERROR) {
+		for (i = 0; i < summary_len && new[i].uid; i++)
+			g_free (new[i].uid);
+		g_free (new);
+		return;
+	}
+	/* Free the final tagged response */
+	g_free (resp);
+
 	/* If we find a UID in the summary that doesn't correspond to
 	 * the UID in the folder, then either: (a) it's a real UID,
 	 * but the message was deleted on the server, or (b) it's a
@@ -521,8 +539,6 @@ imap_rescan (CamelFolder *folder, int exists, CamelException *ex)
 	/* And finally update the summary. */
 	camel_imap_folder_changed (folder, exists, removed, ex);
 	g_array_free (removed, TRUE);
-
-	camel_operation_end (NULL);
 }
 
 /* Find all messages in @folder with flags matching @flags and @mask.
@@ -946,7 +962,7 @@ do_append (CamelFolder *folder, CamelMimeMessage *message,
 
 	/* send the rest of our data - the mime message */
 	g_byte_array_append (ba, "\0", 3);
-	response = camel_imap_command_continuation (store, ex, ba->data);
+	response = camel_imap_command_continuation (store, ba->data, ex);
 	g_byte_array_free (ba, TRUE);
 	if (!response)
 		return response;
@@ -1536,110 +1552,226 @@ imap_cache_message (CamelDiscoFolder *disco_folder, const char *uid,
 		camel_object_unref (CAMEL_OBJECT (stream));
 }
 
+/* We pretend that a FLAGS or RFC822.SIZE response is always exactly
+ * 20 bytes long, and a BODY[HEADERS] response is always 2000 bytes
+ * long. Since we know how many of each kind of response we're
+ * expecting, we can find the total (pretend) amount of server traffic
+ * to expect and then count off the responses as we read them to update
+ * the progress bar.
+ */
+#define IMAP_PRETEND_SIZEOF_FLAGS	  20
+#define IMAP_PRETEND_SIZEOF_SIZE	  20
+#define IMAP_PRETEND_SIZEOF_HEADERS	2000
+
+static void
+add_message_from_data (CamelFolder *folder, GPtrArray *messages,
+		       int first, GData *data)
+{
+	int seq;
+	CamelMimeMessage *msg;
+	CamelStream *stream;
+	CamelMessageInfo *mi;
+
+	seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+	if (seq < first)
+		return;
+	stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
+	if (!stream)
+		return;
+
+	if (seq - first >= messages->len)
+		g_ptr_array_set_size (messages, seq - first + 1);
+
+	msg = camel_mime_message_new ();
+	camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream);
+	mi = camel_folder_summary_info_new_from_message (folder->summary, msg);
+	camel_object_unref (CAMEL_OBJECT (msg));
+
+	messages->pdata[seq - first] = mi;
+}
+
 static void
-imap_update_summary (CamelFolder *folder,
+imap_update_summary (CamelFolder *folder, int exists,
 		     CamelFolderChangeInfo *changes,
 		     GPtrArray *recents,
 		     CamelException *ex)
 {
 	CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 	CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
-	CamelImapResponse *response;
-	GPtrArray *lines, *messages;
-	char *p, *uid;
-	int i, seq, first, exists = 0;
-	CamelMimeMessage *msg;
+	CamelImapResponseType type;
+	GPtrArray *fetch_data = NULL, *messages = NULL, *needheaders;
+	char *uid, *resp;
+	const char *header_spec;
+	int i, seq, first, size, got, uidval;
 	CamelMessageInfo *mi;
-	GData *fetch_data;
 	CamelStream *stream;
+	guint32 flags;
+	GData *data;
 
 	CAMEL_IMAP_STORE_ASSERT_LOCKED (store, command_lock);
+	if (store->server_level >= IMAP_LEVEL_IMAP4REV1)
+		header_spec = "HEADER";
+	else
+		header_spec = "0";
 
-	first = camel_folder_summary_count (folder->summary) + 1;
+	/* Figure out if any of the new messages are already cached (which
+	 * may be the case if we're re-syncing after disconnected operation).
+	 * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
+	 * and ask for the headers too at the same time.
+	 */
+	seq = camel_folder_summary_count (folder->summary);
+	first = seq + 1;
+	if (seq > 0) {
+		mi = camel_folder_summary_index (folder->summary, seq - 1);
+		uidval = atoi (camel_message_info_uid (mi));
+		camel_folder_summary_info_free (folder->summary, mi);
+	} else
+		uidval = 0;
 
-	response = camel_imap_command (store, folder, ex, "FETCH %d:* (UID FLAGS RFC822.SIZE)", first);
-	if (!response)
-		return;
+	size = (exists - seq) * (IMAP_PRETEND_SIZEOF_FLAGS + IMAP_PRETEND_SIZEOF_SIZE);
+	got = 0;
+
+	if (uidval >= camel_imap_message_cache_max_uid (imap_folder->cache)) {
+		/* None of the new messages are cached */
+		size += (exists - seq) * IMAP_PRETEND_SIZEOF_HEADERS;
+		if (!camel_imap_command_start (store, folder, ex,
+					       "UID FETCH %d:* (FLAGS RFC822.SIZE BODY.PEEK[%s])",
+					       uidval + 1, header_spec))
+			return;
+		camel_operation_start (NULL, _("Fetching summary information for new messages"));
+	} else {
+		if (!camel_imap_command_start (store, folder, ex,
+					       "UID FETCH %d:* (FLAGS RFC822.SIZE)",
+					       uidval + 1))
+			return;
+		camel_operation_start (NULL, _("Scanning for new messages"));
+	}
 
-	/* Walk through the responses, looking for UIDs, and make sure
-	 * we have those headers cached.
+	/* Parse the responses. We can't add a message to the summary
+	 * until we've gotten its headers, and there's no guarantee
+	 * the server will send the responses in a useful order...
 	 */
+	fetch_data = g_ptr_array_new ();
 	messages = g_ptr_array_new ();
-	lines = response->untagged;
-	for (i = 0; i < lines->len; i++) {
-		p = lines->pdata[i];
-		if (*p++ != '*' || *p++ != ' ') {
-			g_ptr_array_remove_index_fast (lines, i--);
+	while ((type = camel_imap_command_response (store, &resp, ex)) ==
+	       CAMEL_IMAP_RESPONSE_UNTAGGED) {
+		data = parse_fetch_response (imap_folder, resp);
+		g_free (resp);
+		if (!data)
 			continue;
-		}
-		seq = strtoul (p, &p, 10);
-		if (!g_strcasecmp (p, " EXISTS")) {
-			exists = seq;
-			g_ptr_array_remove_index_fast (lines, i--);
+
+		seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+		if (seq < first) {
+			g_datalist_clear (&data);
 			continue;
 		}
-		if (!seq || seq < first || g_strncasecmp (p, " FETCH (", 8) != 0) {
-			g_ptr_array_remove_index_fast (lines, i--);
-			continue;
+
+		if (g_datalist_get_data (&data, "FLAGS"))
+			got += IMAP_PRETEND_SIZEOF_FLAGS;
+		if (g_datalist_get_data (&data, "RFC822.SIZE"))
+			got += IMAP_PRETEND_SIZEOF_SIZE;
+		stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
+		if (stream) {
+			got += IMAP_PRETEND_SIZEOF_HEADERS;
+
+			/* Use the stream now so we don't tie up many
+			 * many fds if we're fetching many many messages.
+			 */
+			add_message_from_data (folder, messages, first, data);
+			g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
 		}
 
-		if (seq - first >= messages->len)
-			g_ptr_array_set_size (messages, seq - first + 1);
+		camel_operation_progress (NULL, got * 100 / size);
+		g_ptr_array_add (fetch_data, data);
+	}
+	camel_operation_end (NULL);
+
+	if (type == CAMEL_IMAP_RESPONSE_ERROR)
+		goto lose;
 
-		fetch_data = parse_fetch_response (imap_folder, p + 7);
-		uid = g_datalist_get_data (&fetch_data, "UID");
+	/* Figure out which headers we still need to fetch. */
+	needheaders = g_ptr_array_new ();
+	size = got = 0;
+	for (i = 0; i < fetch_data->len; i++) {
+		data = fetch_data->pdata[i];
+		if (g_datalist_get_data (&data, "BODY_PART_LEN"))
+			continue;
+
+		uid = g_datalist_get_data (&data, "UID");
 		if (uid) {
-			stream = camel_imap_folder_fetch_data (
-				imap_folder, uid,
-				store->server_level >= IMAP_LEVEL_IMAP4REV1 ?
-				"HEADER" : "0", FALSE, ex);
-			if (!stream) {
-				camel_imap_response_free_without_processing (store, response);
-				/* XXX messages */
-				return;
-			}
+			g_ptr_array_add (needheaders, uid);
+			size += IMAP_PRETEND_SIZEOF_HEADERS;
+		}
+	}
 
-			msg = camel_mime_message_new ();
-			camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream);
-			camel_object_unref (CAMEL_OBJECT (stream));
-			mi = camel_folder_summary_info_new_from_message (folder->summary, msg);
-			camel_object_unref (CAMEL_OBJECT (msg));
+	/* And fetch them */
+	if (needheaders->len) {
+		char *set;
 
-			messages->pdata[seq - first] = mi;
+		/* FIXME: sort needheaders */
+		set = imap_uid_array_to_set (folder->summary, needheaders);
+		g_ptr_array_free (needheaders, TRUE);
+		if (!camel_imap_command_start (store, folder, ex,
+					       "UID FETCH %s BODY.PEEK[%s]",
+					       set, header_spec)) {
+			g_free (set);
+			goto lose;
 		}
-		g_datalist_clear (&fetch_data);
+		g_free (set);
+
+		camel_operation_start (NULL, _("Fetching summary information for new messages"));
+		while ((type = camel_imap_command_response (store, &resp, ex))
+		       == CAMEL_IMAP_RESPONSE_UNTAGGED) {
+			data = parse_fetch_response (imap_folder, resp);
+			g_free (resp);
+			if (!data)
+				continue;
+
+			stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
+			if (stream) {
+				add_message_from_data (folder, messages, first, data);
+				got += IMAP_PRETEND_SIZEOF_HEADERS;
+				camel_operation_progress (NULL, got * 100 / size);
+			}
+			g_datalist_clear (&data);
+		}
+		camel_operation_end (NULL);
+
+		if (type == CAMEL_IMAP_RESPONSE_ERROR)
+			goto lose;
 	}
 
-	/* Now go back through and create summary items */
-	lines = response->untagged;
-	for (i = 0; i < lines->len; i++) {
-		p = lines->pdata[i];
-		seq = strtoul (p + 2, &p, 10);
-		p = strchr (p, '(');
+	/* Now finish up summary entries (fix UIDs, set flags and size) */
+	for (i = 0; i < fetch_data->len; i++) {
+		data = fetch_data->pdata[i];
 
-		mi = messages->pdata[seq - first];
-		if (!mi) /* ? */
+		seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
+		if (seq >= first + messages->len) {
+			g_datalist_clear (&data);
 			continue;
-		fetch_data = parse_fetch_response (imap_folder, p);
-
-		if (g_datalist_get_data (&fetch_data, "UID"))
-			camel_message_info_set_uid (mi, g_strdup (g_datalist_get_data (&fetch_data, "UID")));
-		if (g_datalist_get_data (&fetch_data, "FLAGS")) {
-			guint32 flags = GPOINTER_TO_INT (g_datalist_get_data (&fetch_data, "FLAGS"));
+		}
+		mi = messages->pdata[seq - first];
 
+		uid = g_datalist_get_data (&data, "UID");
+		if (uid)
+			camel_message_info_set_uid (mi, g_strdup (uid));
+		flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
+		if (flags) {
 			((CamelImapMessageInfo *)mi)->server_flags = flags;
 			/* "or" them in with the existing flags that may
 			 * have been set by summary_info_new_from_message.
 			 */
 			mi->flags |= flags;
 		}
-		if (g_datalist_get_data (&fetch_data, "RFC822.SIZE"))
-			mi->size = GPOINTER_TO_INT (g_datalist_get_data (&fetch_data, "RFC822.SIZE"));
+		size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
+		if (size)
+			mi->size = size;
 
-		g_datalist_clear (&fetch_data);
+		g_datalist_clear (&data);
 	}
-	camel_imap_response_free_without_processing (store, response);
+	g_ptr_array_free (fetch_data, TRUE);
 
+	/* And add the entries to the summary, etc. */
 	for (i = 0; i < messages->len; i++) {
 		mi = messages->pdata[i];
 		if (!mi) {
@@ -1648,14 +1780,28 @@ imap_update_summary (CamelFolder *folder,
 		}
 		camel_folder_summary_add (folder->summary, mi);
 		camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
+
 		if (recents && (mi->flags & CAMEL_IMAP_MESSAGE_RECENT))
 			g_ptr_array_add (recents, (char *)camel_message_info_uid (mi));
 	}
 	g_ptr_array_free (messages, TRUE);
+	return;
 
-	/* Did more mail arrive while we were doing this? */
-	if (exists && exists > camel_folder_summary_count (folder->summary))
-		imap_update_summary (folder, changes, recents, ex);
+ lose:
+	if (fetch_data) {
+		for (i = 0; i < fetch_data->len; i++) {
+			data = fetch_data->pdata[i];
+			g_datalist_clear (&data);
+		}
+		g_ptr_array_free (fetch_data, TRUE);
+	}
+	if (messages) {
+		for (i = 0; i < messages->len; i++) {
+			if (messages->pdata[i])
+				camel_folder_summary_info_free (folder->summary, messages->pdata[i]);
+		}
+		g_ptr_array_free (fetch_data, TRUE);
+	}
 }
 
 /* Called with the store's command_lock locked */
@@ -1697,7 +1843,7 @@ camel_imap_folder_changed (CamelFolder *folder, int exists,
 	if (exists > len) {
 		if (imap_folder->do_filtering)
 			recents = g_ptr_array_new ();
-		imap_update_summary (folder, changes, recents, ex);
+		imap_update_summary (folder, exists, changes, recents, ex);
 	}
 
 	if (camel_folder_change_info_changed (changes)) {
@@ -1827,6 +1973,8 @@ parse_fetch_response (CamelImapFolder *imap_folder, char *response)
 		if (g_strncasecmp (response, " FETCH (", 8) != 0)
 			return NULL;
 		response += 7;
+
+		g_datalist_set_data (&data, "SEQUENCE", GINT_TO_POINTER (seq));
 	}
 
 	do {