Fixes for the summary messageid changes. Hash the messageid and store it.

2000-11-20  Not Zed  <NotZed@HelixCode.com>

	* providers/nntp/camel-nntp-utils.c (get_XOVER_headers): Fixes for
	the summary messageid changes.  Hash the messageid and store it.
	(get_XOVER_headers): Use camel_folder_summary_info_new() to create
	the summary item before adding it.

	* camel-folder-summary.h (CamelMessageInfo): Changed the
	messgae-id to be an 8 byte md5 hash, and the references list to be
	an array of these.

	* providers/local/camel-mh-summary.c (mh_summary_sync_message):
	New function, sync out the message info stuff.  Only updates the
	X-Ev header if it can get away with it, otherwise writes out a
	whole new message.
	(mh_summary_sync): Added more functionality.  All summary info is
	now written to the X-Ev header, etc, and new messages re-written
	if required during the sync process.

	* providers/local/camel-local-folder.c
	(local_set_message_user_flag): Set the XEVCHANGE flag.
	(local_set_message_user_tag): And here too.

	* providers/local/camel-local-summary.h: New flag
	CAMEL_MESSAGE_FOLDER_XEVCHANGE to indicate the XEV header has
	probably changed size and needs to be rewritten in whole.

	* camel-folder-summary.c (next_uid_string): Want this static, not
	const.
	(message_info_new): Store the references and message-id values as
	64 bit, binary hashes.
	(message_info_load): fix for message-id/references changes.
	(message_info_save): Likewise.
	(camel_message_info_dup_to): And here.
	(camel_message_info_free): And here too.  No longer free
	message_id, and simple free for references array.
	(CAMEL_FOLDER_SUMMARY_VERSION): Bumped file revision.
	(camel_folder_summary_init): Init memchunk allocators to empty.
	(camel_folder_summary_finalize): Free memchunk allocators if
	there.
	(message_info_new): Use the chunk allocator to allocate message
	info's.
	(camel_folder_summary_info_new): New helper to allocate the
	message info, and setup the memchunk if required.
	(content_info_alloc): Likewise for content info's.
	(message_info_load): Use summary_info_new_empty.
	(content_info_new): Use content_info_alloc.
	(content_info_load): "
	(content_info_free): Free the content info as a memchunk.
	(message_info_free): Free everything directly and the base as a
	memchunk, rather than calling camel_message_info_free(), which
	assumes a malloc'd array.

	* providers/local/camel-local-summary.c: Include ctype.h, kill a
	warning.
	(local_summary_decode_x_evolution): If we get a NULL message info,
	then dont try and set anything, just check for validity.
	(camel_local_summary_write_headers): New function to write a set
	of headers to an fd.
	(camel_local_summary_check): Added some statistic generation
	stuff for memory profiling.

	* providers/local/camel-mbox-summary.c (header_write): Changed to
	use stdoi functions to write out the header to a buffered stream,
	instead of using writev, which is apparently slow (and writing
	each line separately is slow anyway).
	(mbox_summary_sync_full): New implementation.  Does things
	differently, doesn't use or require the content info stuff.
	(summary_rebuild): Dont return an error if we start scanning at
	the end of file.
	(mbox_summary_sync_full): If we are not writing out new headers,
	make sure we copy the From line as we go, and update frompos
	appropriately.
	(mbox_summary_sync_full): Always copy the From line from the
	existing one, rather than trying to make one up ourselves.
	(mbox_summary_sync): If we can get by with a quick-sync, then try
	it, if that fails, then try a full sync anyway.
	(mbox_summary_sync_quick): Quick sync.  Only update system flags,
	etc.
	(mbox_summary_sync_full): Use the proper local summary encode_xev
	function.
	(header_evolution_decode): Removed, no longer needed.
	(header_evolution_encode): Same.
	(copy_block): No longer needed, removed.
	(header_write): Removed, replaced with
	camel_local_summary_write_headers.
	(mbox_summary_sync_full): Fixed for header_write change.

	* camel-mime-parser.c (folder_scan_step): Implement the new
	optional parser state HSCAN_PRE_FROM, that returns the (currently
	unfiltered) input data.
	(folder_scan_drop_step): Do the right thing for the PRE_FROM
	state.
	(camel_mime_parser_scan_from): Update the doco.
	(camel_mime_parser_scan_pre_from): Ok, make this behaviour
	optional, it simplifies a lot of loops that dont otherwise need to
	know about it.
	(folder_scan_step): Made the PRE_FROM state optional.
	(struct _header_scan_state): Made the bool vars 1 bit.
	(folder_pull_part): Free the from_line buffer if it is there.
	(folder_scan_skip_line): Added a new arg, can save the skpped data
	to a byte_array, as we go.
	(folder_scan_step): Fixed calls to skip_line approrpiately.  Now
	we save the from line as we parse it.
	(camel_mime_parser_read): New function to read from the mime
	parser buffer directly.  Useful if you use the parser to read the
	first/some headers, then need to scan the rest of the data,
	without needing to use a seek(), or allocate your own buffers.

	* camel-mime-parser.h (struct _header_state): Added a new parser state,
	pre-from which returns any data found before a from line during
	parsing (all other data can be retrieved by the caller except
	this).

svn path=/trunk/; revision=6618

1 files changed, 79 insertions, 11 deletions

diff --git a/camel/providers/local/camel-mh-summary.c b/camel/providers/local/camel-mh-summary.c
index b6b31664b4..2f3c829e8d 100644
--- a/camel/providers/local/camel-mh-summary.c
+++ b/camel/providers/local/camel-mh-summary.c
@@ -33,7 +33,7 @@
 
 #include <ctype.h>
 
-#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x))
+#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/
 
 #define CAMEL_MH_SUMMARY_VERSION (0x2000)
 
@@ -279,10 +279,71 @@ mh_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changeinfo, Came
 	return 0;
 }
 
+static int
+mh_summary_sync_message(CamelLocalSummary *cls, CamelMessageInfo *info, CamelException *ex)
+{
+	CamelMimeParser *mp;
+	const char *xev, *buffer;
+	int xevoffset;
+	int fd, outfd, len, outlen, ret=0;
+	char *name, *tmpname, *xevnew;
+
+	name = g_strdup_printf("%s/%s", cls->folder_path, info->uid);
+	fd = open(name, O_RDWR);
+	if (fd == -1)
+		return -1;
 
+	mp = camel_mime_parser_new();
+	camel_mime_parser_init_with_fd(mp, fd);
+	if (camel_mime_parser_step(mp, 0, 0) != HSCAN_EOF) {
+		xev = camel_mime_parser_header(mp, "X-Evolution", &xevoffset);
+		xevnew = camel_local_summary_encode_x_evolution(cls, info);
+		if (xev == NULL
+		    || camel_local_summary_decode_x_evolution(cls, xev, NULL) == -1
+		    || strlen(xev)+1 != strlen(xevnew)) {
+
+			/* need to write a new copy/unlink old */
+			tmpname = g_strdup_printf("%s/.tmp.%d.%s", cls->folder_path, getpid(), info->uid);
+			outfd = open(tmpname, O_CREAT|O_WRONLY|O_TRUNC, 0600);
+			if (outfd != -1) {
+				outlen = 0;
+				if ( (len = camel_local_summary_write_headers(outfd, camel_mime_parser_headers_raw(mp), xevnew)) == 0) {
+					while (outlen != -1 && (len = camel_mime_parser_read(mp, &buffer, 10240)) > 0) {
+						do {
+							outlen = write(fd, buffer, len);
+						} while (outlen == -1 && errno == EINTR);
+					}
+				}
+				if (close(outfd) == -1
+				    || len == -1
+				    || outlen == -1
+				    || rename(tmpname, name) == -1) {
+					unlink(tmpname);
+					ret = -1;
+				}
+			} else {
+				g_warning("sync can't create tmp file: %s", strerror(errno));
+			}
+			g_free(tmpname);
+		} else {
+			/* else, we can just update the flags field */
+			lseek(fd, xevoffset+strlen("X-Evolution: "), SEEK_SET);
+			do {
+				len = write(fd, xevnew, strlen(xevnew));
+			} while (len == -1 && errno == EINTR);
+			if (len == -1)
+				ret = -1;
+		}
+
+		g_free(xevnew);
+	}
 
-/* sync the summary with the ondisk files.
-   It doesnt store the state in the file, the summary only, == MUCH faster */
+	camel_object_unref((CamelObject *)mp);
+	g_free(name);
+	return ret;
+}
+
+/* sync the summary file with the ondisk files */
 static int
 mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo *changes, CamelException *ex)
 {
@@ -292,16 +353,16 @@ mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo
 
 	d(printf("summary_sync(expunge=%s)\n", expunge?"true":"false"));
 
-	if (cls->index) {
-		ibex_save(cls->index);
-	}
-	if (!expunge)
-		return 0;
+	/* we could probably get away without this ... but why not use it, esp if we're going to
+	   be doing any significant io already */
+	if (camel_local_summary_check(cls, changes, ex) == -1)
+		return -1;
 
 	count = camel_folder_summary_count((CamelFolderSummary *)cls);
 	for (i=count-1;i>=0;i--) {
 		info = camel_folder_summary_index((CamelFolderSummary *)cls, i);
-		if (info && info->flags & CAMEL_MESSAGE_DELETED) {
+		g_assert(info);
+		if (expunge && (info->flags & CAMEL_MESSAGE_DELETED)) {
 			name = g_strdup_printf("%s/%s", cls->folder_path, info->uid);
 			d(printf("deleting %s\n", name));
 			if (unlink(name) == 0 || errno==ENOENT) {
@@ -309,12 +370,19 @@ mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo
 				/* FIXME: put this in folder_summary::remove()? */
 				if (cls->index)
 					ibex_unindex(cls->index, info->uid);
-
+				
 				camel_folder_change_info_remove_uid(changes, info->uid);
 				camel_folder_summary_remove((CamelFolderSummary *)cls, info);
 			}
+			g_free(name);
+		} else if (info->flags & (CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_FLAGGED)) {
+			if (mh_summary_sync_message(cls, info, ex) != -1) {
+				info->flags &= 0xffff;
+			} else {
+				g_warning("Problem occured when trying to expunge, ignored");
+			}
 		}
 	}
+
 	return 0;
 }
-