Fixes for the summary messageid changes. Hash the messageid and store it.

2000-11-20  Not Zed  <NotZed@HelixCode.com>

	* providers/nntp/camel-nntp-utils.c (get_XOVER_headers): Fixes for
	the summary messageid changes.  Hash the messageid and store it.
	(get_XOVER_headers): Use camel_folder_summary_info_new() to create
	the summary item before adding it.

	* camel-folder-summary.h (CamelMessageInfo): Changed the
	messgae-id to be an 8 byte md5 hash, and the references list to be
	an array of these.

	* providers/local/camel-mh-summary.c (mh_summary_sync_message):
	New function, sync out the message info stuff.  Only updates the
	X-Ev header if it can get away with it, otherwise writes out a
	whole new message.
	(mh_summary_sync): Added more functionality.  All summary info is
	now written to the X-Ev header, etc, and new messages re-written
	if required during the sync process.

	* providers/local/camel-local-folder.c
	(local_set_message_user_flag): Set the XEVCHANGE flag.
	(local_set_message_user_tag): And here too.

	* providers/local/camel-local-summary.h: New flag
	CAMEL_MESSAGE_FOLDER_XEVCHANGE to indicate the XEV header has
	probably changed size and needs to be rewritten in whole.

	* camel-folder-summary.c (next_uid_string): Want this static, not
	const.
	(message_info_new): Store the references and message-id values as
	64 bit, binary hashes.
	(message_info_load): fix for message-id/references changes.
	(message_info_save): Likewise.
	(camel_message_info_dup_to): And here.
	(camel_message_info_free): And here too.  No longer free
	message_id, and simple free for references array.
	(CAMEL_FOLDER_SUMMARY_VERSION): Bumped file revision.
	(camel_folder_summary_init): Init memchunk allocators to empty.
	(camel_folder_summary_finalize): Free memchunk allocators if
	there.
	(message_info_new): Use the chunk allocator to allocate message
	info's.
	(camel_folder_summary_info_new): New helper to allocate the
	message info, and setup the memchunk if required.
	(content_info_alloc): Likewise for content info's.
	(message_info_load): Use summary_info_new_empty.
	(content_info_new): Use content_info_alloc.
	(content_info_load): "
	(content_info_free): Free the content info as a memchunk.
	(message_info_free): Free everything directly and the base as a
	memchunk, rather than calling camel_message_info_free(), which
	assumes a malloc'd array.

	* providers/local/camel-local-summary.c: Include ctype.h, kill a
	warning.
	(local_summary_decode_x_evolution): If we get a NULL message info,
	then dont try and set anything, just check for validity.
	(camel_local_summary_write_headers): New function to write a set
	of headers to an fd.
	(camel_local_summary_check): Added some statistic generation
	stuff for memory profiling.

	* providers/local/camel-mbox-summary.c (header_write): Changed to
	use stdoi functions to write out the header to a buffered stream,
	instead of using writev, which is apparently slow (and writing
	each line separately is slow anyway).
	(mbox_summary_sync_full): New implementation.  Does things
	differently, doesn't use or require the content info stuff.
	(summary_rebuild): Dont return an error if we start scanning at
	the end of file.
	(mbox_summary_sync_full): If we are not writing out new headers,
	make sure we copy the From line as we go, and update frompos
	appropriately.
	(mbox_summary_sync_full): Always copy the From line from the
	existing one, rather than trying to make one up ourselves.
	(mbox_summary_sync): If we can get by with a quick-sync, then try
	it, if that fails, then try a full sync anyway.
	(mbox_summary_sync_quick): Quick sync.  Only update system flags,
	etc.
	(mbox_summary_sync_full): Use the proper local summary encode_xev
	function.
	(header_evolution_decode): Removed, no longer needed.
	(header_evolution_encode): Same.
	(copy_block): No longer needed, removed.
	(header_write): Removed, replaced with
	camel_local_summary_write_headers.
	(mbox_summary_sync_full): Fixed for header_write change.

	* camel-mime-parser.c (folder_scan_step): Implement the new
	optional parser state HSCAN_PRE_FROM, that returns the (currently
	unfiltered) input data.
	(folder_scan_drop_step): Do the right thing for the PRE_FROM
	state.
	(camel_mime_parser_scan_from): Update the doco.
	(camel_mime_parser_scan_pre_from): Ok, make this behaviour
	optional, it simplifies a lot of loops that dont otherwise need to
	know about it.
	(folder_scan_step): Made the PRE_FROM state optional.
	(struct _header_scan_state): Made the bool vars 1 bit.
	(folder_pull_part): Free the from_line buffer if it is there.
	(folder_scan_skip_line): Added a new arg, can save the skpped data
	to a byte_array, as we go.
	(folder_scan_step): Fixed calls to skip_line approrpiately.  Now
	we save the from line as we parse it.
	(camel_mime_parser_read): New function to read from the mime
	parser buffer directly.  Useful if you use the parser to read the
	first/some headers, then need to scan the rest of the data,
	without needing to use a seek(), or allocate your own buffers.

	* camel-mime-parser.h (struct _header_state): Added a new parser state,
	pre-from which returns any data found before a from line during
	parsing (all other data can be retrieved by the caller except
	this).

svn path=/trunk/; revision=6618

1 files changed, 176 insertions, 6 deletions

diff --git a/camel/providers/local/camel-local-summary.c b/camel/providers/local/camel-local-summary.c
index 7455e96cf8..5ba470990b 100644
--- a/camel/providers/local/camel-local-summary.c
+++ b/camel/providers/local/camel-local-summary.c
@@ -23,6 +23,8 @@
 #include "camel-local-summary.h"
 #include <camel/camel-mime-message.h>
 
+#include <ctype.h>
+
 #include <sys/stat.h>
 #include <sys/uio.h>
 #include <unistd.h>
@@ -31,7 +33,7 @@
 #include <stdlib.h>
 
 #define io(x)
-#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x))
+#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/
 
 #define CAMEL_LOCAL_SUMMARY_VERSION (0x100)
 
@@ -166,10 +168,114 @@ camel_local_summary_decode_x_evolution(CamelLocalSummary *cls, const char *xev,
 	return ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->decode_x_evolution(cls, xev, info);
 }
 
+#define DOSTATS
+#ifdef DOSTATS
+struct _stat_info {
+	int mitotal;
+	int micount;
+	int citotal;
+	int cicount;
+	int msgid;
+	int msgcount;
+};
+
+static void
+do_stat_ci(CamelLocalSummary *cls, struct _stat_info *info, CamelMessageContentInfo *ci)
+{
+	info->cicount++;
+	info->citotal += ((CamelFolderSummary *)cls)->content_info_size /*+ 4 memchunks are 1/4 byte overhead per mi */;
+	if (ci->id)
+		info->citotal += strlen(ci->id) + 4;
+	if (ci->description)
+		info->citotal += strlen(ci->description) + 4;
+	if (ci->encoding)
+		info->citotal += strlen(ci->encoding) + 4;
+	if (ci->type) {
+		struct _header_content_type *ct = ci->type;
+		struct _header_param *param;
+
+		info->citotal += sizeof(*ct) + 4;
+		if (ct->type)
+			info->citotal += strlen(ct->type) + 4;
+		if (ct->subtype)
+			info->citotal += strlen(ct->subtype) + 4;
+		param = ct->params;
+		while (param) {
+			info->citotal += sizeof(*param) + 4;
+			if (param->name)
+				info->citotal += strlen(param->name)+4;
+			if (param->value)
+				info->citotal += strlen(param->value)+4;
+			param = param->next;
+		}
+	}
+	ci = ci->childs;
+	while (ci) {
+		do_stat_ci(cls, info, ci);
+		ci = ci->next;
+	}
+}
+
+static void
+do_stat_mi(CamelLocalSummary *cls, struct _stat_info *info, CamelMessageInfo *mi)
+{
+	info->micount++;
+	info->mitotal += ((CamelFolderSummary *)cls)->content_info_size /*+ 4*/;
+
+	if (mi->subject)
+		info->mitotal += strlen(mi->subject) + 4;
+	if (mi->to)
+		info->mitotal += strlen(mi->to) + 4;
+	if (mi->from)
+		info->mitotal += strlen(mi->from) + 4;
+	if (mi->cc)
+		info->mitotal += strlen(mi->cc) + 4;
+	if (mi->uid)
+		info->mitotal += strlen(mi->uid) + 4;
+
+	if (mi->references) {
+		info->mitotal += (mi->references->size-1) * sizeof(CamelSummaryMessageID) + sizeof(CamelSummaryReferences) + 4;
+		info->msgid += (mi->references->size) * sizeof(CamelSummaryMessageID);
+		info->msgcount += mi->references->size;
+	}
+
+	/* dont have any user flags yet */
+
+	if (mi->content) {
+		do_stat_ci(cls, info, mi->content);
+	}
+}
+
+#endif
+
 int
 camel_local_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changeinfo, CamelException *ex)
 {
-	return ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->check(cls, changeinfo, ex);
+	int ret;
+
+	ret = ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->check(cls, changeinfo, ex);
+
+#ifdef DOSTATS
+	if (ret != -1) {
+		int i;
+		CamelFolderSummary *s = (CamelFolderSummary *)cls;
+		struct _stat_info stats = { 0 };
+
+		for (i=0;i<camel_folder_summary_count(s);i++) {
+			CamelMessageInfo *info = camel_folder_summary_index(s, i);
+			do_stat_mi(cls, &stats, info);
+		}
+
+		printf("\nMemory used by summary:\n\n");
+		printf("Total of %d messages\n", camel_folder_summary_count(s)); 
+		printf("Total: %d bytes (ave %f)\n", stats.citotal + stats.mitotal,
+		       (double)(stats.citotal+stats.mitotal)/(double)camel_folder_summary_count(s));
+		printf("Message Info: %d (ave %f)\n", stats.mitotal, (double)stats.mitotal/(double)stats.micount);
+		printf("Content Info; %d (ave %f) count %d\n", stats.citotal, (double)stats.citotal/(double)stats.cicount, stats.cicount);
+		printf("message id's: %d (ave %f) count %d\n", stats.msgid, (double)stats.msgid/(double)stats.msgcount, stats.msgcount);
+	}
+#endif
+	return ret;
 }
 
 int
@@ -184,6 +290,64 @@ camel_local_summary_add(CamelLocalSummary *cls, CamelMimeMessage *msg, const Cam
 	return ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->add(cls, msg, info, ci, ex);
 }
 
+/**
+ * camel_local_summary_write_headers:
+ * @fd: 
+ * @header: 
+ * @xevline: 
+ * 
+ * Write a bunch of headers to the file @fd.  IF xevline is non NULL, then
+ * an X-Evolution header line is created at the end of all of the headers.
+ * The headers written are termianted with a blank line.
+ * 
+ * Return value: -1 on error, otherwise the number of bytes written.
+ **/
+int
+camel_local_summary_write_headers(int fd, struct _header_raw *header, char *xevline)
+{
+	int outlen = 0, len;
+	int newfd;
+	FILE *out;
+
+	/* dum de dum, maybe the whole sync function should just use stdio for output */
+	newfd = dup(fd);
+	if (newfd == -1)
+		return -1;
+
+	out = fdopen(newfd, "w");
+	if (out == NULL) {
+		close(newfd);
+		errno = EINVAL;
+		return -1;
+	}
+
+	while (header) {
+		if (strcasecmp(header->name, "X-Evolution")) {
+			len = fprintf(out, "%s:%s\n", header->name, header->value);
+			if (len == -1) {
+				fclose(out);
+				return -1;
+			}
+			outlen += len;
+		}
+		header = header->next;
+	}
+
+	if (xevline) {
+		len = fprintf(out, "X-Evolution: %s\n\n", xevline);
+		if (len == -1) {
+			fclose(out);
+			return -1;
+		}
+		outlen += len;
+	}
+
+	if (fclose(out) == -1)
+		return -1;
+
+	return outlen;
+}
+
 #if 0
 static int
 summary_header_load(CamelFolderSummary *s, FILE *in)
@@ -271,6 +435,7 @@ local_summary_add(CamelLocalSummary *cls, CamelMimeMessage *msg, const CamelMess
 
 			mi->flags = mi->flags | (info->flags & 0xffff);
 		}
+		mi->flags &= ~(CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_FLAGGED);
 		xev = camel_local_summary_encode_x_evolution(cls, mi);
 		camel_medium_set_header((CamelMedium *)msg, "X-Evolution", xev);
 		g_free(xev);
@@ -351,16 +516,21 @@ local_summary_decode_x_evolution(CamelLocalSummary *cls, const char *xev, CamelM
 	if (header && strlen(header) == strlen("00000000-0000")
 	    && sscanf(header, "%08x-%04x", &uid, &flags) == 2) {
 		char uidstr[20];
-		sprintf(uidstr, "%u", uid);
-		g_free(mi->uid);
-		mi->uid = g_strdup(uidstr);
-		mi->flags = flags;
+		if (mi) {
+			sprintf(uidstr, "%u", uid);
+			g_free(mi->uid);
+			mi->uid = g_strdup(uidstr);
+			mi->flags = flags;
+		}
 	} else {
 		g_free(header);
 		return -1;
 	}
 	g_free(header);
 
+	if (mi == NULL)
+		return 0;
+
 	/* check for additional data */	
 	header = strchr(xev, ';');
 	if (header) {