Unref the output_stream when done, close doesn't do it. (_append_message):

2000-03-27  NotZed  <NotZed@HelixCode.com>

        * providers/mbox/camel-mbox-folder.c (_append_message): Unref the
        output_stream when done, close doesn't do it.
        (_append_message): Clear all uid's from the appending messages, so
        they are reassigned proper unique id's.

        * gmime-utils.c (get_header_array_from_stream): Actually free the
        header, it is copied elsewhere.

2000-03-26  NotZed  <NotZed@HelixCode.com>

        * providers/mbox/camel-mbox-utils.c (camel_mbox_write_xev): Added
        folder parameter to function.  Fixed callers.
        (index_message): Index a message as it is assigned a unique id.

        * camel-mime-part.c (my_set_content_id): Make sure we malloc and
        copy the content_id, otherwise *poof*

2000-03-25  NotZed  <NotZed@HelixCode.com>

        * camel-medium.c (_finalize): Another leak, unref the content if
        finished with it.

        * camel-recipient.c (camel_recipient_table_free): Plug another
        memory leak - actually free the recipient table.

        * camel-mime-message.c (_finalize): Plugged a memory leak with the
        flags table.

        * gmime-utils.c (_store_header_pair_from_string): A simpler, more
        debuggable and functionally identical header extraction function.

2000-03-24  NotZed  <NotZed@HelixCode.com>

        * gmime-content-field.c (gmime_content_field_set_parameter):
        Remove the hash table entry before freeing its key and data.

svn path=/trunk/; revision=2199

6 files changed, 247 insertions, 41 deletions

diff --git a/camel/providers/mbox/camel-mbox-folder.c b/camel/providers/mbox/camel-mbox-folder.c
index e4136c5b35..faf6bd57cd 100644
--- a/camel/providers/mbox/camel-mbox-folder.c
+++ b/camel/providers/mbox/camel-mbox-folder.c
@@ -256,9 +256,8 @@ _check_get_or_maybe_generate_summary_file (CamelMboxFolder *mbox_folder,
 		if (camel_exception_get_id (ex))
 			return;
 
-		next_uid = camel_mbox_write_xev (mbox_folder->folder_file_path,
-						 message_info_array,
-						 &file_size, next_uid, ex);
+		next_uid = camel_mbox_write_xev (mbox_folder, mbox_folder->folder_file_path, 
+						 message_info_array, &file_size, next_uid, ex);
 
 		if (camel_exception_get_id (ex)) { 
 			/* ** FIXME : free the preparsed information */
@@ -284,6 +283,12 @@ _open (CamelFolder *folder, CamelFolderOpenMode mode, CamelException *ex)
 {
 	CamelMboxFolder *mbox_folder = CAMEL_MBOX_FOLDER (folder);
 
+	mbox_folder->index = ibex_open(mbox_folder->index_file_path, O_CREAT|O_RDWR, 0600);
+	if (mbox_folder->index == NULL) {
+		g_warning("Could not open/create index file: %s: indexing will not function",
+			  strerror(errno));
+	}
+
 	/* call parent class */
 	parent_class->open (folder, mode, ex);
 	if (camel_exception_get_id(ex))
@@ -307,6 +312,11 @@ _close (CamelFolder *folder, gboolean expunge, CamelException *ex)
 	/* call parent implementation */
 	parent_class->close (folder, expunge, ex);
 
+	/* save index */
+	if (mbox_folder->index) {
+		ibex_close(mbox_folder->index);
+	}
+
 	/* save the folder summary on disk */
 	camel_mbox_summary_save (CAMEL_MBOX_SUMMARY (folder->summary),
 				 mbox_folder->summary_file_path, ex);
@@ -331,19 +341,21 @@ _set_name (CamelFolder *folder, const gchar *name, CamelException *ex)
 	g_free (mbox_folder->folder_dir_path);
 	g_free (mbox_folder->index_file_path);
 
-	root_dir_path = camel_mbox_store_get_toplevel_dir (CAMEL_MBOX_STORE (folder->parent_store));
-
-	mbox_folder->folder_file_path =
-		g_strdup_printf ("%s/%s", root_dir_path, folder->full_name);
-	mbox_folder->summary_file_path =
-		g_strdup_printf ("%s/%s-ev-summary", root_dir_path,
-				 folder->full_name);
-	mbox_folder->folder_dir_path =
-		g_strdup_printf ("%s/%s.sdb", root_dir_path,
-				 folder->full_name);
-	mbox_folder->index_file_path =
-		g_strdup_printf ("%s/%s.ibex", root_dir_path,
-				 folder->full_name);
+	root_dir_path = camel_mbox_store_get_toplevel_dir (CAMEL_MBOX_STORE(folder->parent_store));
+
+	CAMEL_LOG_FULL_DEBUG ("CamelMboxFolder::set_name full_name is %s\n", folder->full_name);
+	CAMEL_LOG_FULL_DEBUG ("CamelMboxFolder::set_name root_dir_path is %s\n", root_dir_path);
+
+	mbox_folder->folder_file_path = g_strdup_printf ("%s/%s", root_dir_path, folder->full_name);
+	mbox_folder->summary_file_path = g_strdup_printf ("%s/%s-ev-summary", root_dir_path, folder->full_name);
+	mbox_folder->folder_dir_path = g_strdup_printf ("%s/%s.sdb", root_dir_path, folder->full_name);
+	mbox_folder->index_file_path = g_strdup_printf ("%s/%s.ibex", root_dir_path, folder->full_name);
+
+	CAMEL_LOG_FULL_DEBUG ("CamelMboxFolder::set_name mbox_folder->folder_file_path is %s\n", 
+			      mbox_folder->folder_file_path);
+	CAMEL_LOG_FULL_DEBUG ("CamelMboxFolder::set_name mbox_folder->folder_dir_path is %s\n", 
+			      mbox_folder->folder_dir_path);
+	CAMEL_LOG_FULL_DEBUG ("Leaving CamelMboxFolder::set_name\n");
 }
 
 
@@ -476,9 +488,10 @@ _create (CamelFolder *folder, CamelException *ex)
 	/* it must be rw for the user and none for the others */
 	creat_fd = open (folder_file_path, 
 			 O_WRONLY | O_CREAT | O_APPEND,
-			 S_IRUSR  | S_IWUSR, 0600); 
+			 0600);
 	if (creat_fd == -1)
 		goto io_error;
+
 	close (creat_fd);
 
 	/* create the summary object */
@@ -650,7 +663,7 @@ _delete_messages (CamelFolder *folder, CamelException *ex)
 	/* it must be rw for the user and none for the others */
 	creat_fd = open (folder_file_path, 
 			 O_WRONLY | O_TRUNC,
-			 S_IRUSR  | S_IWUSR, 0600); 
+			 0600); 
 	if (creat_fd == -1)
 		goto io_error;
 	close (creat_fd);
@@ -832,6 +845,7 @@ _append_message (CamelFolder *folder, CamelMimeMessage *message, CamelException
 	GArray *mbox_summary_info;
 	gchar *tmp_message_filename;
 	gint fd1, fd2;
+	int i;
 
 	CAMEL_LOG_FULL_DEBUG ("Entering CamelMboxFolder::append_message\n");
 
@@ -846,6 +860,7 @@ _append_message (CamelFolder *folder, CamelMimeMessage *message, CamelException
 		camel_data_wrapper_write_to_stream (CAMEL_DATA_WRAPPER (message), output_stream);
 	}
 	camel_stream_close (output_stream);
+	gtk_object_unref(output_stream);
 
 	/* at this point we have saved the message to a
 	   temporary file, now, we have to add the x-evolution 
@@ -869,15 +884,21 @@ _append_message (CamelFolder *folder, CamelMimeMessage *message, CamelException
 	close (tmp_file_fd);
 
 	/* get the value of the last available UID
-	   as saved in the summary file */
+	   as saved in the summary file, again */
 	next_uid = summary->next_uid;
 
+	/* make sure all our of message info's have 0 uid - ignore any
+	   set elsewhere */
+	for (i=0;i<message_info_array->len;i++) {
+		g_array_index(message_info_array, CamelMboxParserMessageInfo, i).uid = 0;
+	}
+
 	/* 
 	   OK, this is not very efficient, we should not use the same
 	   method as for parsing an entire mail file, 
 	   but I have no time to write a simpler parser 
 	*/
-	next_uid = camel_mbox_write_xev (tmp_message_filename, 
+	next_uid = camel_mbox_write_xev (folder, tmp_message_filename, 
 					 message_info_array, &tmp_file_size, next_uid, ex);
 	
 	if (camel_exception_get_id (ex)) { 
@@ -904,7 +925,7 @@ _append_message (CamelFolder *folder, CamelMimeMessage *message, CamelException
 	fd1 = open (tmp_message_filename, O_RDONLY);
 	fd2 = open (mbox_folder->folder_file_path, 
 		    O_WRONLY | O_CREAT | O_APPEND,
-		    S_IRUSR  | S_IWUSR, 0600);
+		    0600);
 
 	if (fd2 == -1) {
 		camel_exception_setv (ex, 
diff --git a/camel/providers/mbox/camel-mbox-folder.h b/camel/providers/mbox/camel-mbox-folder.h
index ba17642144..52d3fa6e70 100644
--- a/camel/providers/mbox/camel-mbox-folder.h
+++ b/camel/providers/mbox/camel-mbox-folder.h
@@ -35,6 +35,8 @@ extern "C" {
 
 #include <gtk/gtk.h>
 #include "camel-folder.h"
+#include "camel-mbox-summary.h"
+#include "libibex/ibex.h"
 
 /*  #include "camel-store.h" */
 
@@ -54,6 +56,7 @@ typedef struct {
 
 	GList  *uid_array;
 
+	ibex *index;		/* index for this folder */
 	int search_id;		/* next search id */
 	GList *searches;	/* current searches */
 } CamelMboxFolder;
diff --git a/camel/providers/mbox/camel-mbox-parser.c b/camel/providers/mbox/camel-mbox-parser.c
index b17c5c1e95..7c0eec5379 100644
--- a/camel/providers/mbox/camel-mbox-parser.c
+++ b/camel/providers/mbox/camel-mbox-parser.c
@@ -488,6 +488,8 @@ read_header (CamelMboxPreParser *parser, gchar **header_content)
 		c = buffer[parser->current_position];		
 	}
 
+
+	/* FIXME: this can cause a memory leak, for duplicated headers? */
 	
 	/* copy the buffer in the preparsing information structure */
 	*header_content = g_strndup (parser->tmp_string->str, parser->tmp_string->len);	
@@ -647,10 +649,7 @@ camel_mbox_parse_file (int fd,
 
 	while (parser->buffer[parser->current_position] != '\0') {
 
-		
-
-		
-		/* read the current character */
+	        /* read the current character */
 		if (!newline) {
 			c = parser->buffer[parser->current_position];
 			newline = (c == '\n');
diff --git a/camel/providers/mbox/camel-mbox-search.c b/camel/providers/mbox/camel-mbox-search.c
index 8a55ffee1d..0b1b9c3206 100644
--- a/camel/providers/mbox/camel-mbox-search.c
+++ b/camel/providers/mbox/camel-mbox-search.c
@@ -302,9 +302,9 @@ int camel_mbox_folder_search_by_expression(CamelFolder *folder, const char *expr
 	/* FIXME: the index should be global to the folder */
 	ctx->message_info = CAMEL_MBOX_SUMMARY(ctx->summary)->message_info;
 	ctx->message_current = NULL;
-	ctx->index = ibex_open(CAMEL_MBOX_FOLDER(folder)->index_file_path, FALSE);
+	ctx->index = CAMEL_MBOX_FOLDER(folder)->index;
 	if (!ctx->index) {
-		perror("Cannot open index file (ignored)");
+		g_warning("No folder index, searches will not function fully");
 	}
 
 	((CamelMboxFolder *)folder)->searches = g_list_append(((CamelMboxFolder *)folder)->searches, ctx);
@@ -338,9 +338,6 @@ int camel_mbox_folder_search_by_expression(CamelFolder *folder, const char *expr
 		printf("no result!\n");
 	}
 
-	if (ctx->index)
-		ibex_close(ctx->index);
-
 	gtk_object_unref((GtkObject *)ctx->summary);
 	gtk_object_unref((GtkObject *)f);
 	i = ctx->id;
diff --git a/camel/providers/mbox/camel-mbox-utils.c b/camel/providers/mbox/camel-mbox-utils.c
index 1b16941148..52f28b9fa6 100644
--- a/camel/providers/mbox/camel-mbox-utils.c
+++ b/camel/providers/mbox/camel-mbox-utils.c
@@ -58,8 +58,10 @@
 #include "camel-mbox-utils.h"
 #include "camel-mbox-parser.h"
 #include "camel-mbox-summary.h"
-
-
+#include "camel-mime-message.h"
+#include "camel/camel-mime-part.h"
+#include "camel/camel-multipart.h"
+#include "camel/camel-stream-fs.h"
 
 static gchar b64_alphabet[64] = 
 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
@@ -209,9 +211,180 @@ camel_mbox_copy_file_chunk (gint fd_src,
 	
 }
 
+typedef void (*index_data_callback)(ibex *index, char *text, int len, int *left);
+
+/*
+  needs to handle encoding?
+*/
+static void
+index_text(ibex *index, char *text, int len, int *left)
+{
+	/*printf("indexing %.*s\n", len, text);*/
+
+	ibex_index_buffer(index, "message", text, len, left);
+/*
+	if (left) {
+		printf("%d bytes left from indexing\n", *left);
+	}
+*/
+}
+
+/*
+  index html data, ignore tags for now.
+    could also index attribute values?
+    should also handle encoding types ...
+    should convert everything to utf8
+*/
+static void
+index_html(ibex *index, char *text, int len, int *left)
+{
+	static int state = 0;
+	char indexbuf[128];
+	char *out = indexbuf, *outend = indexbuf+128;
+	char *in, *inend;
+	int c;
+
+	in = text;
+	inend = text+len;
+
+	/*printf("indexing html: %d %d %.*s\n", state, len, len, text);*/
+
+	while (in<inend) {
+		c = *in++;
+		switch (state) {
+		case 0:			/* no tag */
+			if (c=='<')
+				state = 1;
+			else {
+				*out++ = c;
+				if (out==outend) {
+					index_text(index, indexbuf, out-indexbuf, left);
+					memcpy(indexbuf, indexbuf+(out-indexbuf)-*left, *left);
+					out = indexbuf+*left;
+					printf("** %d bytes left\n", *left);
+				}
+			}
+			break;
+		case 1:
+			if (c=='>')
+				state = 0;
+#if 0
+			else if (c=='"')
+				state = 2;
+			break;
+		case 2:
+			if (c=='"') {
+				state = 1;
+			}
+#endif
+			break;
+		}
+	}
+	index_text(index, indexbuf, out-indexbuf, left);
+}
+
+static void
+index_message_content(ibex *index, CamelDataWrapper *object)
+{
+	CamelDataWrapper *containee;
+	CamelStream *stream;
+	int parts, i;
+	int len;
+	int left;
+	char buffer[128];
+
+	containee = camel_medium_get_content_object(CAMEL_MEDIUM(object));
+
+	if (containee) {
+		char *type = gmime_content_field_get_mime_type(containee->mime_type);
+		index_data_callback callback = NULL;
+
+		/*printf("type = %s\n", type);*/
+
+		if (!strcasecmp(type, "text/plain")) {
+			callback = index_text;
+		} else if (!strcasecmp(type, "text/html")) {
+			callback = index_html;
+		} else if (!strncasecmp(type, "multipart/", 10)) {
+			parts = camel_multipart_get_number (CAMEL_MULTIPART(containee));
+			/*printf("multipart message, scanning contents  %d parts ...\n", parts);*/
+			for (i=0;i<parts;i++) {
+				index_message_content(index, camel_multipart_get_part(CAMEL_MULTIPART(containee), i));
+			}
+		} else {
+			/*printf("\nunknwon format, ignored\n");*/
+		}
+
+		if (callback) {
+			int total=0;
+
+			/*printf("reading containee\n");
+
+			  printf("containee = %p\n", containee);*/
+
+			stream = camel_data_wrapper_get_output_stream(containee);
+			left = 0;
+
+			if (stream) {
+				/*printf("stream = %p\n", stream);*/
+				while ( (len = camel_stream_read(stream, buffer+left, sizeof(buffer)-left)) > 0) {
+					total = len+left;
+					callback(index, buffer, total, &left);
+					if (left>0) {
+						memcpy(buffer, buffer+total-left, left);
+					}
+				}
+				callback(index, buffer+total-left, left, NULL);
+				
+				/*camel_stream_close(stream);*/
+				/*printf("\n");*/
+			} else {
+				g_warning("cannot get stream for message?");
+			}
+		}
+
+		g_free(type);
+	} else {
+		printf("no containee?\n");
+	}
+}
+
+
+static void
+index_message(ibex *index, int fd, CamelMboxParserMessageInfo *mi)
+{
+	off_t pos;
+	CamelStream *stream;
+	CamelMimeMessage *message;
+	int newfd;
+	int i;
+
+	if (index != NULL) {
+		/*printf("indexing message\n %s\n %d for %d bytes\n", mi->from, mi->message_position, mi->size);*/
+		pos = lseek(fd, 0, SEEK_CUR);
+		
+		/* the stream will close the fd we have */
+		newfd = dup(fd);
+		stream = camel_stream_fs_new_with_fd_and_bounds(newfd, mi->message_position, mi->message_position + mi->size);
+		message = camel_mime_message_new_with_session( (CamelSession *)NULL);
+		
+		camel_data_wrapper_set_input_stream (
+			CAMEL_DATA_WRAPPER (message), stream);
+		
+		index_message_content(index, message);
+		
+		/*	printf("messageid = '%s'\n", message->message_uid);*/
+		
+		gtk_object_unref (message);
+		gtk_object_unref (GTK_OBJECT (stream));
+		
+		lseek(fd, pos, SEEK_SET);
+	}
+}
 
 guint32
-camel_mbox_write_xev (gchar *mbox_file_name,
+camel_mbox_write_xev (CamelMboxFolder *folder,
+		      gchar *mbox_file_name,
 		      GArray *summary_information, 
 		      guint32 *file_size,
 		      guint32  next_uid, 
@@ -230,14 +403,15 @@ camel_mbox_write_xev (gchar *mbox_file_name,
 	gchar *tmp_file_name_secure;
 	gint rename_result;
 	gint unlink_result;
-	
+	int changed = FALSE;
+
 	tmp_file_name = g_strdup_printf ("%s__.ev_tmp", mbox_file_name);
 	tmp_file_name_secure = g_strdup_printf ("%s__.ev_tmp_secure", mbox_file_name);
 
 	fd1 = open (mbox_file_name, O_RDONLY);
 	fd2 = open (tmp_file_name, 
 		    O_WRONLY | O_CREAT | O_TRUNC ,
-		    S_IRUSR  | S_IWUSR, 0600);
+		    0600);
 
 	if (fd2 == -1) {
 			camel_exception_setv (ex, 
@@ -264,7 +438,11 @@ camel_mbox_write_xev (gchar *mbox_file_name,
 
 			cur_pos = cur_msg_info->message_position 
 				+ cur_msg_info->end_of_headers_offset;
-			
+
+			cur_msg_info->uid = next_free_uid;			
+			index_message(folder->index, fd1, cur_msg_info);
+			changed = TRUE;
+
 			camel_mbox_copy_file_chunk (fd1, fd2, bytes_to_copy, ex);
 			if (camel_exception_get_id (ex)) {
 				close (fd1);
@@ -272,8 +450,8 @@ camel_mbox_write_xev (gchar *mbox_file_name,
 				goto end;
 			}
 			
-			cur_msg_info->uid = next_free_uid;
 			cur_msg_info->status = 0;
+
 			camel_mbox_xev_write_header_content (xev_header + 12, next_free_uid, 0);
 			next_free_uid++;
 			write (fd2, xev_header, 19);
@@ -283,10 +461,16 @@ camel_mbox_write_xev (gchar *mbox_file_name,
 			cur_msg_info->x_evolution = g_strdup_printf ("%.6s", xev_header + 12);
 			cur_msg_info->end_of_headers_offset += 19;
 			*file_size += 19;
-		} 
-		cur_msg_info->message_position += cur_offset;
+			cur_msg_info->message_position += cur_offset;
+		} else {
+			cur_msg_info->message_position += cur_offset;
+		}
+	}
+
+	/* make sure the index is in sync */
+	if (changed) {
+		ibex_write(folder->index);
 	}
-	
 	
 	bytes_to_copy = end_of_last_message - cur_pos;
 		camel_mbox_copy_file_chunk (fd1, fd2, bytes_to_copy, ex);
diff --git a/camel/providers/mbox/camel-mbox-utils.h b/camel/providers/mbox/camel-mbox-utils.h
index 062a284b03..8142f97c91 100644
--- a/camel/providers/mbox/camel-mbox-utils.h
+++ b/camel/providers/mbox/camel-mbox-utils.h
@@ -35,6 +35,7 @@ extern "C" {
 
 
 #include "camel-exception.h"
+#include "camel-mbox-folder.h"
 
 void 
 camel_mbox_xev_parse_header_content (gchar header_content[6], 
@@ -47,7 +48,8 @@ camel_mbox_xev_write_header_content (gchar header_content[6],
 				     guchar status);
 
 guint32
-camel_mbox_write_xev (gchar *mbox_file_name,
+camel_mbox_write_xev (CamelMboxFolder *folder,
+		      gchar *mbox_file_name,
 		      GArray *summary_information, 
 		      guint32 *file_size,
 		      guint32 last_uid,