Revert Not Zed's previous commits as they apparently can cause serious

2001-08-24  Peter Williams  <peterw@ximian.com>

	* Revert Not Zed's previous commits as they apparently can
	cause serious data loss for POP3 users.

svn path=/trunk/; revision=12461

7 files changed, 97 insertions, 104 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index fd1cf97fb4..33d85a4404 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -1,3 +1,8 @@
+2001-08-24  Peter Williams  <peterw@ximian.com>
+
+	* Revert Not Zed's previous commits as they apparently can
+	cause serious data loss for POP3 users.
+	
 2001-08-24  Ettore Perazzoli  <ettore@ximian.com>
 
 	["Summarizing" and "Synchronizing" are spelt with a "z" in the
diff --git a/camel/camel-data-wrapper.c b/camel/camel-data-wrapper.c
index 3a618ead6f..dd2f475b1e 100644
--- a/camel/camel-data-wrapper.c
+++ b/camel/camel-data-wrapper.c
@@ -71,7 +71,6 @@ camel_data_wrapper_init (gpointer object, gpointer klass)
 
 	camel_data_wrapper->mime_type = header_content_type_new ("application", "octet-stream");
 	camel_data_wrapper->offline = FALSE;
-	camel_data_wrapper->rawtext = FALSE;
 }
 
 static void
diff --git a/camel/camel-data-wrapper.h b/camel/camel-data-wrapper.h
index 9d7b62dd0c..d256c5e842 100644
--- a/camel/camel-data-wrapper.h
+++ b/camel/camel-data-wrapper.h
@@ -48,9 +48,7 @@ struct _CamelDataWrapper
 
 	CamelContentType *mime_type;
 	CamelStream *stream;
-
-	unsigned int offline:1;
-	unsigned int rawtext:1;
+	gboolean offline;
 };
 
 typedef struct {
diff --git a/camel/camel-mime-part-utils.c b/camel/camel-mime-part-utils.c
index bcbc1a2583..d1c41377d1 100644
--- a/camel/camel-mime-part-utils.c
+++ b/camel/camel-mime-part-utils.c
@@ -29,7 +29,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
-#include <errno.h>
 
 #include "string-utils.h"
 #include "camel-mime-part-utils.h"
@@ -124,75 +123,20 @@ check_html_charset (CamelMimeParser *mp, CamelMimeFilterBasicType enctype)
 	return charset;
 }
 
-static GByteArray *convert_buffer(GByteArray *in, const char *to, const char *from)
-{
-	iconv_t ic;
-	int inlen, outlen, convlen, i=2;
-	char *inbuf, *outbuf;
-	char *buffer;
-	GByteArray *out = NULL;
-
-	ic = iconv_open(to, from);
-	if (ic == (iconv_t) -1) {
-		g_warning("Cannot convert from '%s' to '%s': %s", from, to, strerror(errno));
-		return NULL;
-	}
-
-	do {
-		/* make plenty of space? */
-		outlen = in->len * i + 16;
-		buffer = g_malloc(outlen);
-
-		inbuf = in->data;
-		inlen = in->len;
-		outbuf = buffer;
-
-		convlen = iconv(ic, (const char **)&inbuf, &inlen, &outbuf, &outlen);
-		if (convlen == -1) {
-			g_free(buffer);
-			g_warning("conversion failed: %s", strerror(errno));
-			/* we didn't have enough space */
-			if (errno == E2BIG && i<6) {
-				i++;
-				continue;
-			}
-			break;
-		}
-
-		out = g_byte_array_new();
-		g_byte_array_append(out, buffer, convlen);
-
-		/* close off the conversion */
-		outbuf = buffer;
-		outlen = in->len * i + 16;
-		convlen = iconv(ic, NULL, 0, &outbuf, &outlen);
-		if (convlen != -1)
-			g_byte_array_append(out, buffer, convlen);
-		g_free(buffer);
-		break;
-	} while (1);
-
-	iconv_close(ic);
-
-	return out;
-}
-
 /* simple data wrapper */
 static void
 simple_data_wrapper_construct_from_parser (CamelDataWrapper *dw, CamelMimeParser *mp)
 {
-	CamelMimeFilter *fdec = NULL, *fcrlf = NULL /*, *fch = NULL*/;
-	int len, decid = -1, crlfid = -1 /*, chrid = -1*/;
+	CamelMimeFilter *fdec = NULL, *fcrlf = NULL, *fch = NULL;
+	int len, decid = -1, crlfid = -1, chrid = -1;
 	struct _header_content_type *ct;
 	CamelSeekableStream *seekable_source = NULL;
 	CamelStream *source;
 	GByteArray *buffer;
-	/*off_t start = 0, end;*/
+	off_t start = 0, end;
 	char *encoding, *buf;
-	const char *charset = NULL;
 	CamelMimeFilterBasicType enctype = 0;
-	CamelStream *mem;
-
+	
 	d(printf("constructing data-wrapper\n"));
 	
 	/* Ok, try and be smart.  If we're storing a small message (typical) convert it,
@@ -229,7 +173,7 @@ simple_data_wrapper_construct_from_parser (CamelDataWrapper *dw, CamelMimeParser
 	/* If we're doing text, we also need to do CRLF->LF and may have to convert it to UTF8 as well. */
 	ct = camel_mime_parser_content_type (mp);
 	if (header_content_type_is (ct, "text", "*")) {
-		charset = header_content_type_param (ct, "charset");
+		const char *charset = header_content_type_param (ct, "charset");
 		
 		if (fdec) {
 			d(printf("Adding CRLF conversion filter\n"));
@@ -239,50 +183,92 @@ simple_data_wrapper_construct_from_parser (CamelDataWrapper *dw, CamelMimeParser
 		}
 		
 		/* Possible Lame Mailer Alert... check the META tags for a charset */
-		/* FIXME: do this as part of the normal data reading, so we dont have to do it twice */
 		if (!charset && header_content_type_is (ct, "text", "html"))
 			charset = check_html_charset (mp, enctype);
+		
+		/* if the charset is not us-ascii or utf-8, then we need to convert to utf-8 */
+		if (charset && !(g_strcasecmp (charset, "us-ascii") == 0 || g_strcasecmp (charset, "utf-8") == 0)) {
+			d(printf("Adding conversion filter from %s to UTF-8\n", charset));
+			fch = (CamelMimeFilter *)camel_mime_filter_charset_new_convert (charset, "UTF-8");
+			if (fch) {
+				chrid = camel_mime_parser_filter_add (mp, (CamelMimeFilter *)fch);
+			} else {
+				g_warning ("Cannot convert '%s' to 'UTF-8', message display may be corrupt", charset);
+			}
+		}
 	}
 	
-	/* read in the entire content */
-	buffer = g_byte_array_new();
-	while (camel_mime_parser_step(mp, &buf, &len) != HSCAN_BODY_END) {
+	buffer = g_byte_array_new ();
+	
+	if (seekable_source /* !cache */) {
+		start = camel_mime_parser_tell (mp) + seekable_source->bound_start;
+	}
+	
+	while (camel_mime_parser_step (mp, &buf, &len) != HSCAN_BODY_END) {
 		d(printf("appending o/p data: %d: %.*s\n", len, len, buf));
-		g_byte_array_append(buffer, buf, len);
+		if (buffer) {
+			if (buffer->len > 20480 && seekable_source) {
+				/* is this a 'big' message?  Yes?  We dont want to convert it all then. */
+				camel_mime_parser_filter_remove (mp, decid);
+				camel_mime_parser_filter_remove (mp, chrid);
+				decid = -1;
+				chrid = -1;
+				g_byte_array_free (buffer, TRUE);
+				buffer = NULL;
+			} else {
+				g_byte_array_append (buffer, buf, len);
+			}
+		}
 	}
-
-	/* if we need to do charset conversion, see if we can/it works/etc */
-	if (charset && !(strcasecmp(charset, "us-ascii") == 0
-			 || strcasecmp(charset, "utf-8") == 0
-			 || strncasecmp(charset, "x-", 2) == 0)) {
-		GByteArray *out;
-
-		out = convert_buffer(buffer, "UTF-8", charset);
-		if (out) {
-			/* converted ok, use this data instead */
-			g_byte_array_free(buffer, TRUE);
-			buffer = out;
+	
+	if (buffer) {
+		CamelStream *mem;
+		
+		d(printf("Small message part, kept in memory!\n"));
+		
+		mem = camel_stream_mem_new_with_byte_array (buffer);
+		camel_data_wrapper_construct_from_stream (dw, mem);
+		camel_object_unref ((CamelObject *)mem);
+	} else {
+		CamelStream *sub;
+		CamelStreamFilter *filter;
+		
+		d(printf("Big message part, left on disk ...\n"));
+		
+		end = camel_mime_parser_tell (mp) + seekable_source->bound_start;
+		sub = camel_seekable_substream_new_with_seekable_stream_and_bounds (seekable_source, start, end);
+		if (fdec || fch) {
+			filter = camel_stream_filter_new_with_stream (sub);
+			if (fdec) {
+				camel_mime_filter_reset (fdec);
+				camel_stream_filter_add (filter, fdec);
+			}
+			if (fcrlf) {
+				camel_mime_filter_reset (fcrlf);
+				camel_stream_filter_add (filter, fcrlf);
+			}
+			if (fch) {
+				camel_mime_filter_reset (fch);
+				camel_stream_filter_add (filter, fch);
+			}
+			camel_data_wrapper_construct_from_stream (dw, (CamelStream *)filter);
+			camel_object_unref ((CamelObject *)filter);
 		} else {
-			g_warning("Storing text as raw, unknown charset '%s' or invalid format", charset);
-			/* else failed to convert, leave as raw? */
-			dw->rawtext = TRUE;
-			/* should we change the content-type header? */
+			camel_data_wrapper_construct_from_stream (dw, sub);
 		}
+		camel_object_unref ((CamelObject *)sub);
 	}
-
-	d(printf("message part kept in memory!\n"));
-		
-	mem = camel_stream_mem_new_with_byte_array(buffer);
-	camel_data_wrapper_construct_from_stream(dw, mem);
-	camel_object_unref((CamelObject *)mem);
-
+	
 	camel_mime_parser_filter_remove (mp, decid);
 	camel_mime_parser_filter_remove (mp, crlfid);
+	camel_mime_parser_filter_remove (mp, chrid);
 	
 	if (fdec)
 		camel_object_unref ((CamelObject *)fdec);
 	if (fcrlf)
 		camel_object_unref ((CamelObject *)fcrlf);
+	if (fch)
+		camel_object_unref ((CamelObject *)fch);
 	if (source)
 		camel_object_unref ((CamelObject *)source);
 }
@@ -308,8 +294,10 @@ camel_mime_part_construct_content_from_parser (CamelMimePart *dw, CamelMimeParse
 		break;
 	case HSCAN_MULTIPART: {
 		CamelDataWrapper *bodypart;
-
-		/* FIXME: we should use a came-mime-mutlipart, not jsut a camel-multipart, but who cares */
+		
+#ifndef NO_WARNINGS
+#warning This should use a camel-mime-multipart
+#endif
 		d(printf("Creating multi-part\n"));
 		content = (CamelDataWrapper *)camel_multipart_new ();
 		
@@ -333,6 +321,9 @@ camel_mime_part_construct_content_from_parser (CamelMimePart *dw, CamelMimeParse
 		g_warning("Invalid state encountered???: %d", camel_mime_parser_state (mp));
 	}
 	if (content) {
+#ifndef NO_WARNINGS
+#warning there just has got to be a better way ... to transfer the mime-type to the datawrapper
+#endif
 		/* would you believe you have to set this BEFORE you set the content object???  oh my god !!!! */
 		camel_data_wrapper_set_mime_type_field (content, 
 							camel_mime_part_get_content_type ((CamelMimePart *)dw));
diff --git a/camel/camel-mime-part.c b/camel/camel-mime-part.c
index a4d9a2eb0d..a6ab3e58d5 100644
--- a/camel/camel-mime-part.c
+++ b/camel/camel-mime-part.c
@@ -606,10 +606,10 @@ write_to_stream(CamelDataWrapper *data_wrapper, CamelStream *stream)
 			break;
 		}
 		
-		if (!data_wrapper->rawtext && header_content_type_is(mp->content_type, "text", "*")) {
-			charset = header_content_type_param(mp->content_type, "charset");
-			if (charset && !(!strcasecmp(charset, "us-ascii") || !strcasecmp(charset, "utf-8"))) {
-				charenc = (CamelMimeFilter *)camel_mime_filter_charset_new_convert("UTF-8", charset);
+		if (header_content_type_is (mp->content_type, "text", "*")) {
+			charset = header_content_type_param (mp->content_type, "charset");
+			if (charset && !(!g_strcasecmp (charset, "us-ascii") || !g_strcasecmp (charset, "utf-8"))) {
+				charenc = (CamelMimeFilter *)camel_mime_filter_charset_new_convert ("UTF-8", charset);
 			} 
 		}
 		
diff --git a/camel/providers/local/camel-mbox-summary.c b/camel/providers/local/camel-mbox-summary.c
index 065355f730..eed8a9d083 100644
--- a/camel/providers/local/camel-mbox-summary.c
+++ b/camel/providers/local/camel-mbox-summary.c
@@ -499,7 +499,7 @@ mbox_summary_sync_full(CamelLocalSummary *cls, gboolean expunge, CamelFolderChan
 
 	d(printf("performing full summary/sync\n"));
 
-	camel_operation_start(NULL, _("Synchronising folder"));
+	camel_operation_start(NULL, _("Synchronizing folder"));
 
 	fd = open(cls->folder_path, O_RDONLY);
 	if (fd == -1) {
diff --git a/camel/providers/local/camel-spool-summary.c b/camel/providers/local/camel-spool-summary.c
index c5872e8cc2..8a3188296b 100644
--- a/camel/providers/local/camel-spool-summary.c
+++ b/camel/providers/local/camel-spool-summary.c
@@ -376,12 +376,12 @@ summary_rebuild(CamelSpoolSummary *cls, off_t offset, CamelException *ex)
 	/* FIXME: If there is a failure, it shouldn't clear the summary and restart,
 	   it should try and merge the summary info's.  This is a bit tricky. */
 
-	camel_operation_start(NULL, _("Summarising folder"));
+	camel_operation_start(NULL, _("Summarizing folder"));
 
 	fd = open(cls->folder_path, O_RDONLY);
 	if (fd == -1) {
 		printf("%s failed to open: %s\n", cls->folder_path, strerror(errno));
-		camel_exception_setv(ex, 1, _("Could not open folder: %s: summarising from position %ld: %s"),
+		camel_exception_setv(ex, 1, _("Could not open folder: %s: summarizing from position %ld: %s"),
 				     cls->folder_path, offset, strerror(errno));
 		camel_operation_end(NULL);
 		return -1;
@@ -641,7 +641,7 @@ spool_summary_sync_full(CamelSpoolSummary *cls, gboolean expunge, CamelFolderCha
 
 	d(printf("performing full summary/sync\n"));
 
-	camel_operation_start(NULL, _("Synchronising folder"));
+	camel_operation_start(NULL, _("Synchronizing folder"));
 
 	fd = open(cls->folder_path, O_RDWR);
 	if (fd == -1) {
@@ -928,7 +928,7 @@ spool_summary_sync_quick(CamelSpoolSummary *cls, gboolean expunge, CamelFolderCh
 
 	d(printf("Performing quick summary sync\n"));
 
-	camel_operation_start(NULL, _("Synchronising folder"));
+	camel_operation_start(NULL, _("Synchronizing folder"));
 
 	fd = open(cls->folder_path, O_RDWR);
 	if (fd == -1) {