/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*-
 *
 *  Copyright (C) 2001 Ximian Inc. (http://www.ximian.com)
 *
 *  Authors: Michael Zucchi <notzed@ximian.com>
 *
 *  This program is free software; you can redistribute it and/or 
 *  modify it under the terms of the GNU General Public License as 
 *  published by the Free Software Foundation; either version 2 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
 *  USA
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <ctype.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>

#include "camel-spool-summary.h"
#include "camel-mime-message.h"
#include "camel-file-utils.h"
#include "camel-operation.h"

#define io(x)
#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/

#define CAMEL_SPOOL_SUMMARY_VERSION (0x400)

struct _CamelSpoolSummaryPrivate {
};

#define _PRIVATE(o) (((CamelSpoolSummary *)(o))->priv)

static int summary_header_load (CamelFolderSummary *, FILE *);
static int summary_header_save (CamelFolderSummary *, FILE *);

static CamelMessageInfo * message_info_new (CamelFolderSummary *, struct _header_raw *);
static CamelMessageInfo * message_info_new_from_parser(CamelFolderSummary *s, CamelMimeParser *mp);
static CamelMessageInfo * message_info_load (CamelFolderSummary *, FILE *);
static int		  message_info_save (CamelFolderSummary *, FILE *, CamelMessageInfo *);

static int spool_summary_decode_x_evolution(CamelSpoolSummary *cls, const char *xev, CamelMessageInfo *mi);
static char *spool_summary_encode_x_evolution(CamelSpoolSummary *cls, const CamelMessageInfo *mi);

static int spool_summary_load(CamelSpoolSummary *cls, int forceindex, CamelException *ex);
static int spool_summary_check(CamelSpoolSummary *cls, CamelFolderChangeInfo *changeinfo, CamelException *ex);
static int spool_summary_sync(CamelSpoolSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex);
static CamelMessageInfo *spool_summary_add(CamelSpoolSummary *cls, CamelMimeMessage *msg, const CamelMessageInfo *info, CamelFolderChangeInfo *, CamelException *ex);

static int spool_summary_sync_full(CamelSpoolSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex);

static void camel_spool_summary_class_init (CamelSpoolSummaryClass *klass);
static void camel_spool_summary_init       (CamelSpoolSummary *obj);
static void camel_spool_summary_finalise   (CamelObject *obj);
static CamelFolderSummaryClass *camel_spool_summary_parent;

CamelType
camel_spool_summary_get_type(void)
{
	static CamelType type = CAMEL_INVALID_TYPE;
	
	if (type == CAMEL_INVALID_TYPE) {
		type = camel_type_register(camel_folder_summary_get_type(), "CamelSpoolSummary",
					   sizeof (CamelSpoolSummary),
					   sizeof (CamelSpoolSummaryClass),
					   (CamelObjectClassInitFunc) camel_spool_summary_class_init,
					   NULL,
					   (CamelObjectInitFunc) camel_spool_summary_init,
					   (CamelObjectFinalizeFunc) camel_spool_summary_finalise);
	}
	
	return type;
}

static void
camel_spool_summary_class_init(CamelSpoolSummaryClass *klass)
{
	CamelFolderSummaryClass *sklass = (CamelFolderSummaryClass *) klass;
	
	camel_spool_summary_parent = CAMEL_FOLDER_SUMMARY_CLASS(camel_type_get_global_classfuncs(camel_folder_summary_get_type()));

	sklass->summary_header_load = summary_header_load;
	sklass->summary_header_save = summary_header_save;

	sklass->message_info_new  = message_info_new;
	sklass->message_info_new_from_parser  = message_info_new_from_parser;
	sklass->message_info_load = message_info_load;
	sklass->message_info_save = message_info_save;

	klass->load = spool_summary_load;
	klass->check = spool_summary_check;
	klass->sync = spool_summary_sync;
	klass->add = spool_summary_add;

	klass->encode_x_evolution = spool_summary_encode_x_evolution;
	klass->decode_x_evolution = spool_summary_decode_x_evolution;
}

static void
camel_spool_summary_init(CamelSpoolSummary *obj)
{
	struct _CamelSpoolSummaryPrivate *p;
	struct _CamelFolderSummary *s = (CamelFolderSummary *)obj;

	p = _PRIVATE(obj) = g_malloc0(sizeof(*p));

	/* subclasses need to set the right instance data sizes */
	s->message_info_size = sizeof(CamelSpoolMessageInfo);
	s->content_info_size = sizeof(CamelMessageContentInfo);

	/* and a unique file version */
	s->version += CAMEL_SPOOL_SUMMARY_VERSION;
}

static void
camel_spool_summary_finalise(CamelObject *obj)
{
	CamelSpoolSummary *mbs = CAMEL_SPOOL_SUMMARY(obj);

	g_free(mbs->folder_path);
}

CamelSpoolSummary *
camel_spool_summary_new(const char *filename)
{
	CamelSpoolSummary *new = (CamelSpoolSummary *)camel_object_new(camel_spool_summary_get_type());

	camel_folder_summary_set_build_content(CAMEL_FOLDER_SUMMARY(new), FALSE);
	new->folder_path = g_strdup(filename);

	return new;
}

static int
spool_summary_load(CamelSpoolSummary *cls, int forceindex, CamelException *ex)
{
	g_warning("spool_summary_load() should nto b e called\n");

	return camel_folder_summary_load((CamelFolderSummary *)cls);
}

/* load/check the summary */
int
camel_spool_summary_load(CamelSpoolSummary *cls, int forceindex, CamelException *ex)
{
	struct stat st;
	CamelFolderSummary *s = (CamelFolderSummary *)cls;

	g_warning("spool_summary_load() should nto b e called\n");

	d(printf("Loading summary ...\n"));

	if (forceindex
	    || stat(s->summary_path, &st) == -1
	    || ((CamelSpoolSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->load(cls, forceindex, ex) == -1) {
		camel_folder_summary_clear((CamelFolderSummary *)cls);
	}

	return camel_spool_summary_check(cls, NULL, ex);
}

char *
camel_spool_summary_encode_x_evolution(CamelSpoolSummary *cls, const CamelMessageInfo *info)
{
	return ((CamelSpoolSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->encode_x_evolution(cls, info);
}

int
camel_spool_summary_decode_x_evolution(CamelSpoolSummary *cls, const char *xev, CamelMessageInfo *info)
{
	return ((CamelSpoolSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->decode_x_evolution(cls, xev, info);
}

int
camel_spool_summary_check(CamelSpoolSummary *cls, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	int ret;

	ret = ((CamelSpoolSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->check(cls, changeinfo, ex);

	return ret;
}

int
camel_spool_summary_sync(CamelSpoolSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	return ((CamelSpoolSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->sync(cls, expunge, changeinfo, ex);
}

CamelMessageInfo *
camel_spool_summary_add(CamelSpoolSummary *cls, CamelMimeMessage *msg, const CamelMessageInfo *info, CamelFolderChangeInfo *ci, CamelException *ex)
{
	return ((CamelSpoolSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->add(cls, msg, info, ci, ex);
}

/**
 * camel_spool_summary_write_headers:
 * @fd: 
 * @header: 
 * @xevline: 
 * 
 * Write a bunch of headers to the file @fd.  IF xevline is non NULL, then
 * an X-Evolution header line is created at the end of all of the headers.
 * The headers written are termianted with a blank line.
 * 
 * Return value: -1 on error, otherwise the number of bytes written.
 **/
int
camel_spool_summary_write_headers(int fd, struct _header_raw *header, char *xevline)
{
	int outlen = 0, len;
	int newfd;
	FILE *out;

	/* dum de dum, maybe the whole sync function should just use stdio for output */
	newfd = dup(fd);
	if (newfd == -1)
		return -1;

	out = fdopen(newfd, "w");
	if (out == NULL) {
		close(newfd);
		errno = EINVAL;
		return -1;
	}

	while (header) {
		if (strcmp(header->name, "X-Evolution")) {
			len = fprintf(out, "%s:%s\n", header->name, header->value);
			if (len == -1) {
				fclose(out);
				return -1;
			}
			outlen += len;
		}
		header = header->next;
	}

	if (xevline) {
		len = fprintf(out, "X-Evolution: %s\n\n", xevline);
		if (len == -1) {
			fclose(out);
			return -1;
		}
		outlen += len;
	}

	if (fclose(out) == -1)
		return -1;

	return outlen;
}

static int
summary_header_load(CamelFolderSummary *s, FILE *in)
{
	CamelSpoolSummary *mbs = CAMEL_SPOOL_SUMMARY(s);

	if (((CamelFolderSummaryClass *)camel_spool_summary_parent)->summary_header_load(s, in) == -1)
		return -1;

	return camel_file_util_decode_uint32(in, &mbs->folder_size);
}

static int
summary_header_save(CamelFolderSummary *s, FILE *out)
{
	CamelSpoolSummary *mbs = CAMEL_SPOOL_SUMMARY(s);

	if (((CamelFolderSummaryClass *)camel_spool_summary_parent)->summary_header_save(s, out) == -1)
		return -1;

	return camel_file_util_encode_uint32(out, mbs->folder_size);
}

static CamelMessageInfo *
message_info_new(CamelFolderSummary *s, struct _header_raw *h)
{
	CamelMessageInfo *mi;
	CamelSpoolSummary *cls = (CamelSpoolSummary *)s;

	mi = ((CamelFolderSummaryClass *)camel_spool_summary_parent)->message_info_new(s, h);
	if (mi) {
		CamelSpoolMessageInfo *mbi = (CamelSpoolMessageInfo *)mi;
		const char *xev;

		xev = header_raw_find(&h, "X-Evolution", NULL);
		if (xev==NULL || camel_spool_summary_decode_x_evolution(cls, xev, mi) == -1) {
			/* to indicate it has no xev header */
			mi->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED | CAMEL_MESSAGE_FOLDER_NOXEV;
			camel_message_info_set_uid(mi, camel_folder_summary_next_uid_string(s));
		}

		mbi->frompos = -1;
	}
	
	return mi;
}

static CamelMessageInfo *
message_info_new_from_parser(CamelFolderSummary *s, CamelMimeParser *mp)
{
	CamelMessageInfo *mi;

	mi = ((CamelFolderSummaryClass *)camel_spool_summary_parent)->message_info_new_from_parser(s, mp);
	if (mi) {
		CamelSpoolMessageInfo *mbi = (CamelSpoolMessageInfo *)mi;

		mbi->frompos = camel_mime_parser_tell_start_from(mp);
	}
	
	return mi;
}

static CamelMessageInfo *
message_info_load(CamelFolderSummary *s, FILE *in)
{
	CamelMessageInfo *mi;

	io(printf("loading spool message info\n"));

	mi = ((CamelFolderSummaryClass *)camel_spool_summary_parent)->message_info_load(s, in);
	if (mi) {
		CamelSpoolMessageInfo *mbi = (CamelSpoolMessageInfo *)mi;
		
		if (camel_file_util_decode_off_t(in, &mbi->frompos) == -1)
			goto error;
	}
	
	return mi;
error:
	camel_folder_summary_info_free(s, mi);
	return NULL;
}

static int
message_info_save(CamelFolderSummary *s, FILE *out, CamelMessageInfo *mi)
{
	CamelSpoolMessageInfo *mbi = (CamelSpoolMessageInfo *)mi;

	io(printf("saving spool message info\n"));

	if (((CamelFolderSummaryClass *)camel_spool_summary_parent)->message_info_save(s, out, mi) == -1
	    || camel_file_util_encode_off_t(out, mbi->frompos) == -1)
		return -1;

	return 0;
}

static int
summary_rebuild(CamelSpoolSummary *cls, off_t offset, CamelException *ex)
{
	CamelFolderSummary *s = (CamelFolderSummary *)cls;
	CamelMimeParser *mp;
	int fd;
	int ok = 0;
	struct stat st;
	off_t size = 0;

	/* FIXME: If there is a failure, it shouldn't clear the summary and restart,
	   it should try and merge the summary info's.  This is a bit tricky. */

	camel_operation_start(NULL, _("Summarising folder"));

	fd = open(cls->folder_path, O_RDONLY);
	if (fd == -1) {
		printf("%s failed to open: %s", cls->folder_path, strerror(errno));
		camel_exception_setv(ex, 1, _("Could not open folder: %s: summarising from position %ld: %s"),
				     cls->folder_path, offset, strerror(errno));
		camel_operation_end(NULL);
		return -1;
	}
	
	if (fstat(fd, &st) == 0)
		size = st.st_size;

	mp = camel_mime_parser_new();
	camel_mime_parser_init_with_fd(mp, fd);
	camel_mime_parser_scan_from(mp, TRUE);
	camel_mime_parser_seek(mp, offset, SEEK_SET);

	if (offset > 0) {
		if (camel_mime_parser_step(mp, NULL, NULL) == HSCAN_FROM) {
			if (camel_mime_parser_tell_start_from(mp) != offset) {
				g_warning("The next message didn't start where I expected, building summary from start");
				camel_mime_parser_drop_step(mp);
				offset = 0;
				camel_mime_parser_seek(mp, offset, SEEK_SET);
				camel_folder_summary_clear(s);
			} else {
				camel_mime_parser_unstep(mp);
			}
		} else {
			d(printf("mime parser state ran out? state is %d\n", camel_mime_parser_state(mp)));
			camel_object_unref(CAMEL_OBJECT(mp));
			/* end of file - no content? no error either */
			camel_operation_end(NULL);
			return 0;
		}
	}

	while (camel_mime_parser_step(mp, NULL, NULL) == HSCAN_FROM) {
		CamelMessageInfo *info;
		off_t pc = camel_mime_parser_tell_start_from (mp) + 1;
		
		camel_operation_progress (NULL, (int) (((float) pc / size) * 100));
		
		info = camel_folder_summary_add_from_parser(s, mp);
		if (info == NULL) {
			camel_exception_setv(ex, 1, _("Fatal mail parser error near position %ld in folder %s"),
					     camel_mime_parser_tell(mp), cls->folder_path);
			ok = -1;
			break;
		}

		g_assert(camel_mime_parser_step(mp, NULL, NULL) == HSCAN_FROM_END);
	}

	camel_object_unref(CAMEL_OBJECT (mp));
	
	/* update the file size/mtime in the summary */
	if (ok != -1) {
		if (stat(cls->folder_path, &st) == 0) {
			camel_folder_summary_touch(s);
			cls->folder_size = st.st_size;
			s->time = st.st_mtime;
		}
	}

	camel_operation_end(NULL);

	return ok;
}

/* like summary_rebuild, but also do changeinfo stuff (if supplied) */
static int
summary_update(CamelSpoolSummary *cls, off_t offset, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	int ret, i, count;
	CamelFolderSummary *s = (CamelFolderSummary *)cls;

	d(printf("Calling summary update, from pos %d\n", (int)offset));

	if (changeinfo) {
		/* we use the diff function of the change_info to build the update list. */
		for (i = 0; i < camel_folder_summary_count(s); i++) {
			CamelMessageInfo *mi = camel_folder_summary_index(s, i);

			camel_folder_change_info_add_source(changeinfo, camel_message_info_uid(mi));
			camel_folder_summary_info_free(s, mi);
		}
	}

	/* do the actual work */
	ret = summary_rebuild(cls, offset, ex);

	if (changeinfo) {
		count = camel_folder_summary_count(s);
		for (i = 0; i < count; i++) {
			CamelMessageInfo *mi = camel_folder_summary_index(s, i);
			camel_folder_change_info_add_update(changeinfo, camel_message_info_uid(mi));
			camel_folder_summary_info_free(s, mi);
		}
		camel_folder_change_info_build_diff(changeinfo);
	}

	return ret;
}

static int
spool_summary_check(CamelSpoolSummary *cls, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	CamelFolderSummary *s = (CamelFolderSummary *)cls;
	struct stat st;
	int ret = 0;

	d(printf("Checking summary\n"));

	/* check if the summary is up-to-date */
	if (stat(cls->folder_path, &st) == -1) {
		camel_folder_summary_clear(s);
		camel_exception_setv(ex, 1, _("Cannot summarise folder: %s: %s"), cls->folder_path, strerror(errno));
		return -1;
	}

	if (st.st_size == 0) {
		/* empty?  No need to scan at all */
		d(printf("Empty spool, clearing summary\n"));
		camel_folder_summary_clear(s);
		ret = 0;
	} else if (s->messages->len == 0) {
		/* if we are empty, then we rebuilt from scratch */
		d(printf("Empty summary, rebuilding from start\n"));
		ret = summary_update(cls, 0, changeinfo, ex);
	} else {
		/* is the summary uptodate? */
		if (st.st_size != cls->folder_size || st.st_mtime != s->time) {
			if (cls->folder_size < st.st_size) {
				/* this will automatically rescan from 0 if there is a problem */
				d(printf("folder grew, attempting to rebuild from %d\n", cls->folder_size));
				ret = summary_update(cls, cls->folder_size, changeinfo, ex);
			} else {
				d(printf("folder shrank!  rebuilding from start\n"));
				camel_folder_summary_clear(s);
				ret = summary_update(cls, 0, changeinfo, ex);
			}
		}
	}

	if (ret != -1) {
		int i, work, count;

		/* check to see if we need to copy/update the file; missing xev headers prompt this */
		work = FALSE;
		count = camel_folder_summary_count(s);
		for (i=0;!work && i<count; i++) {
			CamelMessageInfo *info = camel_folder_summary_index(s, i);
			g_assert(info);
			work = (info->flags & (CAMEL_MESSAGE_FOLDER_NOXEV)) != 0;
			camel_folder_summary_info_free(s, info);
		}

		/* if we do, then write out the headers using sync_full, etc */
		if (work) {
			d(printf("Have to add new headers, re-syncing from the start to accomplish this\n"));
			ret = spool_summary_sync_full(cls, FALSE, changeinfo, ex);

			if (stat(cls->folder_path, &st) == -1) {
				camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Unknown error: %s"), strerror(errno));
				return -1;
			}
		}
		cls->folder_size = st.st_size;
		s->time = st.st_mtime;
	}

	return ret;
}

static char *tz_months[] = {
	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};

static char *tz_days[] = {
	"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
};

/* tries to build a From line, based on message headers */
char *
camel_spool_summary_build_from(struct _header_raw *header)
{
	GString *out = g_string_new("From ");
	char *ret;
	const char *tmp;
	time_t thetime;
	int offset;
	struct tm tm;

	tmp = header_raw_find(&header, "Sender", NULL);
	if (tmp == NULL)
		tmp = header_raw_find(&header, "From", NULL);
	if (tmp != NULL) {
		struct _header_address *addr = header_address_decode(tmp);

		tmp = NULL;
		if (addr) {
			if (addr->type == HEADER_ADDRESS_NAME) {
				g_string_append(out, addr->v.addr);
				tmp = "";
			}
			header_address_unref(addr);
		}
	}
	if (tmp == NULL) {
		g_string_append(out, "unknown@nodomain.now.au");
	}

	/* try use the received header to get the date */
	tmp = header_raw_find(&header, "Received", NULL);
	if (tmp) {
		tmp = strrchr(tmp, ';');
		if (tmp)
			tmp++;
	}

	/* if there isn't one, try the Date field */
	if (tmp == NULL)
		tmp = header_raw_find(&header, "Date", NULL);

	thetime = header_decode_date(tmp, &offset);

	thetime += ((offset / 100) * (60 * 60)) + (offset % 100) * 60;

	/* a pseudo, but still bogus attempt at thread safing the function */
	/*memcpy(&tm, gmtime(&thetime), sizeof(tm));*/
	gmtime_r(&thetime, &tm);

	g_string_sprintfa(out, " %s %s %d %02d:%02d:%02d %4d\n",
			  tz_days[tm.tm_wday],
			  tz_months[tm.tm_mon], tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, tm.tm_year + 1900);

	ret = out->str;
	g_string_free(out, FALSE);
	return ret;
}

/* perform a full sync */
static int
spool_summary_sync_full(CamelSpoolSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	CamelFolderSummary *s = (CamelFolderSummary *)cls;
	CamelMimeParser *mp = NULL;
	int i, count;
	CamelSpoolMessageInfo *info = NULL;
	int fd = -1, fdout = -1;
	char *tmpname = NULL;
	char *buffer, *xevnew = NULL, *p;
	int len;
	const char *fromline;
	int lastdel = FALSE;
	off_t spoollen, outlen;
	int size, sizeout;
	struct stat st;

	d(printf("performing full summary/sync\n"));

	camel_operation_start(NULL, _("Synchronising folder"));

	fd = open(cls->folder_path, O_RDWR);
	if (fd == -1) {
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not open folder to summarise: %s: %s"),
				     cls->folder_path, strerror(errno));
		camel_operation_end(NULL);
		return -1;
	}

	mp = camel_mime_parser_new();
	camel_mime_parser_scan_from(mp, TRUE);
	camel_mime_parser_scan_pre_from(mp, TRUE);
	camel_mime_parser_init_with_fd(mp, fd);

#ifdef HAVE_MKSTEMP
	tmpname = alloca(64);
	sprintf(tmpname, "/tmp/spool.camel.XXXXXX");
	fdout = mkstemp(tmpname);
#else
#warning "Your system has no mkstemp(3), spool updating may be insecure"
	tmpname = alloca(L_tmpnam);
	tmpnam(tmpname);
	fdout = open(tmpname, O_RDWR|O_CREAT|O_EXCL, 0600);
#endif
	d(printf("Writing tmp file to %s\n", tmpname));
	if (fdout == -1) {
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Cannot open temporary mailbox: %s"), strerror(errno));
		goto error;
	}

	count = camel_folder_summary_count(s);
	for (i = 0; i < count; i++) {
		int pc = (i + 1) * 100 / count;

		camel_operation_progress(NULL, pc);

		info = (CamelSpoolMessageInfo *)camel_folder_summary_index(s, i);

		g_assert(info);

		d(printf("Looking at message %s\n", camel_message_info_uid(info)));

		/* only need to seek past deleted messages, otherwise we should be at the right spot/state already */
		if (lastdel) {
			d(printf("seeking to %d\n", (int)info->frompos));
			camel_mime_parser_seek(mp, info->frompos, SEEK_SET);
		}

		if (camel_mime_parser_step(mp, &buffer, &len) != HSCAN_FROM) {
			g_warning("Expected a From line here, didn't get it");
			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
					     _("Summary and folder mismatch, even after a sync"));
			goto error;
		}

		if (camel_mime_parser_tell_start_from(mp) != info->frompos) {
			g_warning("Didn't get the next message where I expected (%d) got %d instead",
				  (int)info->frompos, (int)camel_mime_parser_tell_start_from(mp));
			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
					     _("Summary and folder mismatch, even after a sync"));
			goto error;
		}

		lastdel = FALSE;
		if (expunge && info->info.flags & CAMEL_MESSAGE_DELETED) {
			const char *uid = camel_message_info_uid(info);

			d(printf("Deleting %s\n", uid));

#if 0
			if (cls->index)
				ibex_unindex(cls->index, (char *)uid);
#endif
			/* remove it from the change list */
			camel_folder_change_info_remove_uid(changeinfo, uid);
			camel_folder_summary_remove(s, (CamelMessageInfo *)info);
			camel_folder_summary_info_free(s, (CamelMessageInfo *)info);
			count--;
			i--;
			info = NULL;
			lastdel = TRUE;
		} else {
			/* otherwise, the message is staying, copy its From_ line across */
			if (i>0) {
				write(fdout, "\n", 1);
			}
			info->frompos = lseek(fdout, 0, SEEK_CUR);
			fromline = camel_mime_parser_from_line(mp);
			write(fdout, fromline, strlen(fromline));
		}

		if (info && info->info.flags & (CAMEL_MESSAGE_FOLDER_NOXEV | CAMEL_MESSAGE_FOLDER_FLAGGED)) {
			d(printf("Updating header for %s flags = %08x\n", camel_message_info_uid(info), info->info.flags));

			if (camel_mime_parser_step(mp, &buffer, &len) == HSCAN_FROM_END) {
				g_warning("camel_mime_parser_step failed (2)");
				goto error;
			}

			xevnew = camel_spool_summary_encode_x_evolution(cls, (CamelMessageInfo *)info);
			if (camel_spool_summary_write_headers(fdout, camel_mime_parser_headers_raw(mp), xevnew) == -1) {
				d(printf("Error writing to tmp mailbox\n"));
				camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
						     _("Error writing to temp mailbox: %s"),
						     strerror(errno));
				goto error;
			}
			info->info.flags &= 0xffff;
			g_free(xevnew);
			xevnew = NULL;
			camel_mime_parser_drop_step(mp);
		}

		camel_mime_parser_drop_step(mp);
		if (info) {
			d(printf("looking for message content to copy across from %d\n", (int)camel_mime_parser_tell(mp)));
			while (camel_mime_parser_step(mp, &buffer, &len) == HSCAN_PRE_FROM) {
				d(printf("copying spool contents to tmp: '%.*s'\n", len, buffer));
				if (write(fdout, buffer, len) != len) {
					camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
							     _("Writing to tmp mailbox failed: %s: %s"),
							     cls->folder_path, strerror(errno));
					goto error;
				}
			}
			d(printf("we are now at %d, from = %d\n", (int)camel_mime_parser_tell(mp),
				 (int)camel_mime_parser_tell_start_from(mp)));
			camel_mime_parser_unstep(mp);
			camel_folder_summary_info_free(s, (CamelMessageInfo *)info);
			info = NULL;
		}
	}

	/* sync out content */
	if (fsync(fdout) == -1) {
		g_warning("Cannot sync temporary folder: %s", strerror(errno));
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not sync temporary folder %s: %s"),
				     cls->folder_path, strerror(errno));
		goto error;
	}

	/* see if we can write this much to the spool file */
	if (fstat(fd, &st) == -1) {
		g_warning("Cannot sync temporary folder: %s", strerror(errno));
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not sync temporary folder %s: %s"),
				     cls->folder_path, strerror(errno));
		goto error;
	}
	spoollen = st.st_size;

	if (fstat(fdout, &st) == -1) {
		g_warning("Cannot sync temporary folder: %s", strerror(errno));
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not sync temporary folder %s: %s"),
				     cls->folder_path, strerror(errno));
		goto error;
	}
	outlen = st.st_size;

	/* I think this is the right way to do this */
	if (outlen>0
	    && (lseek(fd, outlen-1, SEEK_SET) == -1
		|| write(fd, "", 1) != 1
		|| fsync(fd) == -1
		|| lseek(fd, 0, SEEK_SET) == -1
		|| lseek(fdout, 0, SEEK_SET) == -1)) {
		g_warning("Cannot sync spool folder: %s", strerror(errno));
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not sync spool folder %s: %s"),
				     cls->folder_path, strerror(errno));
		/* incase we ran out of room, remove any trailing space first */
		ftruncate(fd, spoollen);
		goto error;
	}


	/* now copy content back */
	buffer = g_malloc(8192);
	size = 1;
	while (size>0) {
		do {
			size = read(fdout, buffer, 8192);
		} while (size == -1 && errno == EINTR);

		if (size > 0) {
			p = buffer;
			do {
				sizeout = write(fd, p, size);
				if (sizeout > 0) {
					p+= sizeout;
					size -= sizeout;
				}
			} while ((sizeout == -1 && errno == EINTR) && size > 0);
			size = sizeout;
		}

		if (size == -1) {
			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
					     _("Could not sync spool folder %s: %s\n"
					       "Folder may be corrupt, copy saved in `%s'"),
					     cls->folder_path, strerror(errno), tmpnam);
			/* so we dont delete it */
			close(fdout);
			tmpname = NULL;
			fdout = -1;
			g_free(buffer);
			goto error;
		}
	}

	g_free(buffer);

	d(printf("Closing folders\n"));

	if (ftruncate(fd, outlen) == -1) {
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not sync spool folder %s: %s\n"
				       "Folder may be corrupt, copy saved in `%s'"),
				     cls->folder_path, strerror(errno), tmpnam);
		close(fdout);
		tmpname = NULL;
		fdout = -1;
		goto error;
	}

	if (close(fd) == -1) {
		g_warning("Cannot close source folder: %s", strerror(errno));
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not sync spool folder %s: %s\n"
				       "Folder may be corrupt, copy saved in `%s'"),
				     cls->folder_path, strerror(errno), tmpnam);
		close(fdout);
		tmpname = NULL;
		fdout = -1;
		fd = -1;
		goto error;
	}

	close(fdout);
	unlink(tmpname);

	camel_object_unref((CamelObject *)mp);
	camel_operation_end(NULL);
		
	return 0;
 error:
	if (fd != -1)
		close(fd);
	
	if (fdout != -1)
		close(fdout);
	
	g_free(xevnew);
	
	if (tmpname)
		unlink(tmpname);
	if (mp)
		camel_object_unref((CamelObject *)mp);
	if (info)
		camel_folder_summary_info_free(s, (CamelMessageInfo *)info);

	camel_operation_end(NULL);

	return -1;
}

/* perform a quick sync - only system flags have changed */
static int
spool_summary_sync_quick(CamelSpoolSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	CamelFolderSummary *s = (CamelFolderSummary *)cls;
	CamelMimeParser *mp = NULL;
	int i, count;
	CamelSpoolMessageInfo *info = NULL;
	int fd = -1;
	char *xevnew, *xevtmp;
	const char *xev;
	int len;
	off_t lastpos;

	d(printf("Performing quick summary sync\n"));

	camel_operation_start(NULL, _("Synchronising folder"));

	fd = open(cls->folder_path, O_RDWR);
	if (fd == -1) {
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not open folder to summarise: %s: %s"),
				     cls->folder_path, strerror(errno));

		camel_operation_end(NULL);
		return -1;
	}

	mp = camel_mime_parser_new();
	camel_mime_parser_scan_from(mp, TRUE);
	camel_mime_parser_scan_pre_from(mp, TRUE);
	camel_mime_parser_init_with_fd(mp, fd);

	count = camel_folder_summary_count(s);
	for (i = 0; i < count; i++) {
		int xevoffset;
		int pc = (i+1)*100/count;

		camel_operation_progress(NULL, pc);

		info = (CamelSpoolMessageInfo *)camel_folder_summary_index(s, i);

		g_assert(info);

		d(printf("Checking message %s %08x\n", camel_message_info_uid(info), info->info.flags));

		if ((info->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED) == 0) {
			camel_folder_summary_info_free(s, (CamelMessageInfo *)info);
			info = NULL;
			continue;
		}

		d(printf("Updating message %s\n", camel_message_info_uid(info)));

		camel_mime_parser_seek(mp, info->frompos, SEEK_SET);

		if (camel_mime_parser_step(mp, 0, 0) != HSCAN_FROM) {
			g_warning("Expected a From line here, didn't get it");
			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
					     _("Summary and folder mismatch, even after a sync"));
			goto error;
		}

		if (camel_mime_parser_tell_start_from(mp) != info->frompos) {
			g_warning("Didn't get the next message where I expected (%d) got %d instead",
				  (int)info->frompos, (int)camel_mime_parser_tell_start_from(mp));
			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
					     _("Summary and folder mismatch, even after a sync"));
			goto error;
		}

		if (camel_mime_parser_step(mp, 0, 0) == HSCAN_FROM_END) {
			g_warning("camel_mime_parser_step failed (2)");
			goto error;
		}

		xev = camel_mime_parser_header(mp, "X-Evolution", &xevoffset);
		if (xev == NULL || camel_spool_summary_decode_x_evolution(cls, xev, NULL) == -1) {
			g_warning("We're supposed to have a valid x-ev header, but we dont");
			goto error;
		}
		xevnew = camel_spool_summary_encode_x_evolution(cls, (CamelMessageInfo *)info);
		/* SIGH: encode_param_list is about the only function which folds headers by itself.
		   This should be fixed somehow differently (either parser doesn't fold headers,
		   or param_list doesn't, or something */
		xevtmp = header_unfold(xevnew);
		/* the raw header contains a leading ' ', so (dis)count that too */
		if (strlen(xev)-1 != strlen(xevtmp)) {
			g_free(xevnew);
			g_free(xevtmp);
			g_warning("Hmm, the xev headers shouldn't have changed size, but they did");
			goto error;
		}
		g_free(xevtmp);

		/* we write out the xevnew string, assuming its been folded identically to the original too! */

		lastpos = lseek(fd, 0, SEEK_CUR);
		lseek(fd, xevoffset+strlen("X-Evolution: "), SEEK_SET);
		do {
			len = write(fd, xevnew, strlen(xevnew));
		} while (len == -1 && errno == EINTR);
		lseek(fd, lastpos, SEEK_SET);
		g_free(xevnew);

		camel_mime_parser_drop_step(mp);
		camel_mime_parser_drop_step(mp);

		info->info.flags &= 0xffff;
		camel_folder_summary_info_free(s, (CamelMessageInfo *)info);
	}

	d(printf("Closing folders\n"));

	if (close(fd) == -1) {
		g_warning("Cannot close source folder: %s", strerror(errno));
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
				     _("Could not close source folder %s: %s"),
				     cls->folder_path, strerror(errno));
		fd = -1;
		goto error;
	}

	camel_object_unref((CamelObject *)mp);

	camel_operation_end(NULL);
	
	return 0;
 error:
	if (fd != -1)
		close(fd);
	if (mp)
		camel_object_unref((CamelObject *)mp);
	if (info)
		camel_folder_summary_info_free(s, (CamelMessageInfo *)info);

	camel_operation_end(NULL);

	return -1;
}

static int
spool_summary_sync(CamelSpoolSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex)
{
	struct stat st;
	CamelFolderSummary *s = (CamelFolderSummary *)cls;
	int i, count;
	int quick = TRUE, work=FALSE;
	int ret;

	/* first, sync ourselves up, just to make sure */
	summary_update(cls, cls->folder_size, changeinfo, ex);
	if (camel_exception_is_set(ex))
		return -1;

	count = camel_folder_summary_count(s);
	if (count == 0)
		return 0;

	/* check what work we have to do, if any */
	for (i=0;quick && i<count; i++) {
		CamelMessageInfo *info = camel_folder_summary_index(s, i);
		g_assert(info);
		if ((expunge && (info->flags & CAMEL_MESSAGE_DELETED)) ||
		    (info->flags & (CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_XEVCHANGE)))
			quick = FALSE;
		else
			work |= (info->flags & CAMEL_MESSAGE_FOLDER_FLAGGED) != 0;
		camel_folder_summary_info_free(s, info);
	}

	/* yuck i hate this logic, but its to simplify the 'all ok, update summary' and failover cases */
	ret = -1;
	if (quick) {
		if (work) {
			ret = spool_summary_sync_quick(cls, expunge, changeinfo, ex);
			if (ret == -1) {
				g_warning("failed a quick-sync, trying a full sync");
				camel_exception_clear(ex);
			}
		} else {
			ret = 0;
		}
	}

	if (ret == -1)
		ret = spool_summary_sync_full(cls, expunge, changeinfo, ex);
	if (ret == -1)
		return -1;

	if (stat(cls->folder_path, &st) == -1) {
		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Unknown error: %s"), strerror(errno));
		return -1;
	}

	camel_folder_summary_touch(s);
	s->time = st.st_mtime;
	cls->folder_size = st.st_size;
	/*camel_folder_summary_save(s);*/

	return 0;
}


static CamelMessageInfo *
spool_summary_add(CamelSpoolSummary *cls, CamelMimeMessage *msg, const CamelMessageInfo *info, CamelFolderChangeInfo *ci, CamelException *ex)
{
	CamelMessageInfo *mi;
	char *xev;

	d(printf("Adding message to summary\n"));
	
	mi = camel_folder_summary_add_from_message((CamelFolderSummary *)cls, msg);
	if (mi) {
		d(printf("Added, uid = %s\n", camel_message_info_uid(mi)));
		if (info) {
			CamelTag *tag = info->user_tags;
			CamelFlag *flag = info->user_flags;

			while (flag) {
				camel_flag_set(&mi->user_flags, flag->name, TRUE);
				flag = flag->next;
			}
			
			while (tag) {
				camel_tag_set(&mi->user_tags, tag->name, tag->value);
				tag = tag->next;
			}

			mi->flags = mi->flags | (info->flags & 0xffff);
		}
		mi->flags &= ~(CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_FLAGGED);
		xev = camel_spool_summary_encode_x_evolution(cls, mi);
		camel_medium_set_header((CamelMedium *)msg, "X-Evolution", xev);
		g_free(xev);
		camel_folder_change_info_add_uid(ci, camel_message_info_uid(mi));
	} else {
		d(printf("Failed!\n"));
		camel_exception_set(ex, 1, _("Unable to add message to summary: unknown reason"));
	}
	return mi;
}

static char *
spool_summary_encode_x_evolution(CamelSpoolSummary *cls, const CamelMessageInfo *mi)
{
	GString *out = g_string_new("");
	struct _header_param *params = NULL;
	GString *val = g_string_new("");
	CamelFlag *flag = mi->user_flags;
	CamelTag *tag = mi->user_tags;
	char *ret;
	const char *p, *uidstr;
	guint32 uid;

	/* FIXME: work out what to do with uid's that aren't stored here? */
	/* FIXME: perhaps make that a mbox folder only issue?? */
	p = uidstr = camel_message_info_uid(mi);
	while (*p && isdigit(*p))
		p++;
	if (*p == 0 && sscanf(uidstr, "%u", &uid) == 1) {
		g_string_sprintf(out, "%08x-%04x", uid, mi->flags & 0xffff);
	} else {
		g_string_sprintf(out, "%s-%04x", uidstr, mi->flags & 0xffff);
	}

	if (flag || tag) {
		val = g_string_new("");

		if (flag) {
			while (flag) {
				g_string_append(val, flag->name);
				if (flag->next)
					g_string_append_c(val, ',');
				flag = flag->next;
			}
			header_set_param(&params, "flags", val->str);
			g_string_truncate(val, 0);
		}
		if (tag) {
			while (tag) {
				g_string_append(val, tag->name);
				g_string_append_c(val, '=');
				g_string_append(val, tag->value);
				if (tag->next)
					g_string_append_c(val, ',');
				tag = tag->next;
			}
			header_set_param(&params, "tags", val->str);
		}
		g_string_free(val, TRUE);
		header_param_list_format_append(out, params);
		header_param_list_free(params);
	}
	ret = out->str;
	g_string_free(out, FALSE);
	return ret;
}

static int
spool_summary_decode_x_evolution(CamelSpoolSummary *cls, const char *xev, CamelMessageInfo *mi)
{
	struct _header_param *params, *scan;
	guint32 uid, flags;
	char *header;
	int i;

	/* check for uid/flags */
	header = header_token_decode(xev);
	if (header && strlen(header) == strlen("00000000-0000")
	    && sscanf(header, "%08x-%04x", &uid, &flags) == 2) {
		char uidstr[20];
		if (mi) {
			sprintf(uidstr, "%u", uid);
			camel_message_info_set_uid(mi, g_strdup(uidstr));
			mi->flags = flags;
		}
	} else {
		g_free(header);
		return -1;
	}
	g_free(header);

	if (mi == NULL)
		return 0;

	/* check for additional data */	
	header = strchr(xev, ';');
	if (header) {
		params = header_param_list_decode(header+1);
		scan = params;
		while (scan) {
			if (!strcasecmp(scan->name, "flags")) {
				char **flagv = g_strsplit(scan->value, ",", 1000);

				for (i=0;flagv[i];i++) {
					camel_flag_set(&mi->user_flags, flagv[i], TRUE);
				}
				g_strfreev(flagv);
			} else if (!strcasecmp(scan->name, "tags")) {
				char **tagv = g_strsplit(scan->value, ",", 10000);
				char *val;

				for (i=0;tagv[i];i++) {
					val = strchr(tagv[i], '=');
					if (val) {
						*val++ = 0;
						camel_tag_set(&mi->user_tags, tagv[i], val);
						val[-1]='=';
					}
				}
				g_strfreev(tagv);
			}
			scan = scan->next;
		}
		header_param_list_free(params);
	}
	return 0;
}