Rewritten to use a cache for body searches when online. Will need some

2002-01-14  Not Zed  <NotZed@Ximian.com>

        * providers/imap/camel-imap-search.c (imap_body_contains):
        Rewritten to use a cache for body searches when online.  Will need
        some heavy testing but so far seems to be beneficial.

        * providers/imap/camel-imap-folder.c (imap_search_by_expression,
        search_by_uids): dont initialise search object here.
        (camel_imap_folder_new): Setup search object here with pointer to
        cache dir.

2001-12-01  Not Zed  <NotZed@Ximian.com>

        * camel-store-summary.[ch]: New class to store a store's folder
        list in.  Not yet completed.

svn path=/trunk/; revision=15314

8 files changed, 1480 insertions, 111 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index 873a13f18e..5c35d3eede 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -1,3 +1,19 @@
+2002-01-14  Not Zed  <NotZed@Ximian.com>
+
+	* providers/imap/camel-imap-search.c (imap_body_contains):
+	Rewritten to use a cache for body searches when online.  Will need
+	some heavy testing but so far seems to be beneficial.
+
+	* providers/imap/camel-imap-folder.c (imap_search_by_expression,
+	search_by_uids): dont initialise search object here.
+	(camel_imap_folder_new): Setup search object here with pointer to
+	cache dir.
+
+2001-12-01  Not Zed  <NotZed@Ximian.com>
+
+	* camel-store-summary.[ch]: New class to store a store's folder
+	list in.  Not yet completed.
+
 2002-01-11  Jeffrey Stedfast  <fejj@ximian.com>
 
 	* providers/imap/camel-imap-folder.c (imap_update_summary): Kludge
diff --git a/camel/Makefile.am b/camel/Makefile.am
index 5c2075567a..055c303534 100644
--- a/camel/Makefile.am
+++ b/camel/Makefile.am
@@ -86,6 +86,7 @@ libcamel_la_SOURCES = 				\
 	camel-service.c				\
 	camel-session.c				\
 	camel-store.c				\
+	camel-store-summary.c			\
 	camel-stream-buffer.c			\
 	camel-stream-filter.c			\
 	camel-stream-fs.c			\
@@ -172,6 +173,7 @@ libcamelinclude_HEADERS =			\
 	camel-service.h				\
 	camel-session.h				\
 	camel-store.h				\
+	camel-store-summary.h			\
 	camel-stream-buffer.h			\
 	camel-stream-filter.h			\
 	camel-stream-fs.h			\
diff --git a/camel/camel-private.h b/camel/camel-private.h
index b73ef9a36d..7eb12e6ae2 100644
--- a/camel/camel-private.h
+++ b/camel/camel-private.h
@@ -164,6 +164,23 @@ struct _CamelFolderSummaryPrivate {
 #define CAMEL_SUMMARY_UNLOCK(f, l)
 #endif
 
+struct _CamelStoreSummaryPrivate {
+#ifdef ENABLE_THREADS
+	GMutex *summary_lock;	/* for the summary hashtable/array */
+	GMutex *io_lock;	/* load/save lock, for access to saved_count, etc */
+	GMutex *alloc_lock;	/* for setting up and using allocators */
+	GMutex *ref_lock;	/* for reffing/unreffing messageinfo's ALWAYS obtain before summary_lock */
+#endif
+};
+
+#ifdef ENABLE_THREADS
+#define CAMEL_STORE_SUMMARY_LOCK(f, l) (g_mutex_lock(((CamelStoreSummary *)f)->priv->l))
+#define CAMEL_STORE_SUMMARY_UNLOCK(f, l) (g_mutex_unlock(((CamelStoreSummary *)f)->priv->l))
+#else
+#define CAMEL_STORE_SUMMARY_LOCK(f, l)
+#define CAMEL_STORE_SUMMARY_UNLOCK(f, l)
+#endif
+
 struct _CamelVeeFolderPrivate {
 	GList *folders;			/* lock using subfolder_lock before changing/accessing */
 	GList *folders_changed;		/* for list of folders that have changed between updates */
diff --git a/camel/camel-store-summary.c b/camel/camel-store-summary.c
new file mode 100644
index 0000000000..8cd2bf37dd
--- /dev/null
+++ b/camel/camel-store-summary.c
@@ -0,0 +1,899 @@
+/*
+ * Copyright (C) 2001 Ximian Inc.
+ *
+ * Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <unistd.h>
+#include <ctype.h>
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+#include "camel-store-summary.h"
+
+#include "camel-file-utils.h"
+
+#include "hash-table-utils.h"
+#include "e-util/md5-utils.h"
+#include "e-util/e-memory.h"
+
+#include "camel-private.h"
+
+#define d(x)
+#define io(x)			/* io debug */
+
+#define CAMEL_STORE_SUMMARY_VERSION (13)
+
+#define _PRIVATE(o) (((CamelStoreSummary *)(o))->priv)
+
+static int summary_header_load(CamelStoreSummary *, FILE *);
+static int summary_header_save(CamelStoreSummary *, FILE *);
+
+static CamelFolderInfo * folder_info_new(CamelStoreSummary *, const char *);
+static CamelFolderInfo * folder_info_load(CamelStoreSummary *, FILE *);
+static int		 folder_info_save(CamelStoreSummary *, FILE *, CamelFolderInfo *);
+static void		 folder_info_free(CamelStoreSummary *, CamelFolderInfo *);
+
+static const char *folder_info_string(CamelStoreSummary *, const CamelFolderInfo *, int);
+static void folder_info_set_string(CamelStoreSummary *, CamelFolderInfo *, int, const char *);
+
+static void camel_store_summary_class_init (CamelStoreSummaryClass *klass);
+static void camel_store_summary_init       (CamelStoreSummary *obj);
+static void camel_store_summary_finalise   (CamelObject *obj);
+
+static CamelObjectClass *camel_store_summary_parent;
+
+static void
+camel_store_summary_class_init (CamelStoreSummaryClass *klass)
+{
+	camel_store_summary_parent = camel_type_get_global_classfuncs (camel_object_get_type ());
+
+	klass->summary_header_load = summary_header_load;
+	klass->summary_header_save = summary_header_save;
+
+	klass->folder_info_new  = folder_info_new;
+	klass->folder_info_load = folder_info_load;
+	klass->folder_info_save = folder_info_save;
+	klass->folder_info_free = folder_info_free;
+
+	klass->folder_info_string = folder_info_string;
+	klass->folder_info_set_string = folder_info_set_string;
+}
+
+static void
+camel_store_summary_init (CamelStoreSummary *s)
+{
+	struct _CamelStoreSummaryPrivate *p;
+
+	p = _PRIVATE(s) = g_malloc0(sizeof(*p));
+
+	s->folder_info_size = sizeof(CamelFolderInfo);
+
+	s->folder_info_chunks = NULL;
+
+	s->version = CAMEL_STORE_SUMMARY_VERSION;
+	s->flags = 0;
+	s->count = 0;
+	s->time = 0;
+
+	s->folders = g_ptr_array_new();
+	s->folders_full = g_hash_table_new(g_str_hash, g_str_equal);
+
+#ifdef ENABLE_THREADS
+	p->summary_lock = g_mutex_new();
+	p->io_lock = g_mutex_new();
+	p->alloc_lock = g_mutex_new();
+	p->ref_lock = g_mutex_new();
+#endif
+}
+
+static void
+camel_store_summary_finalise (CamelObject *obj)
+{
+	struct _CamelStoreSummaryPrivate *p;
+	CamelStoreSummary *s = (CamelStoreSummary *)obj;
+
+	p = _PRIVATE(obj);
+
+	camel_store_summary_clear(s);
+	g_ptr_array_free(s->folders, TRUE);
+	g_hash_table_destroy(s->folders_full);
+
+	g_free(s->summary_path);
+
+	if (s->folder_info_chunks)
+		e_memchunk_destroy(s->folder_info_chunks);
+
+#ifdef ENABLE_THREADS
+	g_mutex_free(p->summary_lock);
+	g_mutex_free(p->io_lock);
+	g_mutex_free(p->alloc_lock);
+	g_mutex_free(p->ref_lock);
+#endif
+
+	g_free(p);
+}
+
+CamelType
+camel_store_summary_get_type (void)
+{
+	static CamelType type = CAMEL_INVALID_TYPE;
+	
+	if (type == CAMEL_INVALID_TYPE) {
+		type = camel_type_register (camel_object_get_type (), "CamelStoreSummary",
+					    sizeof (CamelStoreSummary),
+					    sizeof (CamelStoreSummaryClass),
+					    (CamelObjectClassInitFunc) camel_store_summary_class_init,
+					    NULL,
+					    (CamelObjectInitFunc) camel_store_summary_init,
+					    (CamelObjectFinalizeFunc) camel_store_summary_finalise);
+	}
+	
+	return type;
+}
+
+/**
+ * camel_store_summary_new:
+ *
+ * Create a new CamelStoreSummary object.
+ * 
+ * Return value: A new CamelStoreSummary widget.
+ **/
+CamelStoreSummary *
+camel_store_summary_new (void)
+{
+	CamelStoreSummary *new = CAMEL_STORE_SUMMARY ( camel_object_new (camel_store_summary_get_type ()));	return new;
+}
+
+/**
+ * camel_store_summary_set_filename:
+ * @s: 
+ * @name: 
+ * 
+ * Set the filename where the summary will be loaded to/saved from.
+ **/
+void camel_store_summary_set_filename(CamelStoreSummary *s, const char *name)
+{
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	g_free(s->summary_path);
+	s->summary_path = g_strdup(name);
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+}
+
+void camel_store_summary_set_uri_prefix(CamelStoreSummary *s, const char *prefix)
+{
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	g_free(s->uri_prefix);
+	s->uri_prefix = g_strdup(prefix);
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+}
+
+/**
+ * camel_store_summary_count:
+ * @s: 
+ * 
+ * Get the number of summary items stored in this summary.
+ * 
+ * Return value: The number of items int he summary.
+ **/
+int
+camel_store_summary_count(CamelStoreSummary *s)
+{
+	return s->folders->len;
+}
+
+/**
+ * camel_store_summary_index:
+ * @s: 
+ * @i: 
+ * 
+ * Retrieve a summary item by index number.
+ *
+ * A referenced to the summary item is returned, which may be
+ * ref'd or free'd as appropriate.
+ * 
+ * Return value: The summary item, or NULL if the index @i is out
+ * of range.
+ * It must be freed using camel_store_summary_info_free().
+ **/
+CamelFolderInfo *
+camel_store_summary_index(CamelStoreSummary *s, int i)
+{
+	CamelFolderInfo *info = NULL;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, ref_lock);
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	if (i<s->folders->len)
+		info = g_ptr_array_index(s->folders, i);
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+
+	if (info)
+		info->refcount++;
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+
+	return info;
+}
+
+/**
+ * camel_store_summary_index:
+ * @s: 
+ * @i: 
+ * 
+ * Obtain a copy of the summary array.  This is done atomically,
+ * so cannot contain empty entries.
+ *
+ * It must be freed using camel_store_summary_array_free().
+ **/
+GPtrArray *
+camel_store_summary_array(CamelStoreSummary *s)
+{
+	CamelFolderInfo *info;
+	GPtrArray *res = g_ptr_array_new();
+	int i;
+	
+	CAMEL_STORE_SUMMARY_LOCK(s, ref_lock);
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	g_ptr_array_set_size(res, s->folders->len);
+	for (i=0;i<s->folders->len;i++) {
+		info = res->pdata[i] = g_ptr_array_index(s->folders, i);
+		info->refcount++;
+	}
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+	CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+
+	return res;
+}
+
+/**
+ * camel_store_summary_array_free:
+ * @s: 
+ * @array: 
+ * 
+ * Free the folder summary array.
+ **/
+void
+camel_store_summary_array_free(CamelStoreSummary *s, GPtrArray *array)
+{
+	int i;
+
+	for (i=0;i<array->len;i++)
+		camel_store_summary_info_free(s, array->pdata[i]);
+
+	g_ptr_array_free(array, TRUE);
+}
+
+/**
+ * camel_store_summary_full:
+ * @s: 
+ * @full: 
+ * 
+ * Retrieve a summary item by full name.
+ *
+ * A referenced to the summary item is returned, which may be
+ * ref'd or free'd as appropriate.
+ * 
+ * Return value: The summary item, or NULL if the @full name
+ * is not available.
+ * It must be freed using camel_store_summary_info_free().
+ **/
+CamelFolderInfo *
+camel_store_summary_full(CamelStoreSummary *s, const char *full)
+{
+	CamelFolderInfo *info;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, ref_lock);
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	info = g_hash_table_lookup(s->folders_full, full);
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+
+	if (info)
+		info->refcount++;
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+
+	return info;
+}
+
+int
+camel_store_summary_load(CamelStoreSummary *s)
+{
+	FILE *in;
+	int i;
+	CamelFolderInfo *mi;
+
+	g_assert(s->summary_path);
+
+	in = fopen(s->summary_path, "r");
+	if (in == NULL)
+		return -1;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, io_lock);
+	if ( ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, in) == -1)
+		goto error;
+
+	/* now read in each message ... */
+	for (i=0;i<s->count;i++) {
+		mi = ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->folder_info_load(s, in);
+
+		if (mi == NULL)
+			goto error;
+
+		camel_store_summary_add(s, mi);
+	}
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, io_lock);
+	
+	if (fclose(in) == -1)
+		return -1;
+
+	s->flags &= ~CAMEL_STORE_SUMMARY_DIRTY;
+
+	return 0;
+
+error:
+	g_warning("Cannot load summary file: %s", strerror(ferror(in)));
+	CAMEL_STORE_SUMMARY_UNLOCK(s, io_lock);
+	fclose(in);
+	s->flags |= ~CAMEL_STORE_SUMMARY_DIRTY;
+
+	return -1;
+}
+
+/**
+ * camel_store_summary_save:
+ * @s: 
+ * 
+ * Writes the summary to disk.  The summary is only written if changes
+ * have occured.
+ * 
+ * Return value: Returns -1 on error.
+ **/
+int
+camel_store_summary_save(CamelStoreSummary *s)
+{
+	FILE *out;
+	int fd;
+	int i;
+	guint32 count;
+	CamelFolderInfo *mi;
+
+	g_assert(s->summary_path);
+
+	if ((s->flags & CAMEL_STORE_SUMMARY_DIRTY) == 0)
+		return 0;
+
+	fd = open(s->summary_path, O_RDWR|O_CREAT, 0600);
+	if (fd == -1)
+		return -1;
+	out = fdopen(fd, "w");
+	if ( out == NULL ) {
+		close(fd);
+		return -1;
+	}
+
+	io(printf("saving header\n"));
+
+	CAMEL_STORE_SUMMARY_LOCK(s, io_lock);
+
+	if ( ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_save(s, out) == -1) {
+		fclose(out);
+		CAMEL_STORE_SUMMARY_UNLOCK(s, io_lock);
+		return -1;
+	}
+
+	/* now write out each message ... */
+
+	/* FIXME: Locking? */
+
+	count = s->folders->len;
+	for (i=0;i<count;i++) {
+		mi = s->folders->pdata[i];
+		((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->folder_info_save(s, out, mi);
+	}
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, io_lock);
+
+	if (fclose(out) == -1)
+		return -1;
+
+	s->flags &= ~CAMEL_STORE_SUMMARY_DIRTY;
+	return 0;
+}
+
+/**
+ * camel_store_summary_header_load:
+ * @s: Summary object.
+ * 
+ * Only load the header information from the summary,
+ * keep the rest on disk.  This should only be done on
+ * a fresh summary object.
+ * 
+ * Return value: -1 on error.
+ **/
+int camel_store_summary_header_load(CamelStoreSummary *s)
+{
+	FILE *in;
+	int ret;
+
+	g_assert(s->summary_path);
+
+	in = fopen(s->summary_path, "r");
+	if (in == NULL)
+		return -1;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, io_lock);
+	ret = ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, in);
+	CAMEL_STORE_SUMMARY_UNLOCK(s, io_lock);
+	
+	fclose(in);
+	s->flags &= ~CAMEL_STORE_SUMMARY_DIRTY;
+	return ret;
+}
+
+/**
+ * camel_store_summary_add:
+ * @s: 
+ * @info: 
+ * 
+ * Adds a new @info record to the summary.  If @info->uid is NULL, then a new
+ * uid is automatically re-assigned by calling :next_uid_string().
+ *
+ * The @info record should have been generated by calling one of the
+ * info_new_*() functions, as it will be free'd based on the summary
+ * class.  And MUST NOT be allocated directly using malloc.
+ **/
+void camel_store_summary_add(CamelStoreSummary *s, CamelFolderInfo *info)
+{
+	if (info == NULL)
+		return;
+
+	if (camel_folder_info_full(s, info) == NULL) {
+		g_warning("Trying to add a folder info with missing required full name\n");
+		return;
+	}
+
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	g_ptr_array_add(s->folders, info);
+	g_hash_table_insert(s->folders_full, (char *)camel_folder_info_full(s, info), info);
+	s->flags |= CAMEL_STORE_SUMMARY_DIRTY;
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+}
+
+/**
+ * camel_store_summary_add_from_full:
+ * @s: 
+ * @h: 
+ * 
+ * Build a new info record based on the name, and add it to the summary.
+ *
+ * Return value: The newly added record.
+ **/
+CamelFolderInfo *camel_store_summary_add_from_full(CamelStoreSummary *s, const char *full)
+{
+	CamelFolderInfo *info;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+
+	info = g_hash_table_lookup(s->folders_full, full);
+	if (info != NULL) {
+		g_warning("Trying to add folder '%s' to summary that already has it", full);
+		info = NULL;
+	} else {
+		info = camel_store_summary_info_new_from_full(s, full);
+		g_ptr_array_add(s->folders, info);
+		g_hash_table_insert(s->folders_full, (char *)camel_folder_info_full(s, info), info);
+		s->flags |= CAMEL_STORE_SUMMARY_DIRTY;
+	}
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+
+	return info;
+}
+
+/**
+ * camel_store_summary_info_new_from_full:
+ * @s: 
+ * @h: 
+ * 
+ * Create a new info record from a name.
+ * 
+ * Return value: Guess?  This info record MUST be freed using
+ * camel_store_summary_info_free(), camel_folder_info_free() will not work.
+ **/
+CamelFolderInfo *camel_store_summary_info_new_from_full(CamelStoreSummary *s, const char *f)
+{
+	return ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s))) -> folder_info_new(s, f);
+}
+
+/**
+ * camel_store_summary_info_free:
+ * @s: 
+ * @mi: 
+ * 
+ * Unref and potentially free the message info @mi, and all associated memory.
+ **/
+void camel_store_summary_info_free(CamelStoreSummary *s, CamelFolderInfo *mi)
+{
+	g_assert(mi);
+	g_assert(s);
+
+	CAMEL_STORE_SUMMARY_LOCK(s, ref_lock);
+
+	g_assert(mi->refcount >= 1);
+
+	mi->refcount--;
+	if (mi->refcount > 0) {
+		CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+		return;
+	}
+
+	CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+
+	((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->folder_info_free(s, mi);		
+}
+
+/**
+ * camel_store_summary_info_ref:
+ * @s: 
+ * @mi: 
+ * 
+ * Add an extra reference to @mi.
+ **/
+void camel_store_summary_info_ref(CamelStoreSummary *s, CamelFolderInfo *mi)
+{
+	g_assert(mi);
+	g_assert(s);
+
+	CAMEL_STORE_SUMMARY_LOCK(s, ref_lock);
+	g_assert(mi->refcount >= 1);
+	mi->refcount++;
+	CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+}
+
+/**
+ * camel_store_summary_touch:
+ * @s: 
+ * 
+ * Mark the summary as changed, so that a save will save it.
+ **/
+void
+camel_store_summary_touch(CamelStoreSummary *s)
+{
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+	s->flags |= CAMEL_STORE_SUMMARY_DIRTY;
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+}
+
+/**
+ * camel_store_summary_clear:
+ * @s: 
+ * 
+ * Empty the summary contents.
+ **/
+void
+camel_store_summary_clear(CamelStoreSummary *s)
+{
+	int i;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+	if (camel_store_summary_count(s) == 0) {
+		CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+		return;
+	}
+
+	for (i=0;i<s->folders->len;i++)
+		camel_store_summary_info_free(s, s->folders->pdata[i]);
+
+	g_ptr_array_set_size(s->folders, 0);
+	g_hash_table_destroy(s->folders_full);
+	s->folders_full = g_hash_table_new(g_str_hash, g_str_equal);
+	s->flags |= CAMEL_STORE_SUMMARY_DIRTY;
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+}
+
+/**
+ * camel_store_summary_remove:
+ * @s: 
+ * @info: 
+ * 
+ * Remove a specific @info record from the summary.
+ **/
+void camel_store_summary_remove(CamelStoreSummary *s, CamelFolderInfo *info)
+{
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+	g_hash_table_remove(s->folders_full, camel_folder_info_full(s, info));
+	g_ptr_array_remove(s->folders, info);
+	s->flags |= CAMEL_STORE_SUMMARY_DIRTY;
+	CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+
+	camel_store_summary_info_free(s, info);
+}
+
+/**
+ * camel_store_summary_remove_uid:
+ * @s: 
+ * @full: 
+ * 
+ * Remove a specific info record from the summary, by @full.
+ **/
+void camel_store_summary_remove_full(CamelStoreSummary *s, const char *full)
+{
+        CamelFolderInfo *oldinfo;
+        char *oldfull;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, ref_lock);
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+        if (g_hash_table_lookup_extended(s->folders_full, full, (void *)&oldfull, (void *)&oldinfo)) {
+		/* make sure it doesn't vanish while we're removing it */
+		oldinfo->refcount++;
+		CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+		CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+		camel_store_summary_remove(s, oldinfo);
+		camel_store_summary_info_free(s, oldinfo);
+        } else {
+		CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+		CAMEL_STORE_SUMMARY_UNLOCK(s, ref_lock);
+	}
+}
+
+/**
+ * camel_store_summary_remove_index:
+ * @s: 
+ * @index: 
+ * 
+ * Remove a specific info record from the summary, by index.
+ **/
+void camel_store_summary_remove_index(CamelStoreSummary *s, int index)
+{
+	CAMEL_STORE_SUMMARY_LOCK(s, summary_lock);
+	if (index < s->folders->len) {
+		CamelFolderInfo *info = s->folders->pdata[index];
+
+		g_hash_table_remove(s->folders_full, camel_folder_info_full(s, info));
+		g_ptr_array_remove_index(s->folders, index);
+		s->flags |= CAMEL_STORE_SUMMARY_DIRTY;
+
+		CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+		camel_store_summary_info_free(s, info);
+	} else {
+		CAMEL_STORE_SUMMARY_UNLOCK(s, summary_lock);
+	}
+}
+
+static int
+summary_header_load(CamelStoreSummary *s, FILE *in)
+{
+	gint32 version, flags, count;
+	time_t time;
+
+	fseek(in, 0, SEEK_SET);
+
+	io(printf("Loading header\n"));
+
+	if (camel_file_util_decode_fixed_int32(in, &version) == -1
+	    || camel_file_util_decode_fixed_int32(in, &flags) == -1
+	    || camel_file_util_decode_time_t(in, &time) == -1
+	    || camel_file_util_decode_fixed_int32(in, &count) == -1) {
+		return -1;
+	}
+
+	s->flags = flags;
+	s->time = time;
+	s->count = count;
+	if (s->version != version) {
+		g_warning("Summary header version mismatch");
+		return -1;
+	}
+	return 0;
+}
+
+static int
+summary_header_save(CamelStoreSummary *s, FILE *out)
+{
+	fseek(out, 0, SEEK_SET);
+
+	io(printf("Savining header\n"));
+
+	camel_file_util_encode_fixed_int32(out, s->version);
+	camel_file_util_encode_fixed_int32(out, s->flags);
+	camel_file_util_encode_time_t(out, s->time);
+	return camel_file_util_encode_fixed_int32(out, camel_store_summary_count(s));
+}
+
+/**
+ * camel_store_summary_info_new:
+ * @s: 
+ * 
+ * Allocate a new camel message info, suitable for adding
+ * to this summary.
+ * 
+ * Return value: 
+ **/
+CamelFolderInfo *
+camel_store_summary_info_new(CamelStoreSummary *s)
+{
+	CamelFolderInfo *mi;
+
+	CAMEL_STORE_SUMMARY_LOCK(s, alloc_lock);
+	if (s->folder_info_chunks == NULL)
+		s->folder_info_chunks = e_memchunk_new(32, s->folder_info_size);
+	mi = e_memchunk_alloc0(s->folder_info_chunks);
+	CAMEL_STORE_SUMMARY_UNLOCK(s, alloc_lock);
+	mi->refcount = 1;
+	return mi;
+}
+
+const char *camel_folder_info_string(CamelStoreSummary *s, const CamelFolderInfo *mi, int type)
+{
+	return ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->folder_info_string(s, mi, type);
+}
+
+void camel_folder_info_set_string(CamelStoreSummary *s, CamelFolderInfo *mi, int type, const char *value)
+{
+	return ((CamelStoreSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->folder_info_set_string(s, mi, type, value);
+}
+
+static CamelFolderInfo *
+folder_info_new(CamelStoreSummary *s, const char *f)
+{
+	CamelFolderInfo *mi;
+
+	mi = camel_store_summary_info_new(s);
+
+	mi->full = g_strdup(f);
+	mi->unread = CAMEL_STORE_SUMMARY_UNKNOWN;
+	mi->total = CAMEL_STORE_SUMMARY_UNKNOWN;
+
+	return mi;
+}
+
+static CamelFolderInfo *
+folder_info_load(CamelStoreSummary *s, FILE *in)
+{
+	CamelFolderInfo *mi;
+
+	mi = camel_store_summary_info_new(s);
+
+	io(printf("Loading folder info\n"));
+
+	camel_file_util_decode_string(in, &mi->full);
+	camel_file_util_decode_uint32(in, &mi->flags);
+	camel_file_util_decode_uint32(in, &mi->unread);
+	camel_file_util_decode_uint32(in, &mi->total);
+
+	if (!ferror(in))
+		return mi;
+
+	camel_store_summary_info_free(s, mi);
+
+	return NULL;
+}
+
+static int
+folder_info_save(CamelStoreSummary *s, FILE *out, CamelFolderInfo *mi)
+{
+	io(printf("Saving folder info\n"));
+
+	camel_file_util_encode_string(out, camel_folder_info_full(s, mi));
+	camel_file_util_encode_uint32(out, mi->flags);
+	camel_file_util_encode_uint32(out, mi->unread);
+	camel_file_util_encode_uint32(out, mi->total);
+
+	return ferror(out);
+}
+
+static void
+folder_info_free(CamelStoreSummary *s, CamelFolderInfo *mi)
+{
+	g_free(mi->full);
+	g_free(mi->uri);
+	e_memchunk_free(s->folder_info_chunks, mi);
+}
+
+static const char *
+folder_info_string(CamelStoreSummary *s, const CamelFolderInfo *mi, int type)
+{
+	const char *p;
+
+	/* FIXME: Locks? */
+
+	g_assert (mi != NULL);
+
+	switch (type) {
+	case CAMEL_STORE_SUMMARY_FULL:
+		return mi->full;
+	case CAMEL_STORE_SUMMARY_NAME:
+		p = strrchr(mi->full, '/');
+		if (p)
+			return p;
+		else
+			return mi->full;
+	case CAMEL_STORE_SUMMARY_URI:
+		if (mi->uri)
+			return mi->uri;
+		if (s->uri_prefix)
+			return (((CamelFolderInfo *)mi)->uri = g_strdup_printf("%s%s", s->uri_prefix, mi->full));
+	}
+
+	return "";
+}
+
+static void
+folder_info_set_string (CamelStoreSummary *s, CamelFolderInfo *mi, int type, const char *str)
+{
+	const char *p;
+	char *v;
+	int len;
+
+	g_assert (mi != NULL);
+
+	switch(type) {
+	case CAMEL_STORE_SUMMARY_FULL:
+		g_free(mi->full);
+		g_free(mi->uri);
+		mi->full = g_strdup(str);
+		break;
+	case CAMEL_STORE_SUMMARY_NAME:
+		p = strrchr(mi->full, '/');
+		if (p) {
+			len = p-mi->full+1;
+			v = g_malloc(len+strlen(str)+1);
+			memcpy(v, mi->full, len);
+			strcpy(v+len, str);
+		} else {
+			v = g_strdup(str);
+		}
+		g_free(mi->full);
+		mi->full = v;
+		break;
+	case CAMEL_STORE_SUMMARY_URI:
+		if (s->uri_prefix) {
+			len = strlen(s->uri_prefix);
+			if (len > strlen(str)
+			    || strncmp(s->uri_prefix, str, len) != 0) {
+				g_warning("Trying to set folderinfo uri '%s' for summary with prefix '%s'",
+					  str, s->uri_prefix);
+				return;
+			}
+			g_free(mi->full);
+			g_free(mi->uri);
+			mi->full = g_strdup(str + len);
+			mi->uri = g_strdup(str);
+		} else {
+			g_warning("Trying to set folderinfo uri '%s' for summary with no uri prefix", str);
+		}
+		break;
+	}
+}
diff --git a/camel/camel-store-summary.h b/camel/camel-store-summary.h
new file mode 100644
index 0000000000..ae0e49d339
--- /dev/null
+++ b/camel/camel-store-summary.h
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2000 Ximian Inc.
+ *
+ * Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _CAMEL_STORE_SUMMARY_H
+#define _CAMEL_STORE_SUMMARY_H
+
+#include <stdio.h>
+#include <time.h>
+#include <camel/camel-mime-parser.h>
+#include <camel/camel-object.h>
+#include <libibex/ibex.h>
+
+#define CAMEL_STORE_SUMMARY(obj)         CAMEL_CHECK_CAST (obj, camel_store_summary_get_type (), CamelStoreSummary)
+#define CAMEL_STORE_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_store_summary_get_type (), CamelStoreSummaryClass)
+#define CAMEL_IS_FOLDER_SUMMARY(obj)      CAMEL_CHECK_TYPE (obj, camel_store_summary_get_type ())
+
+typedef struct _CamelStoreSummary      CamelStoreSummary;
+typedef struct _CamelStoreSummaryClass CamelStoreSummaryClass;
+
+typedef struct _CamelFolderInfo CamelFolderInfo;
+
+enum _CamelFolderFlags {
+	CAMEL_STORE_SUMMARY_FOLDER_NOSELECT,
+	CAMEL_STORE_SUMMARY_FOLDER_READONLY,
+	CAMEL_STORE_SUMMARY_FOLDER_SUBSCRIBED,
+	CAMEL_STORE_SUMMARY_FOLDER_FLAGGED,
+};
+
+#define CAMEL_STORE_SUMMARY_UNKNOWN (~0)
+
+enum {
+	CAMEL_STORE_SUMMARY_FULL = 0,
+	CAMEL_STORE_SUMMARY_NAME,
+	CAMEL_STORE_SUMMARY_URI,
+	CAMEL_STORE_SUMMARY_LAST,
+};
+
+struct _CamelFolderInfo {
+	guint32 refcount;
+	char *uri;
+	char *full;
+	guint32 flags;
+	guint32 unread;
+	guint32 total;
+};
+
+enum _CamelStoreSummaryFlags {
+	CAMEL_STORE_SUMMARY_DIRTY = 1<<0,
+};
+
+struct _CamelStoreSummary {
+	CamelObject parent;
+
+	struct _CamelStoreSummaryPrivate *priv;
+
+	/* header info */
+	guint32 version;	/* version of file required, should be set by implementors */
+	guint32 flags;		/* flags */
+	guint32 count;		/* how many were saved/loaded */
+	time_t time;		/* timestamp for this summary (for implementors to use) */
+
+	/* sizes of memory objects */
+	guint32 folder_info_size;
+
+	/* memory allocators (setup automatically) */
+	struct _EMemChunk *folder_info_chunks;
+
+	char *summary_path;
+	char *uri_prefix;
+
+	GPtrArray *folders;	/* CamelFolderInfo's */
+	GHashTable *folders_full; /* CamelFolderInfo's by full name */
+};
+
+struct _CamelStoreSummaryClass {
+	CamelObjectClass parent_class;
+
+	/* load/save the global info */
+	int (*summary_header_load)(CamelStoreSummary *, FILE *);
+	int (*summary_header_save)(CamelStoreSummary *, FILE *);
+
+	/* create/save/load an individual message info */
+	CamelFolderInfo * (*folder_info_new)(CamelStoreSummary *, const char *full);
+	CamelFolderInfo * (*folder_info_load)(CamelStoreSummary *, FILE *);
+	int		  (*folder_info_save)(CamelStoreSummary *, FILE *, CamelFolderInfo *);
+	void		  (*folder_info_free)(CamelStoreSummary *, CamelFolderInfo *);
+
+	/* virtualise access methods */
+	const char *(*folder_info_string)(CamelStoreSummary *, const CamelFolderInfo *, int);
+	void (*folder_info_set_string)(CamelStoreSummary *, CamelFolderInfo *, int, const char *);
+};
+
+guint			 camel_store_summary_get_type	(void);
+CamelStoreSummary      *camel_store_summary_new	(void);
+
+void camel_store_summary_set_filename(CamelStoreSummary *, const char *);
+void camel_store_summary_set_uri_prefix(CamelStoreSummary *, const char *);
+
+/* load/save the summary in its entirety */
+int camel_store_summary_load(CamelStoreSummary *);
+int camel_store_summary_save(CamelStoreSummary *);
+
+/* only load the header */
+int camel_store_summary_header_load(CamelStoreSummary *);
+
+/* set the dirty bit on the summary */
+void camel_store_summary_touch(CamelStoreSummary *s);
+
+/* add a new raw summary item */
+void camel_store_summary_add(CamelStoreSummary *, CamelFolderInfo *info);
+
+/* build/add raw summary items */
+CamelFolderInfo *camel_store_summary_add_from_full(CamelStoreSummary *, const char *);
+
+/* Just build raw summary items */
+CamelFolderInfo *camel_store_summary_info_new(CamelStoreSummary *s);
+CamelFolderInfo *camel_store_summary_info_new_from_full(CamelStoreSummary *s, const char *);
+
+void camel_store_summary_info_ref(CamelStoreSummary *, CamelFolderInfo *);
+void camel_store_summary_info_free(CamelStoreSummary *, CamelFolderInfo *);
+
+/* removes a summary item */
+void camel_store_summary_remove(CamelStoreSummary *s, CamelFolderInfo *info);
+void camel_store_summary_remove_full(CamelStoreSummary *s, const char *full);
+void camel_store_summary_remove_index(CamelStoreSummary *s, int);
+
+/* remove all items */
+void camel_store_summary_clear(CamelStoreSummary *s);
+
+/* lookup functions */
+int camel_store_summary_count(CamelStoreSummary *);
+CamelFolderInfo *camel_store_summary_index(CamelStoreSummary *, int);
+CamelFolderInfo *camel_store_summary_full(CamelStoreSummary *, const char *uid);
+GPtrArray *camel_store_summary_array(CamelStoreSummary *s);
+void camel_store_summary_array_free(CamelStoreSummary *s, GPtrArray *array);
+
+const char *camel_folder_info_string(CamelStoreSummary *, const CamelFolderInfo *, int type);
+void camel_folder_info_set_string(CamelStoreSummary *, CamelFolderInfo *, int type, const char *value);
+
+/* helper macro's */
+#define camel_folder_info_full(s, i) (camel_folder_info_string((CamelStoreSummary *)s, (const CamelFolderInfo *)i, CAMEL_STORE_SUMMARY_FULL))
+#define camel_folder_info_uri(s, i) (camel_folder_info_string((CamelStoreSummary *)s, (const CamelFolderInfo *)i, CAMEL_STORE_SUMMARY_URI))
+#define camel_folder_info_name(s, i) (camel_folder_info_string((CamelStoreSummary *)s, (const CamelFolderInfo *)i, CAMEL_STORE_SUMMARY_NAME))
+
+#endif /* ! _CAMEL_STORE_SUMMARY_H */
diff --git a/camel/providers/imap/camel-imap-folder.c b/camel/providers/imap/camel-imap-folder.c
index 6939587f10..714e245643 100644
--- a/camel/providers/imap/camel-imap-folder.c
+++ b/camel/providers/imap/camel-imap-folder.c
@@ -226,6 +226,8 @@ camel_imap_folder_new (CamelStore *parent, const char *folder_name,
 	    !g_strcasecmp (folder_name, "INBOX"))
 		folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 
+	imap_folder->search = camel_imap_search_new(folder_dir);
+
 	return folder;
 }
 
@@ -1314,9 +1316,6 @@ imap_search_by_expression (CamelFolder *folder, const char *expression, CamelExc
 	   command channel too */
 	CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
 
-	if (!imap_folder->search)
-		imap_folder->search = camel_imap_search_new ();
-
 	camel_folder_search_set_folder (imap_folder->search, folder);
 	summary = camel_folder_get_summary(folder);
 	camel_folder_search_set_summary(imap_folder->search, summary);
@@ -1353,9 +1352,6 @@ imap_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uids
 
 	CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
 
-	if (imap_folder->search == NULL)
-		imap_folder->search = camel_imap_search_new();
-
 	camel_folder_search_set_folder(imap_folder->search, folder);
 	camel_folder_search_set_summary(imap_folder->search, summary);
 
diff --git a/camel/providers/imap/camel-imap-search.c b/camel/providers/imap/camel-imap-search.c
index 2adaaef244..b184843f27 100644
--- a/camel/providers/imap/camel-imap-search.c
+++ b/camel/providers/imap/camel-imap-search.c
@@ -4,8 +4,9 @@
 /*
  *  Authors:
  *    Dan Winship <danw@ximian.com>
+ *    Michael Zucchi <notzed@ximian.com>
  *
- *  Copyright 2000, 2001 Ximian, Inc.
+ *  Copyright 2000, 2001, 2002 Ximian, Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of version 2 of the GNU General Public
@@ -35,10 +36,58 @@
 #include "camel-imap-search.h"
 #include "camel-imap-private.h"
 #include "camel-imap-utils.h"
+#include "camel-imap-summary.h"
 
-static ESExpResult *
-imap_body_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv,
-		    CamelFolderSearch *s);
+#include "e-util/md5-utils.h"	/* md5 hash building */
+#include "camel-mime-utils.h"	/* base64 encoding */
+
+#include "camel-seekable-stream.h"
+
+#define d(x) x
+
+/*
+  File is:
+   BODY	 (as in body search)
+   Last uid when search performed
+   termcount: number of search terms
+   matchcount: number of matches
+   term0, term1 ...
+   match0, match1, match2, ...
+*/
+
+/* size of in-memory cache */
+#define MATCH_CACHE_SIZE (32)
+
+/* Also takes care of 'endianness' file magic */
+#define MATCH_MARK (('B' << 24) | ('O' << 16) | ('D' << 8) | 'Y')
+
+/* on-disk header, in native endianness format, matches follow */
+struct _match_header {
+	guint32 mark;
+	guint32 validity;	/* uidvalidity for this folder */
+	guint32 lastuid;
+	guint32 termcount;
+	guint32 matchcount;
+};
+
+/* in-memory record */
+struct _match_record {
+	struct _match_record *next;
+	struct _match_record *prev;
+
+	char hash[17];
+
+	guint32 lastuid;
+	guint32 validity;
+
+	unsigned int termcount;
+	char **terms;
+	GArray *matches;
+};
+
+
+static void free_match(CamelImapSearch *is, struct _match_record *mr);
+static ESExpResult *imap_body_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *s);
 
 static CamelFolderSearchClass *imap_search_parent_class;
 
@@ -49,12 +98,33 @@ camel_imap_search_class_init (CamelImapSearchClass *camel_imap_search_class)
 	CamelFolderSearchClass *camel_folder_search_class =
 		CAMEL_FOLDER_SEARCH_CLASS (camel_imap_search_class);
 
-	imap_search_parent_class = camel_type_get_global_classfuncs (camel_folder_search_get_type ());
+	imap_search_parent_class = (CamelFolderSearchClass *)camel_type_get_global_classfuncs (camel_folder_search_get_type ());
 	
 	/* virtual method overload */
 	camel_folder_search_class->body_contains = imap_body_contains;
 }
 
+static void
+camel_imap_search_init(CamelImapSearch *is)
+{
+	e_dlist_init(&is->matches);
+	is->matches_hash = g_hash_table_new(g_str_hash, g_str_equal);
+	is->matches_count = 0;
+	is->lastuid = 0;
+}
+
+static void
+camel_imap_search_finalise(CamelImapSearch *is)
+{
+	struct _match_record *mr;
+
+	while ( (mr = (struct _match_record *)e_dlist_remtail(&is->matches)) )
+		free_match(is, mr);
+	g_hash_table_destroy(is->matches_hash);
+	if (is->cache)
+		camel_object_unref((CamelObject *)is->cache);
+}
+
 CamelType
 camel_imap_search_get_type (void)
 {
@@ -65,138 +135,334 @@ camel_imap_search_get_type (void)
 			CAMEL_FOLDER_SEARCH_TYPE, "CamelImapSearch",
 			sizeof (CamelImapSearch),
 			sizeof (CamelImapSearchClass),
-			(CamelObjectClassInitFunc) camel_imap_search_class_init,
-			NULL, NULL, NULL);
+			(CamelObjectClassInitFunc) camel_imap_search_class_init, NULL,
+			(CamelObjectInitFunc) camel_imap_search_init,
+			(CamelObjectFinalizeFunc) camel_imap_search_finalise);
 	}
 
 	return camel_imap_search_type;
 }
 
+/**
+ * camel_imap_search_new:
+ *
+ * Return value: A new CamelImapSearch widget.
+ **/
+CamelFolderSearch *
+camel_imap_search_new (const char *cachedir)
+{
+	CamelFolderSearch *new = CAMEL_FOLDER_SEARCH (camel_object_new (camel_imap_search_get_type ()));
+	CamelImapSearch *is = (CamelImapSearch *)new;
+
+	camel_folder_search_construct (new);
+
+	is->cache = camel_data_cache_new(cachedir, 0, NULL);
+	if (is->cache) {
+		/* Expire entries after 14 days of inactivity */
+		camel_data_cache_set_expire_access(is->cache, 60*60*24*14);
+	}
+
+	return new;
+}
+
+
+static void
+hash_match(char hash[17], int argc, struct _ESExpResult **argv)
+{
+	MD5Context ctx;
+	unsigned char digest[16];
+	unsigned int state = 0, save = 0;
+	int i;
+
+	md5_init(&ctx);
+	for (i=0;i<argc;i++) {
+		if (argv[i]->type == ESEXP_RES_STRING)
+			md5_update(&ctx, argv[i]->value.string, strlen(argv[i]->value.string));
+	}
+	md5_final(&ctx, digest);
+
+	base64_encode_close(digest, 12, FALSE, hash, &state, &save);
+
+	for (i=0;i<16;i++) {
+		if (hash[i] == '+')
+			hash[i] = ',';
+		if (hash[i] == '/')
+			hash[i] = '_';
+	}
+
+	hash[16] = 0;
+}
+
 static int
-cmp_uid(const void *ap, const void *bp)
+save_match(CamelImapSearch *is, struct _match_record *mr)
 {
-	unsigned int a, b;
+	guint32 mark = MATCH_MARK;
+	int ret = 0;
+	struct _match_header header;
+	CamelStream *stream;
 
-	a = strtoul(((char **)ap)[0], NULL, 10);
-	b = strtoul(((char **)bp)[0], NULL, 10);
-	if (a<b)
+	/* since its a cache, doesn't matter if it doesn't save, at least we have the in-memory cache
+	   for this session */
+	if (is->cache == NULL)
 		return -1;
-	else if (a>b)
-		return 1;
+
+	stream = camel_data_cache_add(is->cache, "search/body-contains", mr->hash, NULL);
+	if (stream == NULL)
+		return -1;
+
+	d(printf("Saving search cache entry to '%s': %s\n", mr->hash, mr->terms[0]));
+	
+	/* we write the whole thing, then re-write the header magic, saves fancy sync code */
+	memcpy(&header.mark, "    ", 4);
+	header.termcount = 0;
+	header.matchcount = mr->matches->len;
+	header.lastuid = mr->lastuid;
+	header.validity = mr->validity;
+
+	if (camel_stream_write(stream, (char *)&header, sizeof(header)) != sizeof(header)
+	    || camel_stream_write(stream, mr->matches->data, mr->matches->len*sizeof(guint32)) != mr->matches->len*sizeof(guint32)
+	    || camel_seekable_stream_seek((CamelSeekableStream *)stream, 0, CAMEL_STREAM_SET) == -1
+	    || camel_stream_write(stream, (char *)&mark, sizeof(mark)) != sizeof(mark)) {
+		d(printf(" saving failed, removing cache entry\n"));
+		camel_data_cache_remove(is->cache, "search/body-contains", mr->hash, NULL);
+		ret = -1;
+	}
+
+	camel_object_unref((CamelObject *)stream);
+	return ret;
+}
+
+static void
+free_match(CamelImapSearch *is, struct _match_record *mr)
+{
+	int i;
+
+	for (i=0;i<mr->termcount;i++)
+		g_free(mr->terms[i]);
+	g_free(mr->terms);
+	g_array_free(mr->matches, TRUE);
+	g_free(mr);
+}
+
+static struct _match_record *
+load_match(CamelImapSearch *is, char hash[17], int argc, struct _ESExpResult **argv)
+{
+	struct _match_record *mr;
+	CamelStream *stream = NULL;
+	struct _match_header header;
+	int i;
+
+	mr = g_malloc0(sizeof(*mr));
+	mr->matches = g_array_new(0, 0, sizeof(guint32));
+	g_assert(strlen(hash) == 16);
+	strcpy(mr->hash, hash);
+	mr->terms = g_malloc0(sizeof(mr->terms[0]) * argc);
+	for (i=0;i<argc;i++) {
+		if (argv[i]->type == ESEXP_RES_STRING) {
+			mr->termcount++;
+			mr->terms[i] = g_strdup(argv[i]->value.string);
+		}
+	}
+
+	d(printf("Loading search cache entry to '%s': %s\n", mr->hash, mr->terms[0]));
+
+	memset(&header, 0, sizeof(header));
+	if (is->cache)
+		stream = camel_data_cache_get(is->cache, "search/body-contains", mr->hash, NULL);
+	if (stream != NULL) {
+		/* 'cause i'm gonna be lazy, i'm going to have the termcount == 0 for now,
+		   and not load or save them since i can't think of a nice way to do it, the hash
+		   should be sufficient to key it */
+		/* This check should also handle endianness changes, we just throw away
+		   the data (its only a cache) */
+		if (camel_stream_read(stream, (char *)&header, sizeof(header)) == sizeof(header)
+		    && header.validity == is->validity
+		    && header.mark == MATCH_MARK
+		    && header.termcount == 0) {
+			d(printf(" found %d matches\n", header.matchcount));
+			g_array_set_size(mr->matches, header.matchcount);
+			camel_stream_read(stream, mr->matches->data, sizeof(guint32)*header.matchcount);
+		} else {
+			d(printf(" file format invalid/validity changed\n"));
+			memset(&header, 0, sizeof(header));
+		}
+		camel_object_unref((CamelObject *)stream);
+	} else {
+		d(printf(" no cache entry found\n"));
+	}
+
+	mr->validity = header.validity;
+	if (mr->validity != is->validity)
+		mr->lastuid = 0;
+	else
+		mr->lastuid = header.lastuid;
+
+	return mr;
+}
+
+static int
+sync_match(CamelImapSearch *is, struct _match_record *mr)
+{
+	char *p, *result, *lasts = NULL;
+	CamelImapResponse *response;
+	guint32 uid;
+	CamelFolder *folder = ((CamelFolderSearch *)is)->folder;
+	CamelImapStore *store = (CamelImapStore *)folder->parent_store;
+
+	if (mr->lastuid >= is->lastuid && mr->validity == is->validity)
+		return 0;
+
+	d(printf("updating match record for uid's %d:%d\n", mr->lastuid+1, is->lastuid));
+
+	/* TODO: Handle multiple search terms */
+
+	response = camel_imap_command (store, folder, NULL,
+				       "UID SEARCH UID %d:%d BODY \"%s\"",
+				       mr->lastuid+1, is->lastuid, mr->terms[0]);
+	if (!response)
+		return -1;
+	result = camel_imap_response_extract (store, response, "SEARCH", NULL);
+	if (!result)
+		return -1;
+	
+	p = result + sizeof ("* SEARCH");
+	for (p = strtok_r (p, " ", &lasts); p; p = strtok_r (NULL, " ", &lasts)) {
+		uid = strtoul(p, NULL, 10);
+		g_array_append_vals(mr->matches, &uid, 1);
+	}
+	g_free(result);
+
+	mr->validity = is->validity;
+	mr->lastuid = is->lastuid;
+	save_match(is, mr);
 
 	return 0;
 }
 
+static struct _match_record *
+get_match(CamelImapSearch *is, int argc, struct _ESExpResult **argv)
+{
+	char hash[17];
+	struct _match_record *mr;
+
+	hash_match(hash, argc, argv);
+
+	mr = g_hash_table_lookup(is->matches_hash, hash);
+	if (mr == NULL) {
+		while (is->matches_count >= MATCH_CACHE_SIZE) {
+			mr = (struct _match_record *)e_dlist_remtail(&is->matches);
+			if (mr) {
+				printf("expiring match '%s' (%s)\n", mr->hash, mr->terms[0]);
+				g_hash_table_remove(is->matches_hash, mr->hash);
+				free_match(is, mr);
+				is->matches_count--;
+			} else {
+				is->matches_count = 0;
+			}
+		}
+		mr = load_match(is, hash, argc, argv);
+		g_hash_table_insert(is->matches_hash, mr->hash, mr);
+		is->matches_count++;
+	} else {
+		e_dlist_remove((EDListNode *)mr);
+	}
+
+	e_dlist_addhead(&is->matches, (EDListNode *)mr);
+
+	/* what about offline mode? */
+	/* We could cache those results too, or should we cache them elsewhere? */
+	sync_match(is, mr);
+
+	return mr;
+}
+
 static ESExpResult *
-imap_body_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv,
-		    CamelFolderSearch *s)
+imap_body_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *s)
 {
 	CamelImapStore *store = CAMEL_IMAP_STORE (s->folder->parent_store);
-	char *value = argv[0]->value.string;
-	CamelImapResponse *response = NULL;
-	char *result, *p, *lasts = NULL, *real_uid;
-	const char *uid = "";
+	CamelImapSearch *is = (CamelImapSearch *)s;
+	char *uid;
 	ESExpResult *r;
-	CamelMessageInfo *info;
+	CamelMessageInfo *info;	
 	GHashTable *uid_hash = NULL;
-	char *set;
-	GPtrArray *sorted;
-	int i;
+	GPtrArray *array;
+	int i, j;
+	struct _match_record *mr;
+	guint32 uidn, *uidp;
+
+	d(printf("Performing body search '%s'\n", argv[0]->value.string));
+
+	/* TODO: Cache offline searches too? */
 
 	/* If offline, search using the parent class, which can handle this manually */
 	if (!camel_disco_store_check_online (CAMEL_DISCO_STORE (store), NULL))
 		return imap_search_parent_class->body_contains(f, argc, argv, s);
 
-	if (s->current) {
-		uid = camel_message_info_uid (s->current);
-		r = e_sexp_result_new (f, ESEXP_RES_BOOL);
-		r->value.bool = FALSE;
-		response = camel_imap_command (store, s->folder, NULL,
-					       "UID SEARCH UID %s BODY \"%s\"",
-					       uid, value);
-	} else {
-		r = e_sexp_result_new (f, ESEXP_RES_ARRAY_PTR);
-		
-		if (argc == 1 && *value == '\0' && s->folder) {
-			/* optimise the match "" case - match everything */
+	/* optimise the match "" case - match everything */
+	if (argc == 1 && argv[0]->value.string[0] == '\0') {
+		if (s->current) {
+			r = e_sexp_result_new(f, ESEXP_RES_BOOL);
+			r->value.bool = TRUE;
+		} else {
+			r = e_sexp_result_new(f, ESEXP_RES_ARRAY_PTR);
 			r->value.ptrarray = g_ptr_array_new ();
 			for (i = 0; i < s->summary->len; i++) {
-				CamelMessageInfo *info = g_ptr_array_index (s->summary, i);
-				g_ptr_array_add (r->value.ptrarray, (char *)camel_message_info_uid (info));
+				info = g_ptr_array_index(s->summary, i);
+				g_ptr_array_add(r->value.ptrarray, (char *)camel_message_info_uid(info));
 			}
+		}
+	} else if (s->summary->len == 0) {
+		/* nothing to match case, do nothing (should be handled higher up?) */
+		if (s->current) {
+			r = e_sexp_result_new(f, ESEXP_RES_BOOL);
+			r->value.bool = FALSE;
 		} else {
-			/* If searching a (reasonably small) subset of
-                           the real folder size, then use a
-                           message-set to optimise it */
-			/* TODO: This peeks a bunch of 'private'ish data */
-			if (s->summary->len < camel_folder_get_message_count(s->folder)/2) {
-				sorted = g_ptr_array_new();
-				g_ptr_array_set_size(sorted, s->summary->len);
-				for (i=0;i<s->summary->len;i++)
-					sorted->pdata[i] = (void *)camel_message_info_uid((CamelMessageInfo *)s->summary->pdata[i]);
-				qsort(sorted->pdata, sorted->len, sizeof(sorted->pdata[0]), cmp_uid);
-				set = imap_uid_array_to_set(s->folder->summary, sorted);
-				response = camel_imap_command (store, s->folder, NULL,
-							       "UID SEARCH UID %s BODY \"%s\"",
-							       set, value);
-				g_free(set);
-				g_ptr_array_free(sorted, TRUE);
-			} else {
-				response = camel_imap_command (store, s->folder, NULL,
-							       "UID SEARCH BODY \"%s\"",
-							       value);
-			}
-
+			r = e_sexp_result_new(f, ESEXP_RES_ARRAY_PTR);
 			r->value.ptrarray = g_ptr_array_new ();
 		}
-	}
-	
-	if (!response)
-		return r;
-	result = camel_imap_response_extract (store, response, "SEARCH", NULL);
-	if (!result)
-		return r;
-	
-	p = result + sizeof ("* SEARCH");
-	for (p = strtok_r (p, " ", &lasts); p; p = strtok_r (NULL, " ", &lasts)) {
+	} else {
+		int truth = FALSE;
+
+		/* setup lastuid/validity for synchronising */
+		info = g_ptr_array_index(s->summary, s->summary->len-1);
+		is->lastuid = strtoul(camel_message_info_uid(info), NULL, 10);
+		is->validity = ((CamelImapSummary *)(s->folder->summary))->validity;
+
+		mr = get_match(is, argc, argv);
+
 		if (s->current) {
-			if (!strcmp (uid, p)) {
-				r->value.bool = TRUE;
-				break;
-			}
+			uidn = strtoul(camel_message_info_uid(s->current), NULL, 10);
+			uidp = (guint32 *)mr->matches->data;
+			j = mr->matches->len;
+			for (i=0;i<j && !truth;i++)
+				truth = *uidp++ == uidn;
+			r = e_sexp_result_new(f, ESEXP_RES_BOOL);
+			r->value.bool = truth;
 		} else {
-			/* if we need to setup a hash of summary items, this way we get
-			   access to the summary memory which is locked for the duration of
-			   the search, and wont vanish on us */
-			if (uid_hash == NULL) {
-				uid_hash = g_hash_table_new (g_str_hash, g_str_equal);
-				for (i = 0; i < s->summary->len; i++) {
-					info = s->summary->pdata[i];
-					g_hash_table_insert (uid_hash, (char *)camel_message_info_uid (info), info);
-				}
+			r = e_sexp_result_new(f, ESEXP_RES_ARRAY_PTR);
+			array = r->value.ptrarray = g_ptr_array_new();
+
+			/* We use a hash to map the uid numbers to uid strings as required by the search api */
+			/* We use the summary's strings so we dont need to alloc more */
+			uid_hash = g_hash_table_new(NULL, NULL);
+			for (i = 0; i < s->summary->len; i++) {
+				info = s->summary->pdata[i];
+				uid = (char *)camel_message_info_uid(info);
+				uidn = strtoul(uid, NULL, 10);
+				g_hash_table_insert(uid_hash, (void *)uidn, uid);
 			}
-			if (g_hash_table_lookup_extended (uid_hash, p, (void *)&real_uid, (void *)&info))
-				g_ptr_array_add (r->value.ptrarray, real_uid);
+
+			uidp = (guint32 *)mr->matches->data;
+			j = mr->matches->len;
+			for (i=0;i<j && !truth;i++) {
+				uid = g_hash_table_lookup(uid_hash, (void *)*uidp++);
+				if (uid)
+					g_ptr_array_add(array, uid);
+			}
+
+			g_hash_table_destroy(uid_hash);
 		}
 	}
-	
-	/* we could probably cache this globally, but its probably not worth it */
-	if (uid_hash)
-		g_hash_table_destroy (uid_hash);
-	
-	return r;
-}
 
-/**
- * camel_imap_search_new:
- *
- * Return value: A new CamelImapSearch widget.
- **/
-CamelFolderSearch *
-camel_imap_search_new (void)
-{
-	CamelFolderSearch *new = CAMEL_FOLDER_SEARCH (camel_object_new (camel_imap_search_get_type ()));
-
-	camel_folder_search_construct (new);
-	return new;
+	return r;
 }
diff --git a/camel/providers/imap/camel-imap-search.h b/camel/providers/imap/camel-imap-search.h
index 44335f0e82..9d1694c2c9 100644
--- a/camel/providers/imap/camel-imap-search.h
+++ b/camel/providers/imap/camel-imap-search.h
@@ -27,6 +27,8 @@
 #define _CAMEL_IMAP_SEARCH_H
 
 #include <camel/camel-folder-search.h>
+#include <e-util/e-msgport.h>
+#include <camel/camel-data-cache.h>
 
 #define CAMEL_IMAP_SEARCH_TYPE         (camel_imap_search_get_type ())
 #define CAMEL_IMAP_SEARCH(obj)         CAMEL_CHECK_CAST (obj, camel_imap_search_get_type (), CamelImapSearch)
@@ -38,6 +40,15 @@ typedef struct _CamelImapSearchClass CamelImapSearchClass;
 struct _CamelImapSearch {
 	CamelFolderSearch parent;
 
+	guint32 lastuid;	/* current 'last uid' for the folder */
+	guint32 validity;	/* validity of the current folder */
+
+	CamelDataCache *cache;	/* disk-cache for searches */
+
+	/* cache of body search matches */
+	unsigned int matches_count;
+	EDList matches;
+	GHashTable *matches_hash;
 };
 
 struct _CamelImapSearchClass {
@@ -46,6 +57,6 @@ struct _CamelImapSearchClass {
 };
 
 guint              camel_imap_search_get_type (void);
-CamelFolderSearch *camel_imap_search_new      (void);
+CamelFolderSearch *camel_imap_search_new      (const char *cachedir);
 
 #endif /* ! _CAMEL_IMAP_SEARCH_H */