changed to match_threads. (camel_folder_search_search): remove thread

2004-05-12  Not Zed  <NotZed@Ximian.com>

        * camel-folder-search.c (search_threads): changed to match_threads.
        (camel_folder_search_search): remove thread matching stuff from here.

2004-05-06  Not Zed  <NotZed@Ximian.com>

        * camel-digest-folder.c (digest_search_by_expression)
        (digest_search_by_uids):
        * providers/nntp/camel-nntp-folder.c (nntp_folder_search_by_expression)
        (nntp_folder_search_by_uids):
        * providers/imap/camel-imap-folder.c (imap_search_by_expression)
        (imap_search_by_uids):
        * providers/local/camel-local-folder.c (local_search_by_expression)
        (local_search_by_uids): use camel_folder_search_search & some minor cleanups.

        * camel-folder-search.c (search_threads): keep track of the match
        threads option for this search.
        (camel_folder_search_match_expression): Removed, not used anymore.
        (camel_folder_search_search): new api entry point for searching, a
        bit easier to use and needed for thread matching.

        * camel-folder-search.c (camel_folder_search_search): new search
        api entry point, take a full summary and optionally a subset of
        uids to match against.
        (search_match_all): use the uids' passed in to only search a
        subset of uid's.

svn path=/trunk/; revision=25876

7 files changed, 325 insertions, 162 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index 92740bfecd..175d6f392d 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -1,3 +1,8 @@
+2004-05-12  Not Zed  <NotZed@Ximian.com>
+
+	* camel-folder-search.c (search_threads): changed to match_threads.
+	(camel_folder_search_search): remove thread matching stuff from here.
+
 2004-05-11  Jeffrey Stedfast  <fejj@novell.com>
 
 	* camel-smime-context.c (sm_signing_cmsmessage): Fixed a
@@ -22,6 +27,27 @@
 
 2004-05-06  Not Zed  <NotZed@Ximian.com>
 
+	* camel-digest-folder.c (digest_search_by_expression) 
+	(digest_search_by_uids): 
+	* providers/nntp/camel-nntp-folder.c (nntp_folder_search_by_expression)
+	(nntp_folder_search_by_uids): 
+	* providers/imap/camel-imap-folder.c (imap_search_by_expression) 
+	(imap_search_by_uids):
+	* providers/local/camel-local-folder.c (local_search_by_expression) 
+	(local_search_by_uids): use camel_folder_search_search & some minor cleanups.
+
+	* camel-folder-search.c (search_threads): keep track of the match
+	threads option for this search.
+	(camel_folder_search_match_expression): Removed, not used anymore.
+	(camel_folder_search_search): new api entry point for searching, a
+	bit easier to use and needed for thread matching.
+
+	* camel-folder-search.c (camel_folder_search_search): new search
+	api entry point, take a full summary and optionally a subset of
+	uids to match against.
+	(search_match_all): use the uids' passed in to only search a
+	subset of uid's.
+
 	* providers/imap/camel-imap-store.c (connect_to_server): set
 	nodelay and keepalive on the socket.
 
diff --git a/camel/camel-digest-folder.c b/camel/camel-digest-folder.c
index b2c81b5d4d..1e814fdf6a 100644
--- a/camel/camel-digest-folder.c
+++ b/camel/camel-digest-folder.c
@@ -315,31 +315,22 @@ digest_get_message (CamelFolder *folder, const char *uid, CamelException *ex)
 	return message;
 }
 
-
 static GPtrArray *
 digest_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex)
 {
 	CamelDigestFolder *df = (CamelDigestFolder *) folder;
-	CamelFolderSearch *search;
-	GPtrArray *summary, *matches;
-	
-	summary = camel_folder_get_summary (folder);
+	GPtrArray *matches;
 	
 	CAMEL_DIGEST_FOLDER_LOCK (folder, search_lock);
 	
 	if (!df->priv->search)
 		df->priv->search = camel_folder_search_new ();
 	
-	search = df->priv->search;
-	camel_folder_search_set_folder (search, folder);
-	camel_folder_search_set_summary (search, summary);
-	
-	matches = camel_folder_search_execute_expression (search, expression, ex);
+	camel_folder_search_set_folder (df->priv->search, folder);
+	matches = camel_folder_search_search(df->priv->search, expression, NULL, ex);
 	
 	CAMEL_DIGEST_FOLDER_UNLOCK (folder, search_lock);
 	
-	camel_folder_free_summary (folder, summary);
-	
 	return matches;
 }
 
@@ -347,39 +338,21 @@ static GPtrArray *
 digest_search_by_uids (CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
 {
 	CamelDigestFolder *df = (CamelDigestFolder *) folder;
-	CamelFolderSearch *search;
-	GPtrArray *summary, *matches;
-	int i;
-	
-	summary = g_ptr_array_new ();
-	for (i = 0; i < uids->len; i++) {
-		CamelMessageInfo *info;
-		
-		info = camel_folder_get_message_info (folder, uids->pdata[i]);
-		if (info)
-			g_ptr_array_add (summary, info);
-	}
-	
-	if (summary->len == 0)
-		return summary;
-	
+	GPtrArray *matches;
+
+	if (uids->len == 0)
+		return g_ptr_array_new();
+
 	CAMEL_DIGEST_FOLDER_LOCK (folder, search_lock);
 	
 	if (!df->priv->search)
 		df->priv->search = camel_folder_search_new ();
 	
-	search = df->priv->search;
-	camel_folder_search_set_folder (search, folder);
-	camel_folder_search_set_summary (search, summary);
-	
-	matches = camel_folder_search_execute_expression (search, expression, ex);
+	camel_folder_search_set_folder (df->priv->search, folder);
+	matches = camel_folder_search_search(df->priv->search, expression, NULL, ex);
 	
 	CAMEL_DIGEST_FOLDER_UNLOCK (folder, search_lock);
 	
-	for (i = 0; i < summary->len; i++)
-		camel_folder_free_message_info (folder, summary->pdata[i]);
-	g_ptr_array_free (summary, TRUE);
-	
 	return matches;
 }
 
diff --git a/camel/camel-folder-search.c b/camel/camel-folder-search.c
index e50c2acd0f..9e8f66051e 100644
--- a/camel/camel-folder-search.c
+++ b/camel/camel-folder-search.c
@@ -36,6 +36,7 @@
 #include <glib.h>
 
 #include "camel-folder-search.h"
+#include "camel-folder-thread.h"
 
 #include "camel-exception.h"
 #include "camel-medium.h"
@@ -51,6 +52,9 @@
 struct _CamelFolderSearchPrivate {
 	GHashTable *mempool_hash;
 	CamelException *ex;
+
+	CamelFolderThread *threads;
+	GHashTable *threads_hash;
 };
 
 #define _PRIVATE(o) (((CamelFolderSearch *)(o))->priv)
@@ -63,6 +67,7 @@ static ESExpResult *search_header_starts_with(struct _ESExp *f, int argc, struct
 static ESExpResult *search_header_ends_with(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *search);
 static ESExpResult *search_header_exists(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *search);
 static ESExpResult *search_match_all(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFolderSearch *search);
+static ESExpResult *search_match_threads(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFolderSearch *s);
 static ESExpResult *search_body_contains(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *search);
 static ESExpResult *search_user_flag(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *s);
 static ESExpResult *search_user_tag(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *s);
@@ -89,6 +94,7 @@ camel_folder_search_class_init (CamelFolderSearchClass *klass)
 	klass->not = search_not;
 
 	klass->match_all = search_match_all;
+	klass->match_threads = search_match_threads;
 	klass->body_contains = search_body_contains;
 	klass->header_contains = search_header_contains;
 	klass->header_matches = search_header_matches;
@@ -191,6 +197,7 @@ struct {
 	/* these we have to use our own default if there is none */
 	/* they should all be defined in the language? so it parses, or should they not?? */
 	{ "match-all", CAMEL_STRUCT_OFFSET(CamelFolderSearchClass, match_all), 3 },
+	{ "match-threads", CAMEL_STRUCT_OFFSET(CamelFolderSearchClass, match_threads), 3 },
 	{ "body-contains", CAMEL_STRUCT_OFFSET(CamelFolderSearchClass, body_contains), 1 },
 	{ "header-contains", CAMEL_STRUCT_OFFSET(CamelFolderSearchClass, header_contains), 1 },
 	{ "header-matches", CAMEL_STRUCT_OFFSET(CamelFolderSearchClass, header_matches), 1 },
@@ -400,6 +407,13 @@ camel_folder_search_execute_expression(CamelFolderSearch *search, const char *ex
 
 	e_sexp_result_free(search->sexp, r);
 
+	if (p->threads)
+		camel_folder_thread_messages_unref(p->threads);
+	if (p->threads_hash)
+		g_hash_table_destroy(p->threads_hash);
+
+	p->threads = NULL;
+	p->threads_hash = NULL;
 	search->folder = NULL;
 	search->summary = NULL;
 	search->current = NULL;
@@ -409,35 +423,126 @@ camel_folder_search_execute_expression(CamelFolderSearch *search, const char *ex
 }
 
 /**
- * camel_folder_search_match_expression:
+ * camel_folder_search_search:
  * @search: 
  * @expr: 
- * @info: 
+ * @uids: to search against, NULL for all uid's.
  * @ex: 
  * 
- * Returns #TRUE if the expression matches the specific message info @info.
- * Note that the folder and index may need to be set for body searches to
- * operate as well.
+ * Run a search.  Search must have had Folder already set on it, and
+ * it must implement summaries.
  * 
  * Return value: 
  **/
-gboolean
-camel_folder_search_match_expression(CamelFolderSearch *search, const char *expr, const CamelMessageInfo *info, CamelException *ex)
+GPtrArray *
+camel_folder_search_search(CamelFolderSearch *search, const char *expr, GPtrArray *uids, CamelException *ex)
 {
-	GPtrArray *uids;
-	int ret = FALSE;
+	ESExpResult *r;
+	GPtrArray *matches = NULL, *summary_set;
+	int i;
+	GHashTable *results;
+	EMemPool *pool;
+	struct _CamelFolderSearchPrivate *p = _PRIVATE(search);
+
+	g_assert(search->folder);
 
-	search->current = (CamelMessageInfo *)info;
+	p->ex = ex;
+
+	/* setup our search list, summary_hash only contains those we're interested in */
+	search->summary = camel_folder_get_summary(search->folder);
+	search->summary_hash = g_hash_table_new(g_str_hash, g_str_equal);
 
-	uids = camel_folder_search_execute_expression(search, expr, ex);
 	if (uids) {
-		if (uids->len == 1)
-			ret = TRUE;
-		camel_folder_search_free_result(search, uids);
+		GHashTable *uids_hash = g_hash_table_new(g_str_hash, g_str_equal);
+
+		summary_set = search->summary_set = g_ptr_array_new();
+		for (i=0;i<uids->len;i++)
+			g_hash_table_insert(uids_hash, uids->pdata[i], uids->pdata[i]);
+		for (i=0;i<search->summary->len;i++)
+			if (g_hash_table_lookup(uids_hash, camel_message_info_uid(search->summary->pdata[i])))
+				g_ptr_array_add(search->summary_set, search->summary->pdata[i]);
+	} else {
+		summary_set = search->summary;
 	}
+
+	for (i=0;i<summary_set->len;i++)
+		g_hash_table_insert(search->summary_hash, (char *)camel_message_info_uid(summary_set->pdata[i]), summary_set->pdata[i]);
+
+	/* only re-parse if the search has changed */
+	if (search->last_search == NULL
+	    || strcmp(search->last_search, expr)) {
+		e_sexp_input_text(search->sexp, expr, strlen(expr));
+		if (e_sexp_parse(search->sexp) == -1) {
+			camel_exception_setv(ex, 1, _("Cannot parse search expression: %s:\n%s"), e_sexp_error(search->sexp), expr);
+			goto fail;
+		}
+
+		g_free(search->last_search);
+		search->last_search = g_strdup(expr);
+	}
+	r = e_sexp_eval(search->sexp);
+	if (r == NULL) {
+		if (!camel_exception_is_set(ex))
+			camel_exception_setv(ex, 1, _("Error executing search expression: %s:\n%s"), e_sexp_error(search->sexp), expr);
+		goto fail;
+	}
+
+	matches = g_ptr_array_new();
+
+	/* now create a folder summary to return?? */
+	if (r->type == ESEXP_RES_ARRAY_PTR) {
+		d(printf("got result ...\n"));
+
+		/* we use a mempool to store the strings, packed in tight as possible, and freed together */
+		/* because the strings are often short (like <8 bytes long), we would be wasting appx 50%
+		   of memory just storing the size tag that malloc assigns us and alignment padding, so this
+		   gets around that (and is faster to allocate and free as a bonus) */
+		pool = e_mempool_new(512, 256, E_MEMPOOL_ALIGN_BYTE);
+		/* reorder result in summary order */
+		results = g_hash_table_new(g_str_hash, g_str_equal);
+		for (i=0;i<r->value.ptrarray->len;i++) {
+			d(printf("adding match: %s\n", (char *)g_ptr_array_index(r->value.ptrarray, i)));
+			g_hash_table_insert(results, g_ptr_array_index(r->value.ptrarray, i), GINT_TO_POINTER (1));
+		}
+
+		for (i=0;i<summary_set->len;i++) {
+			CamelMessageInfo *info = g_ptr_array_index(summary_set, i);
+			char *uid = (char *)camel_message_info_uid(info);
+			if (g_hash_table_lookup(results, uid))
+				g_ptr_array_add(matches, e_mempool_strdup(pool, uid));
+		}
+		g_hash_table_destroy(results);
+
+		/* instead of putting the mempool_hash in the structure, we keep the api clean by
+		   putting a reference to it in a hashtable.  Lets us do some debugging and catch
+		   unfree'd results as well. */
+		g_hash_table_insert(p->mempool_hash, matches, pool);
+	} else {
+		g_warning("Search returned an invalid result type");
+	}
+
+	e_sexp_result_free(search->sexp, r);
+fail:
+	/* these might be allocated by match-threads */
+	if (p->threads)
+		camel_folder_thread_messages_unref(p->threads);
+	if (p->threads_hash)
+		g_hash_table_destroy(p->threads_hash);
+	if (search->summary_set)
+		g_ptr_array_free(search->summary_set, TRUE);
+	g_hash_table_destroy(search->summary_hash);
+	camel_folder_free_summary(search->folder, search->summary);
+
+	p->threads = NULL;
+	p->threads_hash = NULL;
+	search->folder = NULL;
+	search->summary = NULL;
+	search->summary_hash = NULL;
+	search->summary_set = NULL;
 	search->current = NULL;
+	search->body_index = NULL;
 
-	return ret;
+	return matches;
 }
 
 void camel_folder_search_free_result(CamelFolderSearch *search, GPtrArray *result)
@@ -457,9 +562,6 @@ void camel_folder_search_free_result(CamelFolderSearch *search, GPtrArray *resul
 	g_ptr_array_free(result, TRUE);
 }
 
-
-
-
 /* dummy function, returns false always, or an empty match array */
 static ESExpResult *
 search_dummy(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *search)
@@ -516,7 +618,7 @@ search_not(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSe
 				for (i=0;i<v->len;i++)
 					g_hash_table_insert(have, s[i], s[i]);
 
-				v = search->summary;
+				v = search->summary_set?search->summary_set:search->summary;
 				m = (CamelMessageInfo **)v->pdata;
 				for (i=0;i<v->len;i++) {
 					char *uid = (char *)camel_message_info_uid(m[i]);
@@ -548,6 +650,7 @@ search_match_all(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFold
 {
 	int i;
 	ESExpResult *r, *r1;
+	GPtrArray *v;
 
 	if (argc>1) {
 		g_warning("match-all only takes a single argument, other arguments ignored");
@@ -585,21 +688,25 @@ search_match_all(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFold
 		return r;
 	}
 
-	/* TODO: Could make this a bit faster in the uncommon case (of match-everything) */
-	for (i=0;i<search->summary->len;i++) {
-		search->current = g_ptr_array_index(search->summary, i);
+	v = search->summary_set?search->summary_set:search->summary;
+	for (i=0;i<v->len;i++) {
+		const char *uid;
+
+		search->current = g_ptr_array_index(v, i);
+		uid = camel_message_info_uid(search->current);
+
 		if (argc>0) {
 			r1 = e_sexp_term_eval(f, argv[0]);
 			if (r1->type == ESEXP_RES_BOOL) {
 				if (r1->value.bool)
-					g_ptr_array_add(r->value.ptrarray, (char *)camel_message_info_uid(search->current));
+					g_ptr_array_add(r->value.ptrarray, (char *)uid);
 			} else {
 				g_warning("invalid syntax, matches require a single bool result");
 				e_sexp_fatal_error(f, _("(match-all) requires a single bool result"));
 			}
 			e_sexp_result_free(f, r1);
 		} else {
-			g_ptr_array_add(r->value.ptrarray, (char *)camel_message_info_uid(search->current));
+			g_ptr_array_add(r->value.ptrarray, (char *)uid);
 		}
 	}
 	search->current = NULL;
@@ -607,6 +714,129 @@ search_match_all(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFold
 	return r;
 }
 
+static void
+fill_thread_table(struct _CamelFolderThreadNode *root, GHashTable *id_hash)
+{
+	while (root) {
+		g_hash_table_insert(id_hash, (char *)camel_message_info_uid(root->message), root);
+		if (root->child)
+			fill_thread_table(root->child, id_hash);
+		root = root->next;
+	}
+}
+
+static void
+add_thread_results(struct _CamelFolderThreadNode *root, GHashTable *result_hash)
+{
+	while (root) {
+		g_hash_table_insert(result_hash, (char *)camel_message_info_uid(root->message), GINT_TO_POINTER (1));
+		if (root->child)
+			add_thread_results(root->child, result_hash);
+		root = root->next;
+	}
+}
+
+static void
+add_results(char *uid, void *dummy, GPtrArray *result)
+{
+	g_ptr_array_add(result, uid);
+}
+
+static ESExpResult *
+search_match_threads(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFolderSearch *search)
+{
+	ESExpResult *r;
+	struct _CamelFolderSearchPrivate *p = search->priv;
+	int i, type;
+	GHashTable *results;
+
+	/* not supported in match-all */
+	if (search->current)
+		e_sexp_fatal_error(f, _("(match-threads) now allowed inside match-all"));
+
+	if (argc == 0)
+		e_sexp_fatal_error(f, _("(match-threads) requires a match type string"));
+
+	r = e_sexp_term_eval(f, argv[0]);
+	if (r->type != ESEXP_RES_STRING)
+		e_sexp_fatal_error(f, _("(match-threads) requires a match type string"));
+
+	type = 0;
+	if (!strcmp(r->value.string, "none"))
+		type = 0;
+	else if (!strcmp(r->value.string, "all"))
+		type = 1;
+	else if (!strcmp(r->value.string, "replies"))
+		type = 2;
+	else if (!strcmp(r->value.string, "replies_parents"))
+		type = 3;
+	e_sexp_result_free(f, r);
+
+	/* behave as (begin does */
+	r = NULL;
+	for (i=1;i<argc;i++) {
+		if (r)
+			e_sexp_result_free(f, r);
+		r = e_sexp_term_eval(f, argv[i]);
+	}
+
+	printf("match-threads, result %d\n", r==NULL?-1:r->type);
+	if (r == NULL || r->type != ESEXP_RES_ARRAY_PTR)
+		e_sexp_fatal_error(f, _("(match-threads) expects an array result"));
+
+	if (type == 0)
+		return r;
+
+	if (search->folder == NULL)
+		e_sexp_fatal_error(f, _("(match-threads) requires the folder set"));
+
+	/* cache this, so we only have to re-calculate once per search at most */
+	if (p->threads == NULL) {
+		p->threads = camel_folder_thread_messages_new(search->folder, NULL, TRUE);
+		p->threads_hash = g_hash_table_new(g_str_hash, g_str_equal);
+
+		fill_thread_table(p->threads->tree, p->threads_hash);
+	}
+
+	results = g_hash_table_new(g_str_hash, g_str_equal);
+	for (i=0;i<r->value.ptrarray->len;i++) {
+		d(printf("adding match: %s\n", (char *)g_ptr_array_index(r->value.ptrarray, i)));
+		g_hash_table_insert(results, g_ptr_array_index(r->value.ptrarray, i), GINT_TO_POINTER (1));
+	}
+
+	for (i=0;i<r->value.ptrarray->len;i++) {
+		struct _CamelFolderThreadNode *node, *scan;
+
+		node = g_hash_table_lookup(p->threads_hash, (char *)g_ptr_array_index(r->value.ptrarray, i));
+		if (node == NULL) /* this shouldn't happen but why cry over spilt milk */
+			continue;
+
+		/* select messages in thread according to search criteria */
+		if (type == 3) {
+			scan = node;
+			while (scan && scan->parent) {
+				scan = scan->parent;
+				g_hash_table_insert(results, (char *)camel_message_info_uid(scan->message), GINT_TO_POINTER(1));
+			}
+		} else if (type == 1) {
+			while (node && node->parent)
+				node = node->parent;
+		}
+		g_hash_table_insert(results, (char *)camel_message_info_uid(node->message), GINT_TO_POINTER(1));
+		if (node->child)
+			add_thread_results(node->child, results);
+	}
+	e_sexp_result_free(f, r);
+
+	r = e_sexp_result_new(f, ESEXP_RES_ARRAY_PTR);
+	r->value.ptrarray = g_ptr_array_new();
+
+	g_hash_table_foreach(results, (GHFunc)add_results, r->value.ptrarray);
+	g_hash_table_destroy(results);
+
+	return r;
+}
+
 static ESExpResult *
 check_header(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *search, camel_search_match_t how)
 {
@@ -917,8 +1147,10 @@ match_words_messages(CamelFolderSearch *search, struct _camel_search_words *word
 		
 		g_ptr_array_free(indexed, TRUE);
 	} else {
-		for (i=0;i<search->summary->len;i++) {
-			CamelMessageInfo *info = g_ptr_array_index(search->summary, i);
+		GPtrArray *v = search->summary_set?search->summary_set:search->summary;
+
+		for (i=0;i<v->len;i++) {
+			CamelMessageInfo *info = g_ptr_array_index(v, i);
 			const char *uid = camel_message_info_uid(info);
 			
 			if (match_words_message(search->folder, uid, words, ex))
@@ -966,8 +1198,10 @@ search_body_contains(struct _ESExp *f, int argc, struct _ESExpResult **argv, Cam
 		r->value.ptrarray = g_ptr_array_new();
 
 		if (argc == 1 && argv[0]->value.string[0] == 0) {
-			for (i=0;i<search->summary->len;i++) {
-				CamelMessageInfo *info = g_ptr_array_index(search->summary, i);
+			GPtrArray *v = search->summary_set?search->summary_set:search->summary;
+
+			for (i=0;i<v->len;i++) {
+				CamelMessageInfo *info = g_ptr_array_index(v, i);
 
 				g_ptr_array_add(r->value.ptrarray, (char *)camel_message_info_uid(info));
 			}
diff --git a/camel/camel-folder-search.h b/camel/camel-folder-search.h
index e4d2253224..a5239f2a0e 100644
--- a/camel/camel-folder-search.h
+++ b/camel/camel-folder-search.h
@@ -50,6 +50,7 @@ struct _CamelFolderSearch {
 	/* these are only valid during the search, and are reset afterwards */
 	CamelFolder *folder;	/* folder for current search */
 	GPtrArray *summary;	/* summary array for current search */
+	GPtrArray *summary_set;	/* subset of summary to actually include in search */
 	GHashTable *summary_hash; /* hashtable of summary items */
 	CamelMessageInfo *current; /* current message info, when searching one by one */
 	CamelMimeMessage *current_message; /* cache of current message, if required */
@@ -71,6 +72,9 @@ struct _CamelFolderSearchClass {
 	/* (match-all [boolean expression]) Apply match to all messages */
 	ESExpResult * (*match_all)(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFolderSearch *s);
 
+	/* (match-threads "type" [array expression]) add all related threads */
+	ESExpResult * (*match_threads)(struct _ESExp *f, int argc, struct _ESExpTerm **argv, CamelFolderSearch *s);
+
 	/* (body-contains "string1" "string2" ...) Returns a list of matches, or true if in single-message mode */
 	ESExpResult * (*body_contains)(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *s);
 
@@ -118,12 +122,14 @@ CamelType		camel_folder_search_get_type	(void);
 CamelFolderSearch      *camel_folder_search_new	(void);
 void camel_folder_search_construct (CamelFolderSearch *search);
 
+/* This stuff currently gets cleared when you run a search ... what on earth was i thinking ... */
 void camel_folder_search_set_folder(CamelFolderSearch *search, CamelFolder *folder);
 void camel_folder_search_set_summary(CamelFolderSearch *search, GPtrArray *summary);
 void camel_folder_search_set_body_index(CamelFolderSearch *search, CamelIndex *index);
+/* this interface is deprecated */
 GPtrArray *camel_folder_search_execute_expression(CamelFolderSearch *search, const char *expr, CamelException *ex);
-gboolean camel_folder_search_match_expression(CamelFolderSearch *search, const char *expr,
-					      const CamelMessageInfo *info, CamelException *ex);
+
+GPtrArray *camel_folder_search_search(CamelFolderSearch *search, const char *expr, GPtrArray *uids, CamelException *ex);
 void camel_folder_search_free_result(CamelFolderSearch *search, GPtrArray *);
 
 #ifdef __cplusplus
diff --git a/camel/providers/imap/camel-imap-folder.c b/camel/providers/imap/camel-imap-folder.c
index 4e4745c6ec..7915441047 100644
--- a/camel/providers/imap/camel-imap-folder.c
+++ b/camel/providers/imap/camel-imap-folder.c
@@ -1574,7 +1574,7 @@ static GPtrArray *
 imap_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex)
 {
 	CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
-	GPtrArray *matches, *summary;
+	GPtrArray *matches;
 
 	/* we could get around this by creating a new search object each time,
 	   but i doubt its worth it since any long operation would lock the
@@ -1582,14 +1582,10 @@ imap_search_by_expression (CamelFolder *folder, const char *expression, CamelExc
 	CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
 
 	camel_folder_search_set_folder (imap_folder->search, folder);
-	summary = camel_folder_get_summary(folder);
-	camel_folder_search_set_summary(imap_folder->search, summary);
-	matches = camel_folder_search_execute_expression (imap_folder->search, expression, ex);
+	matches = camel_folder_search_search(imap_folder->search, expression, NULL, ex);
 
 	CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
 
-	camel_folder_free_summary(folder, summary);
-
 	return matches;
 }
 
@@ -1597,39 +1593,18 @@ static GPtrArray *
 imap_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
 {
 	CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER(folder);
-	GPtrArray *summary, *matches;
-	int i;
-
-	/* NOTE: could get away without the search lock by creating a new
-	   search object each time */
-	
-	qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
-	
-	summary = g_ptr_array_new();
-	for (i=0;i<uids->len;i++) {
-		CamelMessageInfo *info;
-
-		info = camel_folder_get_message_info(folder, uids->pdata[i]);
-		if (info)
-			g_ptr_array_add(summary, info);
-	}
+	GPtrArray *matches;
 
-	if (summary->len == 0)
-		return summary;
+	if (uids->len == 0)
+		return g_ptr_array_new();
 
 	CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
 
 	camel_folder_search_set_folder(imap_folder->search, folder);
-	camel_folder_search_set_summary(imap_folder->search, summary);
-
-	matches = camel_folder_search_execute_expression(imap_folder->search, expression, ex);
+	matches = camel_folder_search_search(imap_folder->search, expression, uids, ex);
 
 	CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
 
-	for (i=0;i<summary->len;i++)
-		camel_folder_free_message_info(folder, summary->pdata[i]);
-	g_ptr_array_free(summary, TRUE);
-
 	return matches;
 }
 
diff --git a/camel/providers/local/camel-local-folder.c b/camel/providers/local/camel-local-folder.c
index bd9d259dce..2c352c749d 100644
--- a/camel/providers/local/camel-local-folder.c
+++ b/camel/providers/local/camel-local-folder.c
@@ -575,10 +575,7 @@ static GPtrArray *
 local_search_by_expression(CamelFolder *folder, const char *expression, CamelException *ex)
 {
 	CamelLocalFolder *local_folder = CAMEL_LOCAL_FOLDER(folder);
-	GPtrArray *summary, *matches;
-
-	/* NOTE: could get away without the search lock by creating a new
-	   search object each time */
+	GPtrArray *matches;
 
 	CAMEL_LOCAL_FOLDER_LOCK(folder, search_lock);
 
@@ -587,15 +584,10 @@ local_search_by_expression(CamelFolder *folder, const char *expression, CamelExc
 
 	camel_folder_search_set_folder(local_folder->search, folder);
 	camel_folder_search_set_body_index(local_folder->search, local_folder->index);
-	summary = camel_folder_get_summary(folder);
-	camel_folder_search_set_summary(local_folder->search, summary);
-
-	matches = camel_folder_search_execute_expression(local_folder->search, expression, ex);
+	matches = camel_folder_search_search(local_folder->search, expression, NULL, ex);
 
 	CAMEL_LOCAL_FOLDER_UNLOCK(folder, search_lock);
 
-	camel_folder_free_summary(folder, summary);
-
 	return matches;
 }
 
@@ -603,23 +595,10 @@ static GPtrArray *
 local_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
 {
 	CamelLocalFolder *local_folder = CAMEL_LOCAL_FOLDER(folder);
-	GPtrArray *summary, *matches;
-	int i;
-
-	/* NOTE: could get away without the search lock by creating a new
-	   search object each time */
+	GPtrArray *matches;
 
-	summary = g_ptr_array_new();
-	for (i=0;i<uids->len;i++) {
-		CamelMessageInfo *info;
-
-		info = camel_folder_get_message_info(folder, uids->pdata[i]);
-		if (info)
-			g_ptr_array_add(summary, info);
-	}
-
-	if (summary->len == 0)
-		return summary;
+	if (uids->len == 0)
+		return g_ptr_array_new();
 
 	CAMEL_LOCAL_FOLDER_LOCK(folder, search_lock);
 
@@ -628,16 +607,10 @@ local_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uid
 
 	camel_folder_search_set_folder(local_folder->search, folder);
 	camel_folder_search_set_body_index(local_folder->search, local_folder->index);
-	camel_folder_search_set_summary(local_folder->search, summary);
-
-	matches = camel_folder_search_execute_expression(local_folder->search, expression, ex);
+	matches = camel_folder_search_search(local_folder->search, expression, uids, ex);
 
 	CAMEL_LOCAL_FOLDER_UNLOCK(folder, search_lock);
 
-	for (i=0;i<summary->len;i++)
-		camel_folder_free_message_info(folder, summary->pdata[i]);
-	g_ptr_array_free(summary, TRUE);
-
 	return matches;
 }
 
diff --git a/camel/providers/nntp/camel-nntp-folder.c b/camel/providers/nntp/camel-nntp-folder.c
index 9fc236e215..4762cf974a 100644
--- a/camel/providers/nntp/camel-nntp-folder.c
+++ b/camel/providers/nntp/camel-nntp-folder.c
@@ -287,7 +287,7 @@ static GPtrArray*
 nntp_folder_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex)
 {
 	CamelNNTPFolder *nntp_folder = CAMEL_NNTP_FOLDER (folder);
-	GPtrArray *matches, *summary;
+	GPtrArray *matches;
 	
 	CAMEL_NNTP_FOLDER_LOCK(nntp_folder, search_lock);
 	
@@ -295,15 +295,10 @@ nntp_folder_search_by_expression (CamelFolder *folder, const char *expression, C
 		nntp_folder->search = camel_folder_search_new ();
 	
 	camel_folder_search_set_folder (nntp_folder->search, folder);
-	summary = camel_folder_get_summary (folder);
-	camel_folder_search_set_summary (nntp_folder->search, summary);
-	
-	matches = camel_folder_search_execute_expression (nntp_folder->search, expression, ex);
+	matches = camel_folder_search_search(nntp_folder->search, expression, NULL, ex);
 	
 	CAMEL_NNTP_FOLDER_UNLOCK(nntp_folder, search_lock);
 	
-	camel_folder_free_summary (folder, summary);
-	
 	return matches;
 }
 
@@ -311,22 +306,10 @@ static GPtrArray *
 nntp_folder_search_by_uids (CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
 {
 	CamelNNTPFolder *nntp_folder = (CamelNNTPFolder *) folder;
-	GPtrArray *summary, *matches;
-	int i;
-	
-	/* NOTE: could get away without the search lock by creating a new
-	   search object each time */
-	
-	summary = g_ptr_array_new ();
-	for (i = 0; i < uids->len; i++) {
-		CamelMessageInfo *info;
-		
-		if ((info = camel_folder_get_message_info (folder, uids->pdata[i])))
-			g_ptr_array_add (summary, info);
-	}
-	
-	if (summary->len == 0)
-		return summary;
+	GPtrArray *matches;
+
+	if (uids->len == 0)
+		return g_ptr_array_new();
 	
 	CAMEL_NNTP_FOLDER_LOCK(folder, search_lock);
 	
@@ -334,17 +317,10 @@ nntp_folder_search_by_uids (CamelFolder *folder, const char *expression, GPtrArr
 		nntp_folder->search = camel_folder_search_new ();
 	
 	camel_folder_search_set_folder (nntp_folder->search, folder);
-	camel_folder_search_set_summary (nntp_folder->search, summary);
-	
-	matches = camel_folder_search_execute_expression (nntp_folder->search, expression, ex);
+	matches = camel_folder_search_search(nntp_folder->search, expression, uids, ex);
 	
 	CAMEL_NNTP_FOLDER_UNLOCK(folder, search_lock);
 	
-	for (i = 0; i < summary->len; i++)
-		camel_folder_free_message_info (folder, summary->pdata[i]);
-	
-	g_ptr_array_free (summary, TRUE);
-	
 	return matches;
 }