From c6fc4e27a953c5213cff8400ec8e40f6f051e914 Mon Sep 17 00:00:00 2001 From: Not Zed Date: Mon, 25 Mar 2002 12:11:44 +0000 Subject: When we add a new name, up all of the cache limits, because we're probably 2002-03-25 Not Zed * camel-text-index.c (text_index_add_name): When we add a new name, up all of the cache limits, because we're probably going to be adding more. (text_index_sync): Drop the cache limits back down again, we dont need them when looking words up. ** MERGE camel_index branch. * camel-text-index.[ch]: Added files i forgot to add (eep nearly lost all this work!) * camel-block-file.c (sync_nolock): Fix an infinite loop in syncing. svn path=/trunk/; revision=16242 --- camel/ChangeLog | 130 ++ camel/Makefile.am | 18 +- camel/camel-block-file.c | 1124 ++++++++++++++++ camel/camel-block-file.h | 142 +++ camel/camel-filter-driver.c | 7 +- camel/camel-folder-search.c | 132 +- camel/camel-folder-search.h | 7 +- camel/camel-folder-summary.c | 184 +-- camel/camel-folder-summary.h | 4 +- camel/camel-index-control.c | 212 ++++ camel/camel-index.c | 341 +++++ camel/camel-index.h | 159 +++ camel/camel-mime-filter-index.c | 40 +- camel/camel-mime-filter-index.h | 11 +- camel/camel-partition-table.c | 955 ++++++++++++++ camel/camel-partition-table.h | 150 +++ camel/camel-private.h | 4 +- camel/camel-text-index.c | 1687 +++++++++++++++++++++++++ camel/camel-text-index.h | 109 ++ camel/providers/local/camel-local-folder.c | 29 +- camel/providers/local/camel-local-folder.h | 6 +- camel/providers/local/camel-local-store.c | 15 +- camel/providers/local/camel-local-summary.c | 10 +- camel/providers/local/camel-local-summary.h | 6 +- camel/providers/local/camel-maildir-folder.c | 4 +- camel/providers/local/camel-maildir-summary.c | 51 +- camel/providers/local/camel-maildir-summary.h | 4 +- camel/providers/local/camel-mbox-folder.c | 4 +- camel/providers/local/camel-mbox-store.c | 2 + camel/providers/local/camel-mbox-summary.c | 8 +- camel/providers/local/camel-mbox-summary.h | 2 +- camel/providers/local/camel-mh-folder.c | 4 +- camel/providers/local/camel-mh-summary.c | 14 +- camel/providers/local/camel-mh-summary.h | 4 +- camel/providers/local/camel-spool-folder.h | 4 +- camel/providers/local/camel-spool-summary.c | 5 +- camel/providers/local/camel-spool-summary.h | 4 +- camel/providers/local/camel-spoold-store.c | 2 +- 38 files changed, 5346 insertions(+), 248 deletions(-) create mode 100644 camel/camel-block-file.c create mode 100644 camel/camel-block-file.h create mode 100644 camel/camel-index-control.c create mode 100644 camel/camel-index.c create mode 100644 camel/camel-index.h create mode 100644 camel/camel-partition-table.c create mode 100644 camel/camel-partition-table.h create mode 100644 camel/camel-text-index.c create mode 100644 camel/camel-text-index.h diff --git a/camel/ChangeLog b/camel/ChangeLog index 6b10dade6e..ea534ae72c 100644 --- a/camel/ChangeLog +++ b/camel/ChangeLog @@ -1,8 +1,138 @@ +2002-03-25 Not Zed + + * camel-text-index.c (text_index_add_name): When we add a new + name, up all of the cache limits, because we're probably going to + be adding more. + (text_index_sync): Drop the cache limits back down again, we dont + need them when looking words up. + + ** MERGE camel_index branch. + + * camel-text-index.[ch]: Added files i forgot to add (eep nearly + lost all this work!) + + * camel-block-file.c (sync_nolock): Fix an infinite loop in syncing. + 2002-03-21 Jeffrey Stedfast * camel-folder-summary.c (camel_message_info_new_from_header): Use the date in the received header for the received_date. +2002-03-22 Not Zed + + * providers/local/camel-local-folder.c + (camel_local_folder_construct): Use the right option to remove the + index file and reset the index file on creation. + (camel_local_folder_construct): Remove any existing '.ibex' files + - these are the old format index files. + + * camel-block-file.c (camel_block_file_rename): Lock io lock while + renaming. + (camel_key_file_rename): Lock around rename. + (block_file_validate_root): Implement sync flag checking. + (camel_block_file_touch_block): Turn off the sync flag if we're + touching any non-root block and write it to disk. Shoudl this + fsync()? + (sync_nolock): sync the root block only if we need to. + + * providers/local/camel-local-store.c (rename_folder): Re-enable + index rename code. Not sure how race-free it is though. + (delete_folder): Delete the index file properly. + + * camel-partition-table.c (camel_key_table_lookup): Initialise + output values to 0 before doing anything. + (camel_key_table_add): Do some range-checking on values. + + * camel-text-index.c (text_index_compress): Changed to call sync here. + (text_index_compress_nolock): and not here - stops a recursive + sync call when sync performs a compress also. + (text_index_compress_nolock): Change to _nosync, since the locking + is irrelevent (recursive lock). Fixed callers. + (text_index_add_name_to_word): If we get a failure with key table + ops, fail immediately. + (text_index_compress_nosync): Likewise. + (text_index_write_name): If the nameid is 0, do nothing. + (text_index_add_name): If we can't get a keyid, dont add it to the + partition table. + (camel_text_index_remove): Function to delete an index file. + (text_index_compress_nosync): Clean up temp files when done. + + * camel-folder-search.c (match_messages_index): New function, + split from body_contains, matches a regex against all words in an + index. + (match_message_index): Similar to above but matches against an + individual message. + (search_body_contains): Changed to use above functions for + matching - substring matches should now work on indexed data. + +2002-03-21 Not Zed + + * camel-index.c (camel_index_words/names): New virtual + methods/stubs to get a cursor of all words and names. + + * camel-text-index.c (text_index_compress_nolock): Split from + text_index_compress, so we can call the compressor while locked + also, removed lock calls. + (text_index_compress): Changed to stub which calls + text_index_compress_nolock. + (camel_text_index_key_cursor_new): New object to iterate through + a key table. + (text_index_words, text_index_names): Implement virtual functions + for iterating through all words or names. + + * camel-block-file.c: Turn off some debug. + +2002-03-20 Not Zed + + ** New body index implementation. + + * camel-index*: Code for camel index, a new class to replace ibex. + + * camel-block-file.[ch]: block-based and link based + filesystem-in-a-file classes. + + * camel-partition-table.[ch]: An implementation of a partition + table (automatically extending on-disk hash-table using ideal + hash), and a key-table, a key<>name mapping table. Used by + camelindex. + + * providers/local/*, camel-folder-summary.[ch]: Changed to use + camel-index interface rather than ibex. + +2002-03-05 Not Zed + + * providers/local/camel-maildir-summary.c (maildir_summary_check): + Do progress reporting of operations. + (maildir_summary_sync): Same here. + +2002-03-04 Not Zed + + * providers/local/camel-spoold-store.c (scan_dir): Kill a warning + with a cast. + + * providers/local/camel-*.c: Changed for ibex->camelindex. + + * camel-folder-search.c (camel_folder_search_set_summary): Init + summary_hash to point to 'static' uid strings. + (search_body_contains): Use the static uid memory to return + results rather than the values from the index library. + + * camel-folder-search.[ch]: Changed to use camelindex object. + + * camel-folder-summary.c (summary_build_content_info_message): + Use a stream to index content, also filter html mail first. + (camel_folder_summary_info_new_from_message): Use a stream + filtered to index content. + (main): Removed the test main code. Added headers for open call + (ibex must've had them before). + + * camel-folder-summary.[ch]: Changed from ibex to CamelIndex. + + * camel-mime-filter-index.c (camel_mime_filter_index_finalize): + Unref index. + + * camel-mime-filter-index.[ch]: Changed from ibex to CamelIndex. + 2002-03-19 Jeffrey Stedfast * camel-mime-utils.c (header_encode_param): Fix this to work diff --git a/camel/Makefile.am b/camel/Makefile.am index 5c00f8c0f4..d38787c218 100644 --- a/camel/Makefile.am +++ b/camel/Makefile.am @@ -4,7 +4,7 @@ SUBDIRS = providers tests libcamelincludedir = $(includedir)/camel -sbin_PROGRAMS = camel-lock-helper +sbin_PROGRAMS = camel-lock-helper camel-index-control lib_LTLIBRARIES = libcamel.la noinst_LTLIBRARIES = libcamel-static.la @@ -20,6 +20,7 @@ INCLUDES = -I.. -I$(srcdir)/.. \ libcamel_la_SOURCES = \ broken-date-parser.c \ camel-address.c \ + camel-block-file.c \ camel-cipher-context.c \ camel-cms-context.c \ camel-data-cache.c \ @@ -39,6 +40,7 @@ libcamel_la_SOURCES = \ camel-folder.c \ camel-html-parser.c \ camel-http-stream.c \ + camel-index.c \ camel-internet-address.c \ camel-lock.c \ camel-lock-client.c \ @@ -64,6 +66,7 @@ libcamel_la_SOURCES = \ camel-multipart.c \ camel-object.c \ camel-operation.c \ + camel-partition-table.c \ camel-pgp-context.c \ camel-pgp-mime.c \ camel-smime-context.c \ @@ -92,6 +95,7 @@ libcamel_la_SOURCES = \ camel-stream-mem.c \ camel-stream-null.c \ camel-stream.c \ + camel-text-index.c \ camel-tcp-stream-raw.c \ camel-tcp-stream-ssl.c \ camel-tcp-stream-openssl.c \ @@ -111,6 +115,7 @@ libcamel_la_SOURCES = \ libcamelinclude_HEADERS = \ broken-date-parser.h \ camel-address.h \ + camel-block-file.h \ camel-charset-map.h \ camel-cipher-context.h \ camel-cms-context.h \ @@ -131,6 +136,7 @@ libcamelinclude_HEADERS = \ camel-folder-thread.h \ camel-folder.h \ camel-http-stream.h \ + camel-index.h \ camel-internet-address.h \ camel-lock.h \ camel-lock-client.h \ @@ -156,6 +162,7 @@ libcamelinclude_HEADERS = \ camel-multipart.h \ camel-object.h \ camel-operation.h \ + camel-partition-table.h \ camel-pgp-context.h \ camel-pgp-mime.h \ camel-smime-context.h \ @@ -183,6 +190,7 @@ libcamelinclude_HEADERS = \ camel-stream-mem.h \ camel-stream-null.h \ camel-stream.h \ + camel-text-index.h \ camel-tcp-stream-raw.h \ camel-tcp-stream-ssl.h \ camel-tcp-stream.h \ @@ -214,6 +222,14 @@ camel_lock_helper_SOURCES = \ camel-lock.c \ camel-lock.h +camel_index_control_SOURCES = \ + camel-index-control.c + +camel_index_control_LDADD = \ + $(libcamel_la_LIBADD) \ + libcamel.la \ + $(EVOLUTION_MAIL_LIBS) + install-exec-hook: @if test -n "$(CAMEL_LOCK_HELPER_USER)"; then \ if test `whoami` = root ; then \ diff --git a/camel/camel-block-file.c b/camel/camel-block-file.c new file mode 100644 index 0000000000..0340f9ecdf --- /dev/null +++ b/camel/camel-block-file.c @@ -0,0 +1,1124 @@ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "e-util/e-msgport.h" + +#include "camel-block-file.h" + +#ifdef ENABLE_THREADS +#include +#endif + +#define d(x) /*(printf("%s(%d):%s: ", __FILE__, __LINE__, __PRETTY_FUNCTION__),(x))*/ + +#ifdef ENABLE_THREADS + +/* Locks must be obtained in the order defined */ + +struct _CamelBlockFilePrivate { + /* We use the private structure to form our lru list from */ + struct _CamelBlockFilePrivate *next; + struct _CamelBlockFilePrivate *prev; + + struct _CamelBlockFile *base; + + pthread_mutex_t root_lock; /* for modifying the root block */ + pthread_mutex_t cache_lock; /* for refcounting, flag manip, cache manip */ + pthread_mutex_t io_lock; /* for all io ops */ +}; + +#define CAMEL_BLOCK_FILE_LOCK(kf, lock) (pthread_mutex_lock(&(kf)->priv->lock)) +#define CAMEL_BLOCK_FILE_TRYLOCK(kf, lock) (pthread_mutex_trylock(&(kf)->priv->lock)) +#define CAMEL_BLOCK_FILE_UNLOCK(kf, lock) (pthread_mutex_unlock(&(kf)->priv->lock)) + +#define LOCK(x) pthread_mutex_lock(&x) +#define UNLOCK(x) pthread_mutex_unlock(&x) + +static pthread_mutex_t block_file_lock = PTHREAD_MUTEX_INITIALIZER; + +#else +#define CAMEL_BLOCK_FILE_LOCK(kf, lock) +#define CAMEL_BLOCK_FILE_TRYLOCK(kf, lock) +#define CAMEL_BLOCK_FILE_UNLOCK(kf, lock) +#define LOCK(x) +#define UNLOCK(x) +#endif + +/* lru cache of block files */ +static EDList block_file_list = E_DLIST_INITIALISER(block_file_list); +/* list to store block files that are actually intialised */ +static EDList block_file_active_list = E_DLIST_INITIALISER(block_file_active_list); +static int block_file_count = 0; +static int block_file_threshhold = 10; + +#define CBF_CLASS(o) ((CamelBlockFileClass *)(((CamelObject *)o)->classfuncs)) + +static int sync_nolock(CamelBlockFile *bs); +static int sync_block_nolock(CamelBlockFile *bs, CamelBlock *bl); + +static int +block_file_validate_root(CamelBlockFile *bs) +{ + struct stat st; + CamelBlockRoot *br; + + br = bs->root; + + fstat(bs->fd, &st); + + d(printf("Validate root:\n")); + d(printf("version: %.8s (%.8s)\n", bs->root->version, bs->version)); + d(printf("block size: %d (%d)\n", br->block_size, bs->block_size)); + d(printf("free: %d (%d add size < %d)\n", br->free, br->free / bs->block_size * bs->block_size, (int)st.st_size)); + d(printf("last: %d (%d and size: %d)\n", br->free, br->free / bs->block_size * bs->block_size, (int)st.st_size)); + + if (br->last == 0 + || memcmp(bs->root->version, bs->version, 8) != 0 + || br->block_size != bs->block_size + || (br->free % bs->block_size) != 0 + || (br->last % bs->block_size) != 0 + || fstat(bs->fd, &st) == -1 + || st.st_size != br->last + || br->free > st.st_size + || (br->flags & CAMEL_BLOCK_FILE_SYNC) == 0) { + return -1; + } + + return 0; +} + +static int +block_file_init_root(CamelBlockFile *bs) +{ + CamelBlockRoot *br = bs->root; + + memset(br, 0, bs->block_size); + memcpy(br->version, bs->version, 8); + br->last = bs->block_size; + br->flags = CAMEL_BLOCK_FILE_SYNC; + br->free = 0; + br->block_size = bs->block_size; + + return 0; +} + +static void +camel_block_file_class_init(CamelBlockFileClass *klass) +{ + klass->validate_root = block_file_validate_root; + klass->init_root = block_file_init_root; +} + +static guint +block_hash_func(const void *v) +{ + return ((camel_block_t)v) >> CAMEL_BLOCK_SIZE_BITS; +} + +static void +camel_block_file_init(CamelBlockFile *bs) +{ + struct _CamelBlockFilePrivate *p; + + bs->fd = -1; + bs->block_size = CAMEL_BLOCK_SIZE; + e_dlist_init(&bs->block_cache); + bs->blocks = g_hash_table_new((GHashFunc)block_hash_func, NULL); + /* this cache size and the text index size have been tuned for about the best + with moderate memory usage. Doubling the memory usage barely affects performance. */ + bs->block_cache_limit = 256; + + p = bs->priv = g_malloc0(sizeof(*bs->priv)); + p->base = bs; + +#ifdef ENABLE_THREADS + pthread_mutex_init(&p->root_lock, NULL); + pthread_mutex_init(&p->cache_lock, NULL); + pthread_mutex_init(&p->io_lock, NULL); +#endif + + /* link into lru list */ + LOCK(block_file_lock); + e_dlist_addhead(&block_file_list, (EDListNode *)p); + +#if 0 + { + printf("dumping block list\n"); + printf(" head = %p p = %p\n", block_file_list.head, p); + p = block_file_list.head; + while (p->next) { + printf(" '%s'\n", p->base->path); + p = p->next; + } + } +#endif + + UNLOCK(block_file_lock); +} + +static void +camel_block_file_finalise(CamelBlockFile *bs) +{ + CamelBlock *bl, *bn; + struct _CamelBlockFilePrivate *p; + + p = bs->priv; + + if (bs->root_block) + camel_block_file_sync(bs); + + /* remove from lru list */ + LOCK(block_file_lock); + if (bs->fd != -1) + block_file_count--; + e_dlist_remove((EDListNode *)p); + UNLOCK(block_file_lock); + + bl = (CamelBlock *)bs->block_cache.head; + bn = bl->next; + while (bn) { + if (bl->refcount != 0) + g_warning("Block '%d' still referenced", bl->id); + g_free(bl); + bl = bn; + bn = bn->next; + } + + if (bs->root_block) + camel_block_file_unref_block(bs, bs->root_block); + g_free(bs->path); + close(bs->fd); + +#ifdef ENABLE_THREADS + pthread_mutex_destroy(&p->io_lock); + pthread_mutex_destroy(&p->cache_lock); + pthread_mutex_destroy(&p->root_lock); +#endif + g_free(p); +} + +CamelType +camel_block_file_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelBlockFile", + sizeof (CamelBlockFile), + sizeof (CamelBlockFileClass), + (CamelObjectClassInitFunc) camel_block_file_class_init, + NULL, + (CamelObjectInitFunc) camel_block_file_init, + (CamelObjectFinalizeFunc) camel_block_file_finalise); + } + + return type; +} + +/* 'use' a block file for io */ +static int +block_file_use(CamelBlockFile *bs) +{ + struct _CamelBlockFilePrivate *nw, *nn, *p = bs->priv; + CamelBlockFile *bf; + int err; + + /* We want to: + remove file from active list + lock it + + Then when done: + unlock it + add it back to end of active list + */ + + CAMEL_BLOCK_FILE_LOCK(bs, io_lock); + + if (bs->fd != -1) + return 0; + else + d(printf("Turning block file online: %s\n", bs->path)); + + if ((bs->fd = open(bs->path, bs->flags, 0600)) == -1) { + err = errno; + CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock); + errno = err; + return -1; + } + + LOCK(block_file_lock); + e_dlist_remove((EDListNode *)p); + e_dlist_addtail(&block_file_active_list, (EDListNode *)p); + + block_file_count++; + + nw = (struct _CamelBlockFilePrivate *)block_file_list.head; + nn = nw->next; + while (block_file_count > block_file_threshhold && nn) { + /* We never hit the current blockfile here, as its removed from the list first */ + bf = nw->base; + if (bf->fd != -1) { + /* Need to trylock, as any of these lock levels might be trying + to lock the block_file_lock, so we need to check and abort if so */ + if (CAMEL_BLOCK_FILE_TRYLOCK(bf, root_lock) == 0) { + if (CAMEL_BLOCK_FILE_TRYLOCK(bf, cache_lock) == 0) { + if (CAMEL_BLOCK_FILE_TRYLOCK(bf, io_lock) == 0) { + d(printf("[%d] Turning block file offline: %s\n", block_file_count-1, bf->path)); + sync_nolock(bf); + close(bf->fd); + bf->fd = -1; + block_file_count--; + CAMEL_BLOCK_FILE_UNLOCK(bf, io_lock); + } + CAMEL_BLOCK_FILE_UNLOCK(bf, cache_lock); + } + CAMEL_BLOCK_FILE_UNLOCK(bf, root_lock); + } + } + nw = nn; + nn = nw->next; + } + + UNLOCK(block_file_lock); + + return 0; +} + +static void +block_file_unuse(CamelBlockFile *bs) +{ + LOCK(block_file_lock); + e_dlist_remove((EDListNode *)bs->priv); + e_dlist_addtail(&block_file_list, (EDListNode *)bs->priv); + UNLOCK(block_file_lock); + + CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock); +} + +/* +o = camel_cache_get(c, key); +camel_cache_unref(c, key); +camel_cache_add(c, key, o); +camel_cache_remove(c, key); +*/ + +/** + * camel_block_file_new: + * @path: + * @: + * @block_size: + * + * Allocate a new block file, stored at @path. @version contains an 8 character + * version string which must match the head of the file, or the file will be + * intitialised. + * + * @block_size is currently ignored and is set to CAMEL_BLOCK_SIZE. + * + * Return value: The new block file, or NULL if it could not be created. + **/ +CamelBlockFile *camel_block_file_new(const char *path, int flags, const char version[8], size_t block_size) +{ + CamelBlockFile *bs; + + bs = (CamelBlockFile *)camel_object_new(camel_block_file_get_type()); + memcpy(bs->version, version, 8); + bs->path = g_strdup(path); + bs->flags = flags; + + bs->root_block = camel_block_file_get_block(bs, 0); + if (bs->root_block == NULL) { + camel_object_unref((CamelObject *)bs); + return NULL; + } + camel_block_file_detach_block(bs, bs->root_block); + bs->root = (CamelBlockRoot *)&bs->root_block->data; + + /* we only need these flags on first open */ + bs->flags &= ~(O_CREAT|O_EXCL|O_TRUNC); + + /* Do we need to init the root block? */ + if (CBF_CLASS(bs)->validate_root(bs) == -1) { + d(printf("Initialise root block: %.8s\n", version)); + + CBF_CLASS(bs)->init_root(bs); + camel_block_file_touch_block(bs, bs->root_block); + if (block_file_use(bs) == -1) { + camel_object_unref((CamelObject *)bs); + return NULL; + } + if (sync_block_nolock(bs, bs->root_block) == -1 + || ftruncate(bs->fd, bs->root->last) == -1) { + block_file_unuse(bs); + camel_object_unref((CamelObject *)bs); + return NULL; + } + block_file_unuse(bs); + } + + return bs; +} + +int +camel_block_file_rename(CamelBlockFile *bs, const char *path) +{ + int ret; + struct stat st; + int err; + + CAMEL_BLOCK_FILE_LOCK(bs, io_lock); + + ret = rename(bs->path, path); + if (ret == -1) { + /* Maybe the rename actually worked */ + err = errno; + if (stat(path, &st) == 0 + && stat(bs->path, &st) == -1 + && errno == ENOENT) + ret = 0; + errno = err; + } + + if (ret != -1) { + g_free(bs->path); + bs->path = g_strdup(path); + } + + CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock); + + return ret; +} + +/** + * camel_block_file_new_block: + * @bs: + * + * Allocate a new block, return a pointer to it. Old blocks + * may be flushed to disk during this call. + * + * Return value: The block, or NULL if an error occured. + **/ +CamelBlock *camel_block_file_new_block(CamelBlockFile *bs) +{ + CamelBlock *bl; + + CAMEL_BLOCK_FILE_LOCK(bs, root_lock); + + if (bs->root->free) { + bl = camel_block_file_get_block(bs, bs->root->free); + if (bl == NULL) + goto fail; + bs->root->free = ((camel_block_t *)bl->data)[0]; + } else { + bl = camel_block_file_get_block(bs, bs->root->last); + if (bl == NULL) + goto fail; + bs->root->last += CAMEL_BLOCK_SIZE; + } + + bs->root_block->flags |= CAMEL_BLOCK_DIRTY; + + bl->flags |= CAMEL_BLOCK_DIRTY; + memset(bl->data, 0, CAMEL_BLOCK_SIZE); +fail: + CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock); + + return bl; +} + +/** + * camel_block_file_free_block: + * @bs: + * @id: + * + * + **/ +int camel_block_file_free_block(CamelBlockFile *bs, camel_block_t id) +{ + CamelBlock *bl; + + bl = camel_block_file_get_block(bs, id); + if (bl == NULL) + return -1; + + CAMEL_BLOCK_FILE_LOCK(bs, root_lock); + + ((camel_block_t *)bl->data)[0] = bs->root->free; + bs->root->free = bl->id; + bl->flags |= CAMEL_BLOCK_DIRTY; + camel_block_file_unref_block(bs, bl); + + CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock); + + return 0; +} + +/** + * camel_block_file_get_block: + * @bs: + * @id: + * + * Retreive a block @id. + * + * Return value: The block, or NULL if blockid is invalid or a file error + * occured. + **/ +CamelBlock *camel_block_file_get_block(CamelBlockFile *bs, camel_block_t id) +{ + CamelBlock *bl, *flush, *prev; + + /* Sanity check: Dont allow reading of root block (except before its been read) + or blocks with invalid block id's */ + if ((bs->root == NULL && id != 0) + || (bs->root != NULL && (id > bs->root->last || id == 0)) + || (id % bs->block_size) != 0) { + errno = EINVAL; + return NULL; + } + + CAMEL_BLOCK_FILE_LOCK(bs, cache_lock); + + bl = g_hash_table_lookup(bs->blocks, (void *)id); + + d(printf("Get block %08x: %s\n", id, bl?"cached":"must read")); + + if (bl == NULL) { + /* LOCK io_lock */ + if (block_file_use(bs) == -1) { + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); + return NULL; + } + + bl = g_malloc0(sizeof(*bl)); + bl->id = id; + if (lseek(bs->fd, id, SEEK_SET) == -1 + || read(bs->fd, bl->data, CAMEL_BLOCK_SIZE) == -1) { + block_file_unuse(bs); + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); + g_free(bl); + return NULL; + } + + bs->block_cache_count++; + g_hash_table_insert(bs->blocks, (void *)bl->id, bl); + + /* flush old blocks */ + flush = (CamelBlock *)bs->block_cache.tailpred; + prev = flush->prev; + while (bs->block_cache_count > bs->block_cache_limit && prev) { + if (flush->refcount == 0) { + if (sync_block_nolock(bs, flush) != -1) { + g_hash_table_remove(bs->blocks, (void *)flush->id); + e_dlist_remove((EDListNode *)flush); + g_free(flush); + bs->block_cache_count--; + } + } + flush = prev; + prev = prev->prev; + } + /* UNLOCK io_lock */ + block_file_unuse(bs); + } else { + e_dlist_remove((EDListNode *)bl); + } + + e_dlist_addhead(&bs->block_cache, (EDListNode *)bl); + bl->refcount++; + + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); + + d(printf("Got block %08x\n", id)); + + return bl; +} + +/** + * camel_block_file_detach_block: + * @bs: + * @bl: + * + * Detatch a block from the block file's cache. The block should + * be unref'd or attached when finished with. The block file will + * perform no writes of this block or flushing of it if the cache + * fills. + **/ +void camel_block_file_detach_block(CamelBlockFile *bs, CamelBlock *bl) +{ + CAMEL_BLOCK_FILE_LOCK(bs, cache_lock); + + g_hash_table_remove(bs->blocks, (void *)bl->id); + e_dlist_remove((EDListNode *)bl); + bl->flags |= CAMEL_BLOCK_DETACHED; + + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); +} + +/** + * camel_block_file_attach_block: + * @bs: + * @bl: + * + * Reattach a block that has been detached. + **/ +void camel_block_file_attach_block(CamelBlockFile *bs, CamelBlock *bl) +{ + CAMEL_BLOCK_FILE_LOCK(bs, cache_lock); + + g_hash_table_insert(bs->blocks, (void *)bl->id, bl); + e_dlist_addtail(&bs->block_cache, (EDListNode *)bl); + bl->flags &= ~CAMEL_BLOCK_DETACHED; + + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); +} + +/** + * camel_block_file_touch_block: + * @bs: + * @bl: + * + * Mark a block as dirty. The block will be written to disk if + * it ever expires from the cache. + **/ +void camel_block_file_touch_block(CamelBlockFile *bs, CamelBlock *bl) +{ + CAMEL_BLOCK_FILE_LOCK(bs, root_lock); + CAMEL_BLOCK_FILE_LOCK(bs, cache_lock); + + bl->flags |= CAMEL_BLOCK_DIRTY; + + if ((bs->root->flags & CAMEL_BLOCK_FILE_SYNC) && bl != bs->root_block) { + d(printf("turning off sync flag\n")); + bs->root->flags &= ~CAMEL_BLOCK_FILE_SYNC; + bs->root_block->flags |= CAMEL_BLOCK_DIRTY; + camel_block_file_sync_block(bs, bs->root_block); + } + + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); + CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock); +} + +/** + * camel_block_file_unref_block: + * @bs: + * @bl: + * + * Mark a block as unused. If a block is used it will not be + * written to disk, or flushed from memory. + * + * If a block is detatched and this is the last reference, the + * block will be freed. + **/ +void camel_block_file_unref_block(CamelBlockFile *bs, CamelBlock *bl) +{ + CAMEL_BLOCK_FILE_LOCK(bs, cache_lock); + + if (bl->refcount == 1 && (bl->flags & CAMEL_BLOCK_DETACHED)) + g_free(bl); + else + bl->refcount--; + + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); +} + +static int +sync_block_nolock(CamelBlockFile *bs, CamelBlock *bl) +{ + d(printf("Sync block %08x: %s\n", bl->id, (bl->flags & CAMEL_BLOCK_DIRTY)?"dirty":"clean")); + + if (bl->flags & CAMEL_BLOCK_DIRTY) { + if (lseek(bs->fd, bl->id, SEEK_SET) == -1 + || write(bs->fd, bl->data, CAMEL_BLOCK_SIZE) != CAMEL_BLOCK_SIZE) { + return -1; + } + bl->flags &= ~CAMEL_BLOCK_DIRTY; + } + + return 0; +} + +static int +sync_nolock(CamelBlockFile *bs) +{ + CamelBlock *bl, *bn; + int work = FALSE; + + bl = (CamelBlock *)bs->block_cache.head; + bn = bl->next; + while (bn) { + if (bl->flags & CAMEL_BLOCK_DIRTY) { + work = TRUE; + if (sync_block_nolock(bs, bl) == -1) + return -1; + bl = bn; + } + bn = bn->next; + } + + if (!work + && (bs->root_block->flags & CAMEL_BLOCK_DIRTY) == 0 + && (bs->root->flags & CAMEL_BLOCK_FILE_SYNC) != 0) + return 0; + + d(printf("turning on sync flag\n")); + + bs->root->flags |= CAMEL_BLOCK_FILE_SYNC; + + return sync_block_nolock(bs, bs->root_block); +} + +/** + * camel_block_file_sync_block: + * @bs: + * @bl: + * + * Flush a block to disk immediately. The block will only + * be flushed to disk if it is marked as dirty (touched). + * + * Return value: -1 on io error. + **/ +int camel_block_file_sync_block(CamelBlockFile *bs, CamelBlock *bl) +{ + int ret; + + /* LOCK io_lock */ + if (block_file_use(bs) == -1) + return -1; + + ret = sync_block_nolock(bs, bl); + + block_file_unuse(bs); + + return ret; +} + +/** + * camel_block_file_sync: + * @bs: + * + * Sync all dirty blocks to disk, including the root block. + * + * Return value: -1 on io error. + **/ +int camel_block_file_sync(CamelBlockFile *bs) +{ + int ret; + + CAMEL_BLOCK_FILE_LOCK(bs, root_lock); + CAMEL_BLOCK_FILE_LOCK(bs, cache_lock); + + /* LOCK io_lock */ + if (block_file_use(bs) == -1) + ret = -1; + else { + ret = sync_nolock(bs); + block_file_unuse(bs); + } + + CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock); + CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock); + + return ret; +} + +/* ********************************************************************** */ + +struct _CamelKeyFilePrivate { + struct _CamelKeyFilePrivate *next; + struct _CamelKeyFilePrivate *prev; + + struct _CamelKeyFile *base; + +#ifdef ENABLE_THREADS + pthread_mutex_t lock; +#endif +}; + +#ifdef ENABLE_THREADS +#define CAMEL_KEY_FILE_LOCK(kf, lock) (pthread_mutex_lock(&(kf)->priv->lock)) +#define CAMEL_KEY_FILE_TRYLOCK(kf, lock) (pthread_mutex_trylock(&(kf)->priv->lock)) +#define CAMEL_KEY_FILE_UNLOCK(kf, lock) (pthread_mutex_unlock(&(kf)->priv->lock)) + +static pthread_mutex_t key_file_lock = PTHREAD_MUTEX_INITIALIZER; + +#else +#define CAMEL_KEY_FILE_LOCK(kf, lock) +#define CAMEL_KEY_FILE_TRYLOCK(kf, lock) +#define CAMEL_KEY_FILE_UNLOCK(kf, lock) +#endif + +/* lru cache of block files */ +static EDList key_file_list = E_DLIST_INITIALISER(key_file_list); +static EDList key_file_active_list = E_DLIST_INITIALISER(key_file_active_list); +static int key_file_count = 0; +static int key_file_threshhold = 10; + +static void +camel_key_file_class_init(CamelKeyFileClass *klass) +{ +} + +static void +camel_key_file_init(CamelKeyFile *bs) +{ + struct _CamelKeyFilePrivate *p; + + p = bs->priv = g_malloc0(sizeof(*bs->priv)); + p->base = bs; + +#ifdef ENABLE_THREADS + pthread_mutex_init(&p->lock, NULL); +#endif + + LOCK(key_file_lock); + e_dlist_addhead(&key_file_list, (EDListNode *)p); + UNLOCK(key_file_lock); +} + +static void +camel_key_file_finalise(CamelKeyFile *bs) +{ + struct _CamelKeyFilePrivate *p = bs->priv; + + LOCK(key_file_lock); + e_dlist_remove((EDListNode *)p); + UNLOCK(key_file_lock); + + if (bs->fp) + fclose(bs->fp); + g_free(bs->path); + +#ifdef ENABLE_THREADS + pthread_mutex_destroy(&p->lock); +#endif + + g_free(p); +} + +CamelType +camel_key_file_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelKeyFile", + sizeof (CamelKeyFile), + sizeof (CamelKeyFileClass), + (CamelObjectClassInitFunc) camel_key_file_class_init, + NULL, + (CamelObjectInitFunc) camel_key_file_init, + (CamelObjectFinalizeFunc) camel_key_file_finalise); + } + + return type; +} + +/* 'use' a key file for io */ +static int +key_file_use(CamelKeyFile *bs) +{ + struct _CamelKeyFilePrivate *nw, *nn, *p = bs->priv; + CamelKeyFile *bf; + int err, fd; + char *flag; + + /* We want to: + remove file from active list + lock it + + Then when done: + unlock it + add it back to end of active list + */ + + /* TODO: Check header on reset? */ + + CAMEL_KEY_FILE_LOCK(bs, lock); + + if (bs->fp != NULL) + return 0; + else + d(printf("Turning key file online: '%s'\n", bs->path)); + + if ((bs->flags & O_ACCMODE) == O_RDONLY) + flag = "r"; + else + flag = "a+"; + + if ((fd = open(bs->path, bs->flags, 0600)) == -1 + || (bs->fp = fdopen(fd, flag)) == NULL) { + err = errno; + close(fd); + CAMEL_KEY_FILE_UNLOCK(bs, lock); + errno = err; + return -1; + } + + LOCK(key_file_lock); + e_dlist_remove((EDListNode *)p); + e_dlist_addtail(&key_file_active_list, (EDListNode *)p); + + key_file_count++; + + nw = (struct _CamelKeyFilePrivate *)key_file_list.head; + nn = nw->next; + while (key_file_count > key_file_threshhold && nn) { + /* We never hit the current keyfile here, as its removed from the list first */ + bf = nw->base; + if (bf->fp != NULL) { + /* Need to trylock, as any of these lock levels might be trying + to lock the key_file_lock, so we need to check and abort if so */ + if (CAMEL_BLOCK_FILE_TRYLOCK(bf, lock) == 0) { + d(printf("Turning key file offline: %s\n", bf->path)); + fclose(bf->fp); + bf->fp = NULL; + key_file_count--; + CAMEL_BLOCK_FILE_UNLOCK(bf, lock); + } + } + nw = nn; + nn = nw->next; + } + + UNLOCK(key_file_lock); + + return 0; +} + +static void +key_file_unuse(CamelKeyFile *bs) +{ + LOCK(key_file_lock); + e_dlist_remove((EDListNode *)bs->priv); + e_dlist_addtail(&key_file_list, (EDListNode *)bs->priv); + UNLOCK(key_file_lock); + + CAMEL_KEY_FILE_UNLOCK(bs, lock); +} + +/** + * camel_key_file_new: + * @path: + * @flags: open flags + * @version[]: Version string (header) of file. Currently + * written but not checked. + * + * Create a new key file. A linked list of record blocks. + * + * Return value: A new key file, or NULL if the file could not + * be opened/created/initialised. + **/ +CamelKeyFile * +camel_key_file_new(const char *path, int flags, const char version[8]) +{ + CamelKeyFile *kf; + off_t last; + int err; + + d(printf("New key file '%s'\n", path)); + + kf = (CamelKeyFile *)camel_object_new(camel_key_file_get_type()); + kf->path = g_strdup(path); + kf->fp = NULL; + kf->flags = flags; + kf->last = 8; + + if (key_file_use(kf) == -1) { + camel_object_unref((CamelObject *)kf); + kf = NULL; + } else { + fseek(kf->fp, 0, SEEK_END); + last = ftell(kf->fp); + if (last == 0) { + fwrite(version, 8, 1, kf->fp); + last += 8; + } + kf->last = last; + + err = ferror(kf->fp); + key_file_unuse(kf); + + /* we only need these flags on first open */ + kf->flags &= ~(O_CREAT|O_EXCL|O_TRUNC); + + if (err) { + camel_object_unref((CamelObject *)kf); + kf = NULL; + } + } + + return kf; +} + +int +camel_key_file_rename(CamelKeyFile *kf, const char *path) +{ + int ret; + struct stat st; + int err; + + CAMEL_KEY_FILE_LOCK(kf, lock); + + ret = rename(kf->path, path); + if (ret == -1) { + /* Maybe the rename actually worked */ + err = errno; + if (stat(path, &st) == 0 + && stat(kf->path, &st) == -1 + && errno == ENOENT) + ret = 0; + errno = err; + } + + if (ret != -1) { + g_free(kf->path); + kf->path = g_strdup(path); + } + + CAMEL_KEY_FILE_UNLOCK(kf, lock); + + return ret; +} + +/** + * camel_key_file_write: + * @kf: + * @parent: + * @len: + * @records: + * + * Write a new list of records to the key file. + * + * Return value: -1 on io error. The key file will remain unchanged. + **/ +int +camel_key_file_write(CamelKeyFile *kf, camel_block_t *parent, size_t len, camel_key_t *records) +{ + camel_block_t next; + guint32 size; + int ret = -1; + + d(printf("write key %08x len = %d\n", *parent, len)); + + if (len == 0) { + d(printf(" new parent = %08x\n", *parent)); + return 0; + } + + /* LOCK */ + if (key_file_use(kf) == -1) + return -1; + + size = len; + + /* FIXME: Use io util functions? */ + next = kf->last; + fseek(kf->fp, kf->last, SEEK_SET); + fwrite(parent, sizeof(*parent), 1, kf->fp); + fwrite(&size, sizeof(size), 1, kf->fp); + fwrite(records, sizeof(records[0]), len, kf->fp); + + if (ferror(kf->fp)) { + clearerr(kf->fp); + } else { + kf->last = ftell(kf->fp); + *parent = next; + ret = len; + } + + /* UNLOCK */ + key_file_unuse(kf); + + d(printf(" new parent = %08x\n", *parent)); + + return ret; +} + +/** + * camel_key_file_read: + * @kf: + * @start: The record pointer. This will be set to the next record pointer on success. + * @len: Number of records read, if != NULL. + * @records: Records, allocated, must be freed with g_free, if != NULL. + * + * Read the next block of data from the key file. Returns the number of + * records. + * + * Return value: -1 on io error. + **/ +int +camel_key_file_read(CamelKeyFile *kf, camel_block_t *start, size_t *len, camel_key_t **records) +{ + guint32 size; + long pos = *start; + camel_block_t next; + int ret = -1; + + if (pos == 0) + return 0; + + /* LOCK */ + if (key_file_use(kf) == -1) + return -1; + + if (fseek(kf->fp, pos, SEEK_SET) == -1 + || fread(&next, sizeof(next), 1, kf->fp) != 1 + || fread(&size, sizeof(size), 1, kf->fp) != 1 + || size > 1024) { + clearerr(kf->fp); + goto fail; + } + + if (len) + *len = size; + + if (records) { + camel_key_t *keys = g_malloc(size * sizeof(camel_key_t)); + + if (fread(keys, sizeof(camel_key_t), size, kf->fp) != size) { + g_free(keys); + goto fail; + } + *records = keys; + } + + *start = next; + + ret = 0; +fail: + /* UNLOCK */ + key_file_unuse(kf); + + return ret; +} diff --git a/camel/camel-block-file.h b/camel/camel-block-file.h new file mode 100644 index 0000000000..e78254e1ed --- /dev/null +++ b/camel/camel-block-file.h @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef _CAMEL_BLOCK_FILE_H +#define _CAMEL_BLOCK_FILE_H + +#include +#include +#include +#include + +typedef guint32 camel_block_t; /* block offset, absolute, bottom BLOCK_SIZE_BITS always 0 */ +typedef guint32 camel_key_t; /* this is a bitfield of (block offset:BLOCK_SIZE_BITS) */ + +typedef struct _CamelBlockRoot CamelBlockRoot; +typedef struct _CamelBlock CamelBlock; +typedef struct _CamelBlockFile CamelBlockFile; +typedef struct _CamelBlockFileClass CamelBlockFileClass; + +#define CAMEL_BLOCK_FILE_SYNC (1<<0) + +#define CAMEL_BLOCK_SIZE (1024) +#define CAMEL_BLOCK_SIZE_BITS (10) /* # bits to contain block_size bytes */ + +#define CAMEL_BLOCK_DIRTY (1<<0) +#define CAMEL_BLOCK_DETACHED (1<<1) + +struct _CamelBlockRoot { + char version[8]; /* version number */ + + guint32 flags; /* flags for file */ + guint32 block_size; /* block size of this file */ + camel_block_t free; /* free block list */ + camel_block_t last; /* pointer to end of blocks */ + + /* subclasses tack on, but no more than CAMEL_BLOCK_SIZE! */ +}; + +/* LRU cache of blocks */ +struct _CamelBlock { + struct _CamelBlock *next; + struct _CamelBlock *prev; + + camel_block_t id; + guint32 flags; + guint32 refcount; + guint32 align00; + + unsigned char data[CAMEL_BLOCK_SIZE]; +}; + +struct _CamelBlockFile { + CamelObject parent; + + struct _CamelBlockFilePrivate *priv; + + char version[8]; + char *path; + int flags; + + int fd; + size_t block_size; + + CamelBlockRoot *root; + CamelBlock *root_block; + + /* make private? */ + int block_cache_limit; + int block_cache_count; + EDList block_cache; + GHashTable *blocks; +}; + +struct _CamelBlockFileClass { + CamelObjectClass parent; + + int (*validate_root)(CamelBlockFile *); + int (*init_root)(CamelBlockFile *); +}; + +CamelType camel_block_file_get_type(void); + +CamelBlockFile *camel_block_file_new(const char *path, int flags, const char version[8], size_t block_size); +int camel_block_file_rename(CamelBlockFile *bs, const char *path); + +CamelBlock *camel_block_file_new_block(CamelBlockFile *bs); +int camel_block_file_free_block(CamelBlockFile *bs, camel_block_t id); +CamelBlock *camel_block_file_get_block(CamelBlockFile *bs, camel_block_t id); +void camel_block_file_detach_block(CamelBlockFile *bs, CamelBlock *bl); +void camel_block_file_attach_block(CamelBlockFile *bs, CamelBlock *bl); +void camel_block_file_touch_block(CamelBlockFile *bs, CamelBlock *bl); +void camel_block_file_unref_block(CamelBlockFile *bs, CamelBlock *bl); +int camel_block_file_sync_block(CamelBlockFile *bs, CamelBlock *bl); +int camel_block_file_sync(CamelBlockFile *bs); + +/* ********************************************************************** */ + +typedef struct _CamelKeyFile CamelKeyFile; +typedef struct _CamelKeyFileClass CamelKeyFileClass; + +struct _CamelKeyFile { + CamelObject parent; + + struct _CamelKeyFilePrivate *priv; + + FILE *fp; + char *path; + int flags; + off_t last; +}; + +struct _CamelKeyFileClass { + CamelObjectClass parent; +}; + +CamelType camel_key_file_get_type(void); + +CamelKeyFile * camel_key_file_new(const char *path, int flags, const char version[8]); +int camel_key_file_rename(CamelKeyFile *kf, const char *path); + +int camel_key_file_write(CamelKeyFile *kf, camel_block_t *parent, size_t len, camel_key_t *records); +int camel_key_file_read(CamelKeyFile *kf, camel_block_t *start, size_t *len, camel_key_t **records); + + +#endif /* ! _CAMEL_BLOCK_FILE_H */ diff --git a/camel/camel-filter-driver.c b/camel/camel-filter-driver.c index d8b255da63..e696388586 100644 --- a/camel/camel-filter-driver.c +++ b/camel/camel-filter-driver.c @@ -24,14 +24,17 @@ #include #endif +#include +#include +#include +#include + #include #include #include #include -#include - #include "camel-filter-driver.h" #include "camel-filter-search.h" diff --git a/camel/camel-folder-search.c b/camel/camel-folder-search.c index 993de40801..e0e5052a5f 100644 --- a/camel/camel-folder-search.c +++ b/camel/camel-folder-search.c @@ -275,7 +275,14 @@ camel_folder_search_set_folder(CamelFolderSearch *search, CamelFolder *folder) void camel_folder_search_set_summary(CamelFolderSearch *search, GPtrArray *summary) { + int i; + search->summary = summary; + if (search->summary_hash) + g_hash_table_destroy(search->summary_hash); + search->summary_hash = g_hash_table_new(g_str_hash, g_str_equal); + for (i=0;ilen;i++) + g_hash_table_insert(search->summary_hash, (char *)camel_message_info_uid(summary->pdata[i]), summary->pdata[i]); } /** @@ -283,15 +290,19 @@ camel_folder_search_set_summary(CamelFolderSearch *search, GPtrArray *summary) * @search: * @index: * - * Set the index (ibex) representing the contents of all messages + * Set the index representing the contents of all messages * in this folder. If this is not set, then the folder implementation * should sub-class the CamelFolderSearch and provide its own * body-contains function. **/ void -camel_folder_search_set_body_index(CamelFolderSearch *search, ibex *index) +camel_folder_search_set_body_index(CamelFolderSearch *search, CamelIndex *index) { + if (search->body_index) + camel_object_unref((CamelObject *)search->body_index); search->body_index = index; + if (index) + camel_object_ref((CamelObject *)index); } /** @@ -728,12 +739,79 @@ match_message(CamelFolder *folder, const char *uid, regex_t *pattern, CamelExcep return truth; } +/* perform a regex match against words in an index */ +/* uids = hash table of messageinfo's by uid's */ +static GPtrArray * +match_messages_index(CamelIndex *idx, regex_t *pattern, GHashTable *uids, CamelException *ex) +{ + GPtrArray *result = g_ptr_array_new(); + GHashTable *ht = g_hash_table_new(g_str_hash, g_str_equal); + struct _glib_sux_donkeys lambdafoo; + CamelIndexCursor *wc, *nc; + const char *word, *name; + CamelMessageInfo *mi; + + wc = camel_index_words(idx); + if (wc) { + while ((word = camel_index_cursor_next(wc))) { + if (regexec(pattern, word, 0, NULL, 0) == 0) { + /* perf: could have the wc cursor return the name cursor */ + nc = camel_index_find(idx, word); + if (nc) { + while ((name = camel_index_cursor_next(nc))) { + mi = g_hash_table_lookup(uids, name); + if (mi) + g_hash_table_insert(ht, (char *)camel_message_info_uid(mi), (void *)1); + } + camel_object_unref((CamelObject *)nc); + } + } + } + camel_object_unref((CamelObject *)wc); + + lambdafoo.uids = result; + g_hash_table_foreach(ht, (GHFunc)g_lib_sux_htor, &lambdafoo); + g_hash_table_destroy(ht); + } + + return result; +} + +/* perform a regex match against an individual uid in an index */ +/* this would benefit greatly in practice if there was a hashtalbe of uid's to amtch against */ +static int +match_message_index(CamelIndex *idx, const char *uid, regex_t *pattern, CamelException *ex) +{ + CamelIndexCursor *wc, *nc; + const char *word, *name; + int truth = FALSE; + + wc = camel_index_words(idx); + if (wc) { + while (!truth && (word = camel_index_cursor_next(wc))) { + if (regexec(pattern, word, 0, NULL, 0) == 0) { + /* perf: could have the wc cursor return the name cursor */ + nc = camel_index_find(idx, word); + if (nc) { + while (!truth && (name = camel_index_cursor_next(nc))) + truth = strcmp(name, uid) == 0; + camel_object_unref((CamelObject *)nc); + } + } + } + camel_object_unref((CamelObject *)wc); + } + + return truth; +} + static ESExpResult * search_body_contains(struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFolderSearch *search) { ESExpResult *r; - int i, j; + int i; regex_t pattern; + CamelException *ex = search->priv->ex; if (search->current) { int truth = FALSE; @@ -741,19 +819,14 @@ search_body_contains(struct _ESExp *f, int argc, struct _ESExpResult **argv, Cam if (argc == 1 && argv[0]->value.string[0] == 0 && search->folder) { truth = TRUE; } else if (search->body_index) { - for (i=0;itype == ESEXP_RES_STRING) { - truth = ibex_find_name(search->body_index, (char *)camel_message_info_uid(search->current), - argv[i]->value.string); - } else { - e_sexp_resultv_free(f, argc, argv); - e_sexp_fatal_error(f, _("Invalid type in body-contains, expecting string")); - } + if (camel_search_build_match_regex(&pattern, CAMEL_SEARCH_MATCH_ICASE, argc, argv, ex) == 0) { + truth = match_message_index(search->body_index, camel_message_info_uid(search->current), &pattern, ex); + regfree(&pattern); } } else if (search->folder) { /* we do a 'slow' direct search */ - if (camel_search_build_match_regex(&pattern, CAMEL_SEARCH_MATCH_ICASE, argc, argv, search->priv->ex) == 0) { - truth = match_message(search->folder, camel_message_info_uid(search->current), &pattern, search->priv->ex); + if (camel_search_build_match_regex(&pattern, CAMEL_SEARCH_MATCH_ICASE, argc, argv, ex) == 0) { + truth = match_message(search->folder, camel_message_info_uid(search->current), &pattern, ex); regfree(&pattern); } } else { @@ -772,42 +845,19 @@ search_body_contains(struct _ESExp *f, int argc, struct _ESExpResult **argv, Cam g_ptr_array_add(r->value.ptrarray, (char *)camel_message_info_uid(info)); } } else if (search->body_index) { - if (argc==1) { - /* common case */ - r->value.ptrarray = ibex_find(search->body_index, argv[0]->value.string); - } else { - GHashTable *ht = g_hash_table_new(g_str_hash, g_str_equal); - GPtrArray *pa; - struct _glib_sux_donkeys lambdafoo; - - /* this sux, perform an or operation on the result(s) of each word */ - for (i=0;itype == ESEXP_RES_STRING) { - pa = ibex_find(search->body_index, argv[i]->value.string); - for (j=0;jlen;j++) { - g_hash_table_insert(ht, g_ptr_array_index(pa, j), (void *)1); - } - g_ptr_array_free(pa, FALSE); - } else { - e_sexp_result_free(f, r); - e_sexp_resultv_free(f, argc, argv); - e_sexp_fatal_error(f, _("Invalid type in body-contains, expecting string")); - } - } - lambdafoo.uids = g_ptr_array_new(); - g_hash_table_foreach(ht, (GHFunc)g_lib_sux_htor, &lambdafoo); - r->value.ptrarray = lambdafoo.uids; - g_hash_table_destroy(ht); + if (camel_search_build_match_regex(&pattern, CAMEL_SEARCH_MATCH_ICASE, argc, argv, ex) == 0) { + r->value.ptrarray = match_messages_index(search->body_index, &pattern, search->summary_hash, ex); + regfree(&pattern); } } else if (search->folder) { /* do a slow search */ r->value.ptrarray = g_ptr_array_new(); - if (camel_search_build_match_regex(&pattern, CAMEL_SEARCH_MATCH_ICASE, argc, argv, search->priv->ex) == 0) { + if (camel_search_build_match_regex(&pattern, CAMEL_SEARCH_MATCH_ICASE, argc, argv, ex) == 0) { if (search->summary) { for (i=0;isummary->len;i++) { CamelMessageInfo *info = g_ptr_array_index(search->summary, i); - if (match_message(search->folder, camel_message_info_uid(info), &pattern, search->priv->ex)) + if (match_message(search->folder, camel_message_info_uid(info), &pattern, ex)) g_ptr_array_add(r->value.ptrarray, (char *)camel_message_info_uid(info)); } } /* else? we could always get the summary from the folder, but then diff --git a/camel/camel-folder-search.h b/camel/camel-folder-search.h index 59f31d3628..3f29567aed 100644 --- a/camel/camel-folder-search.h +++ b/camel/camel-folder-search.h @@ -28,9 +28,9 @@ extern "C" { #endif /* __cplusplus */ #include -#include #include #include +#include #define CAMEL_FOLDER_SEARCH_TYPE (camel_folder_search_get_type ()) #define CAMEL_FOLDER_SEARCH(obj) CAMEL_CHECK_CAST (obj, camel_folder_search_get_type (), CamelFolderSearch) @@ -50,10 +50,11 @@ struct _CamelFolderSearch { /* these are only valid during the search, and are reset afterwards */ CamelFolder *folder; /* folder for current search */ GPtrArray *summary; /* summary array for current search */ + GHashTable *summary_hash; /* hashtable of summary items */ CamelMessageInfo *current; /* current message info, when searching one by one */ CamelMessageInfo *match1; /* message info, when searching a single message only */ CamelMimeMessage *current_message; /* cache of current message, if required */ - ibex *body_index; + CamelIndex *body_index; }; struct _CamelFolderSearchClass { @@ -117,7 +118,7 @@ void camel_folder_search_construct (CamelFolderSearch *search); void camel_folder_search_set_folder(CamelFolderSearch *search, CamelFolder *folder); void camel_folder_search_set_summary(CamelFolderSearch *search, GPtrArray *summary); -void camel_folder_search_set_body_index(CamelFolderSearch *search, ibex *index); +void camel_folder_search_set_body_index(CamelFolderSearch *search, CamelIndex *index); GPtrArray *camel_folder_search_execute_expression(CamelFolderSearch *search, const char *expr, CamelException *ex); gboolean camel_folder_search_match_expression(CamelFolderSearch *search, const char *expr, const CamelMessageInfo *info, CamelException *ex); diff --git a/camel/camel-folder-summary.c b/camel/camel-folder-summary.c index a424aef8de..c681ff6123 100644 --- a/camel/camel-folder-summary.c +++ b/camel/camel-folder-summary.c @@ -22,6 +22,10 @@ #include #endif +#include +#include +#include + #include #include #include @@ -42,6 +46,9 @@ #include #include +#include +#include + #include "hash-table-utils.h" #include "e-util/md5-utils.h" #include "e-util/e-memory.h" @@ -213,6 +220,11 @@ camel_folder_summary_finalize (CamelObject *obj) if (p->filter_html) camel_object_unref((CamelObject *)p->filter_html); + if (p->filter_stream) + camel_object_unref((CamelObject *)p->filter_stream); + if (p->index) + camel_object_unref((CamelObject *)p->index); + #ifdef ENABLE_THREADS g_mutex_free(p->summary_lock); g_mutex_free(p->io_lock); @@ -283,11 +295,16 @@ void camel_folder_summary_set_filename(CamelFolderSummary *s, const char *name) * * Unlike earlier behaviour, build_content need not be set to perform indexing. **/ -void camel_folder_summary_set_index(CamelFolderSummary *s, ibex *index) +void camel_folder_summary_set_index(CamelFolderSummary *s, CamelIndex *index) { struct _CamelFolderSummaryPrivate *p = _PRIVATE(s); + if (p->index) + camel_object_unref((CamelObject *)p->index); + p->index = index; + if (index) + camel_object_ref((CamelObject *)index); } /** @@ -853,6 +870,7 @@ CamelMessageInfo *camel_folder_summary_info_new_from_parser(CamelFolderSummary * int len; struct _CamelFolderSummaryPrivate *p = _PRIVATE(s); off_t start; + CamelIndexName *name = NULL; /* should this check the parser is in the right state, or assume it is?? */ @@ -871,15 +889,21 @@ CamelMessageInfo *camel_folder_summary_info_new_from_parser(CamelFolderSummary * if (p->index) { if (p->filter_index == NULL) - p->filter_index = camel_mime_filter_index_new_ibex(p->index); - camel_mime_filter_index_set_name(p->filter_index, (char *)camel_message_info_uid(info)); - ibex_unindex(p->index, (char *)camel_message_info_uid(info)); - ibex_index_buffer(p->index, (char *)camel_message_info_uid(info), "ibexindexed", strlen("ibexindexed"), NULL); + p->filter_index = camel_mime_filter_index_new_index(p->index); + camel_index_delete_name(p->index, camel_message_info_uid(info)); + name = camel_index_add_name(p->index, camel_message_info_uid(info)); + camel_mime_filter_index_set_name(p->filter_index, name); } /* always scan the content info, even if we dont save it */ info->content = summary_build_content_info(s, info, mp); + if (name) { + camel_index_write_name(p->index, name); + camel_object_unref((CamelObject *)name); + camel_mime_filter_index_set_name(p->filter_index, NULL); + } + CAMEL_SUMMARY_UNLOCK(s, filter_lock); info->size = camel_mime_parser_tell(mp) - start; @@ -900,6 +924,7 @@ CamelMessageInfo *camel_folder_summary_info_new_from_message(CamelFolderSummary { CamelMessageInfo *info; struct _CamelFolderSummaryPrivate *p = _PRIVATE(s); + CamelIndexName *name = NULL; info = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_new_from_message(s, msg); @@ -907,13 +932,33 @@ CamelMessageInfo *camel_folder_summary_info_new_from_message(CamelFolderSummary * know if we are going to store this in the summary, but no matter */ summary_assign_uid(s, info); + CAMEL_SUMMARY_LOCK(s, filter_lock); + if (p->index) { - ibex_unindex(p->index, (char *)camel_message_info_uid(info)); - ibex_index_buffer(p->index, (char *)camel_message_info_uid(info), "ibexindexed", strlen("ibexindexed"), NULL); + if (p->filter_index == NULL) + p->filter_index = camel_mime_filter_index_new_index(p->index); + camel_index_delete_name(p->index, camel_message_info_uid(info)); + name = camel_index_add_name(p->index, camel_message_info_uid(info)); + camel_mime_filter_index_set_name(p->filter_index, name); + + if (p->filter_stream == NULL) { + CamelStream *null = camel_stream_null_new(); + + p->filter_stream = camel_stream_filter_new_with_stream(null); + camel_object_unref((CamelObject *)null); + } } info->content = summary_build_content_info_message(s, info, (CamelMimePart *)msg); + if (name) { + camel_index_write_name(p->index, name); + camel_object_unref((CamelObject *)name); + camel_mime_filter_index_set_name(p->filter_index, NULL); + } + + CAMEL_SUMMARY_UNLOCK(s, filter_lock); + return info; } @@ -2000,7 +2045,7 @@ summary_build_content_info(CamelFolderSummary *s, CamelMessageInfo *msginfo, Cam } /* build the content-info, from a message */ -/* this needs no lock, as we copy all data, and ibex is threadsafe */ +/* this needs the filter lock since it uses filters to perform indexing */ static CamelMessageContentInfo * summary_build_content_info_message(CamelFolderSummary *s, CamelMessageInfo *msginfo, CamelMimePart *object) { @@ -2046,14 +2091,25 @@ summary_build_content_info_message(CamelFolderSummary *s, CamelMessageInfo *msgi child->parent = info; my_list_append((struct _node **)&info->childs, (struct _node *)child); } - } else if (p->index + } else if (p->filter_stream && header_content_type_is(CAMEL_DATA_WRAPPER(containee)->mime_type, "text", "*")) { - /* index all text parts if we're indexing */ - CamelStreamMem *mem = (CamelStreamMem *)camel_stream_mem_new(); + int html_id = -1, idx_id = -1; + + /* pre-attach html filter if required, otherwise just index filter */ + if (header_content_type_is(CAMEL_DATA_WRAPPER(containee)->mime_type, "text", "html")) { + if (p->filter_html == NULL) + p->filter_html = camel_mime_filter_html_new(); + else + camel_mime_filter_reset((CamelMimeFilter *)p->filter_html); + html_id = camel_stream_filter_add(p->filter_stream, (CamelMimeFilter *)p->filter_html); + } + idx_id = camel_stream_filter_add(p->filter_stream, (CamelMimeFilter *)p->filter_index); - camel_data_wrapper_write_to_stream(containee, (CamelStream *)mem); - ibex_index_buffer(p->index, (char *)camel_message_info_uid(msginfo), mem->buffer->data, mem->buffer->len, NULL); - camel_object_unref((CamelObject *)mem); + camel_data_wrapper_write_to_stream(containee, (CamelStream *)p->filter_stream); + camel_stream_flush((CamelStream *)p->filter_stream); + + camel_stream_filter_remove(p->filter_stream, idx_id); + camel_stream_filter_remove(p->filter_stream, html_id); } return info; @@ -2680,104 +2736,4 @@ message_info_dump(CamelMessageInfo *mi) printf("Flags: %04x\n", mi->flags & 0xffff); content_info_dump(mi->content, 0); } - -int main(int argc, char **argv) -{ - CamelMimeParser *mp; - int fd; - CamelFolderSummary *s; - char *buffer; - int len; - int i; - ibex *index; - - /*g_tk_init(&argc, &argv);*/ - -#if 0 - { - int i; - char *s; - char buf[1024]; - - for (i=0;i<434712;i++) { - memcpy(buf, " ", 50); - buf[50] = 0; -#if 0 - s = g_strdup(buf); - g_free(s); -#endif - } - return 0; - } -#endif - - if (argc < 2 ) { - printf("usage: %s mbox\n", argv[0]); - return 1; - } - - fd = open(argv[1], O_RDONLY); - - index = ibex_open("index.ibex", O_CREAT|O_RDWR, 0600); - - mp = camel_mime_parser_new(); - camel_mime_parser_scan_from(mp, TRUE); -/* camel_mime_parser_set_header_regex(mp, "^(content-[^:]*|subject|from|to|date):");*/ - camel_mime_parser_init_with_fd(mp, fd); - - s = camel_folder_summary_new(); - camel_folder_summary_set_build_content(s, TRUE); -/* camel_folder_summary_set_index(s, index);*/ - - while (camel_mime_parser_step(mp, &buffer, &len) == HSCAN_FROM) { - /*printf("Parsing message ...\n");*/ - camel_folder_summary_add_from_parser(s, mp); - if (camel_mime_parser_step(mp, &buffer, &len) != HSCAN_FROM_END) { - g_warning("Uknown state encountered, excpecting %d, got %d\n", HSCAN_FROM_END, camel_mime_parser_state(mp)); - break; - } - } - - printf("Printing summary\n"); - for (i=0;i #include #include -#include +#include #define CAMEL_FOLDER_SUMMARY(obj) CAMEL_CHECK_CAST (obj, camel_folder_summary_get_type (), CamelFolderSummary) #define CAMEL_FOLDER_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_folder_summary_get_type (), CamelFolderSummaryClass) @@ -213,7 +213,7 @@ guint camel_folder_summary_get_type (void); CamelFolderSummary *camel_folder_summary_new (void); void camel_folder_summary_set_filename(CamelFolderSummary *, const char *); -void camel_folder_summary_set_index(CamelFolderSummary *, ibex *); +void camel_folder_summary_set_index(CamelFolderSummary *, CamelIndex *); void camel_folder_summary_set_build_content(CamelFolderSummary *, gboolean state); guint32 camel_folder_summary_next_uid (CamelFolderSummary *s); diff --git a/camel/camel-index-control.c b/camel/camel-index-control.c new file mode 100644 index 0000000000..9968278446 --- /dev/null +++ b/camel/camel-index-control.c @@ -0,0 +1,212 @@ + +/* Command to manually force a compression/dump of an index file */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "e-util/e-msgport.h" +#include "e-util/e-memory.h" + +#include "camel/camel-object.h" + +#include "camel-text-index.h" + +#include + +static void +do_usage(char *argv0) +{ + fprintf(stderr, "Usage: %s [ compress | dump | info ] file(s) ...\n", argv0); + fprintf(stderr, " compress - compress (an) index file(s)\n"); + fprintf(stderr, " dump - dump (an) index file's content to stdout\n"); + fprintf(stderr, " info - dump summary info to stdout\n"); + exit(1); +} + +static int +do_compress(int argc, char **argv) +{ + int i; + CamelIndex *idx; + + for (i=2;i +#include +#include "camel-stream-null.h" +#include "camel-stream-filter.h" +#include "camel-mime-filter-index.h" +#include "camel-stream-fs.h" + +static int +do_perf(int argc, char **argv) +{ + CamelIndex *idx; + DIR *dir; + char *path = "/home/notzed/evolution/local/Inbox/mbox/cur"; + struct dirent *d; + CamelStream *null, *filter, *stream; + CamelMimeFilterIndex *filter_index; + char *name; + CamelIndexName *idn; + + dir = opendir(path); + if (dir == NULL) { + perror("open dir"); + return 1; + } + + idx = (CamelIndex *)camel_text_index_new("/tmp/index", O_TRUNC|O_CREAT|O_RDWR); + if (idx == NULL) { + perror("open index"); + return 1; + } + + null = camel_stream_null_new(); + filter = (CamelStream *)camel_stream_filter_new_with_stream(null); + camel_object_unref((CamelObject *)null); + filter_index = camel_mime_filter_index_new_index(idx); + camel_stream_filter_add((CamelStreamFilter *)filter, (CamelMimeFilter *)filter_index); + + while ((d = readdir(dir))) { + printf("indexing '%s'\n", d->d_name); + + idn = camel_index_add_name(idx, d->d_name); + camel_mime_filter_index_set_name(filter_index, idn); + name = g_strdup_printf("%s/%s", path, d->d_name); + stream = camel_stream_fs_new_with_name(name, O_RDONLY, 0); + camel_stream_write_to_stream(stream, filter); + camel_object_unref((CamelObject *)stream); + g_free(name); + + camel_index_write_name(idx, idn); + camel_object_unref((CamelObject *)idn); + camel_mime_filter_index_set_name(filter_index, NULL); + } + + closedir(dir); + + camel_index_sync(idx); + camel_object_unref((CamelObject *)idx); + + camel_object_unref((CamelObject *)filter); + camel_object_unref((CamelObject *)filter_index); + + return 0; +} + diff --git a/camel/camel-index.c b/camel/camel-index.c new file mode 100644 index 0000000000..cb8ebd120b --- /dev/null +++ b/camel/camel-index.c @@ -0,0 +1,341 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*- */ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "camel-index.h" +#include "camel/camel-object.h" + +#define w(x) +#define io(x) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ + +#define CAMEL_INDEX_VERSION (0x01) + +struct _CamelIndexPrivate { + void *dummy; +}; + +#define _PRIVATE(o) (((CamelIndex *)(o))->priv) + +#define CI_CLASS(o) ((CamelIndexClass *)(((CamelObject *)o)->classfuncs)) + +/* ********************************************************************** */ +/* CamelIndex */ +/* ********************************************************************** */ + +static CamelObjectClass *camel_index_parent; + +static void +camel_index_class_init(CamelIndexClass *klass) +{ + camel_index_parent = CAMEL_OBJECT_CLASS(camel_type_get_global_classfuncs(camel_object_get_type())); +} + +static void +camel_index_init(CamelIndex *idx) +{ + struct _CamelIndexPrivate *p; + + p = _PRIVATE(idx) = g_malloc0(sizeof(*p)); + + idx->version = CAMEL_INDEX_VERSION; +} + +static void +camel_index_finalise(CamelIndex *idx) +{ + g_free(idx->path); + g_free(idx->priv); +} + +CamelType +camel_index_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelIndex", + sizeof (CamelIndex), + sizeof (CamelIndexClass), + (CamelObjectClassInitFunc) camel_index_class_init, + NULL, + (CamelObjectInitFunc) camel_index_init, + (CamelObjectFinalizeFunc) camel_index_finalise); + } + + return type; +} + +CamelIndex * +camel_index_new(const char *path, int flags) +{ + CamelIndex *idx = (CamelIndex *)camel_object_new(camel_index_get_type()); + + camel_index_construct(idx, path, flags); + + return idx; +} + +void +camel_index_construct(CamelIndex *idx, const char *path, int flags) +{ + g_free(idx->path); + idx->path = g_strdup_printf("%s.index", path); + idx->flags = flags; +} + +int +camel_index_rename(CamelIndex *idx, const char *path) +{ + return CI_CLASS(idx)->rename(idx, path); +} + +void +camel_index_set_normalise(CamelIndex *idx, CamelIndexNorm func, void *data) +{ + idx->normalise = func; + idx->normalise_data = data; +} + +int +camel_index_sync(CamelIndex *idx) +{ + return CI_CLASS(idx)->sync(idx); +} + +int +camel_index_compress(CamelIndex *idx) +{ + return CI_CLASS(idx)->compress(idx); +} + +int +camel_index_has_name(CamelIndex *idx, const char *name) +{ + return CI_CLASS(idx)->has_name(idx, name); +} + +CamelIndexName * +camel_index_add_name(CamelIndex *idx, const char *name) +{ + return CI_CLASS(idx)->add_name(idx, name); +} + +int +camel_index_write_name(CamelIndex *idx, CamelIndexName *idn) +{ + return CI_CLASS(idx)->write_name(idx, idn); +} + +CamelIndexCursor * +camel_index_find_name(CamelIndex *idx, const char *name) +{ + return CI_CLASS(idx)->find_name(idx, name); +} + +void +camel_index_delete_name(CamelIndex *idx, const char *name) +{ + return CI_CLASS(idx)->delete_name(idx, name); +} + +CamelIndexCursor * +camel_index_find(CamelIndex *idx, const char *word) +{ + char *b = (char *)word; + CamelIndexCursor *ret; + + if (idx->normalise) + b = idx->normalise(idx, word, idx->normalise_data); + + ret = CI_CLASS(idx)->find(idx, b); + + if (b != word) + g_free(b); + + return ret; +} + +CamelIndexCursor * +camel_index_words(CamelIndex *idx) +{ + return CI_CLASS(idx)->words(idx); +} + +CamelIndexCursor * +camel_index_names(CamelIndex *idx) +{ + return CI_CLASS(idx)->names(idx); +} + +/* ********************************************************************** */ +/* CamelIndexName */ +/* ********************************************************************** */ + +static CamelObjectClass *camel_index_name_parent; + +#define CIN_CLASS(o) ((CamelIndexNameClass *)(((CamelObject *)o)->classfuncs)) + +static void +camel_index_name_class_init(CamelIndexNameClass *klass) +{ + camel_index_name_parent = CAMEL_OBJECT_CLASS(camel_type_get_global_classfuncs(camel_object_get_type())); +} + +static void +camel_index_name_init(CamelIndexName *idn) +{ +} + +static void +camel_index_name_finalise(CamelIndexName *idn) +{ + if (idn->index) + camel_object_unref((CamelObject *)idn->index); +} + +CamelType +camel_index_name_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelIndexName", + sizeof (CamelIndexName), + sizeof (CamelIndexNameClass), + (CamelObjectClassInitFunc) camel_index_name_class_init, + NULL, + (CamelObjectInitFunc) camel_index_name_init, + (CamelObjectFinalizeFunc) camel_index_name_finalise); + } + + return type; +} + +CamelIndexName * +camel_index_name_new(CamelIndex *idx, const char *name) +{ + CamelIndexName *idn = (CamelIndexName *)camel_object_new(camel_index_name_get_type()); + + idn->index = idx; + camel_object_ref((CamelObject *)idx); + + return idn; +} + +void +camel_index_name_add_word(CamelIndexName *idn, const char *word) +{ + char *b = (char *)word; + + if (idn->index->normalise) + b = idn->index->normalise(idn->index, word, idn->index->normalise_data); + + CIN_CLASS(idn)->add_word(idn, b); + + if (b != word) + g_free(b); +} + +size_t +camel_index_name_add_buffer(CamelIndexName *idn, const char *buffer, size_t len) +{ + return CIN_CLASS(idn)->add_buffer(idn, buffer, len); +} + +/* ********************************************************************** */ +/* CamelIndexCursor */ +/* ********************************************************************** */ + +static CamelObjectClass *camel_index_cursor_parent; + +#define CIC_CLASS(o) ((CamelIndexCursorClass *)(((CamelObject *)o)->classfuncs)) + +static void +camel_index_cursor_class_init(CamelIndexCursorClass *klass) +{ + camel_index_cursor_parent = CAMEL_OBJECT_CLASS(camel_type_get_global_classfuncs(camel_object_get_type())); +} + +static void +camel_index_cursor_init(CamelIndexCursor *idc) +{ +} + +static void +camel_index_cursor_finalise(CamelIndexCursor *idc) +{ + if (idc->index) + camel_object_unref((CamelObject *)idc->index); +} + +CamelType +camel_index_cursor_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelIndexCursor", + sizeof (CamelIndexCursor), + sizeof (CamelIndexCursorClass), + (CamelObjectClassInitFunc) camel_index_cursor_class_init, + NULL, + (CamelObjectInitFunc) camel_index_cursor_init, + (CamelObjectFinalizeFunc) camel_index_cursor_finalise); + } + + return type; +} + +CamelIndexCursor * +camel_index_cursor_new(CamelIndex *idx, const char *name) +{ + CamelIndexCursor *idc = (CamelIndexCursor *)camel_object_new(camel_index_cursor_get_type()); + + idc->index = idx; + camel_object_ref((CamelObject *)idx); + + return idc; +} + +const char * +camel_index_cursor_next(CamelIndexCursor *idc) +{ + return CIC_CLASS(idc)->next(idc); +} + +void +camel_index_cursor_reset(CamelIndexCursor *idc) +{ + return CIC_CLASS(idc)->reset(idc); +} + diff --git a/camel/camel-index.h b/camel/camel-index.h new file mode 100644 index 0000000000..eba57a43b0 --- /dev/null +++ b/camel/camel-index.h @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef _CAMEL_INDEX_H +#define _CAMEL_INDEX_H + +#include +#include + +#define CAMEL_INDEX(obj) CAMEL_CHECK_CAST (obj, camel_index_get_type (), CamelIndex) +#define CAMEL_INDEX_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_index_get_type (), CamelIndexClass) +#define CAMEL_IS_INDEX(obj) CAMEL_CHECK_TYPE (obj, camel_index_get_type ()) + +typedef struct _CamelIndex CamelIndex; +typedef struct _CamelIndexClass CamelIndexClass; + +#define CAMEL_INDEX_NAME(obj) CAMEL_CHECK_CAST (obj, camel_index_name_get_type (), CamelIndexName) +#define CAMEL_INDEX_NAME_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_index_name_get_type (), CamelIndexNameClass) +#define CAMEL_IS_INDEX_NAME(obj) CAMEL_CHECK_TYPE (obj, camel_index_name_get_type ()) + +typedef struct _CamelIndexName CamelIndexName; +typedef struct _CamelIndexNameClass CamelIndexNameClass; + +#define CAMEL_INDEX_CURSOR(obj) CAMEL_CHECK_CAST (obj, camel_index_cursor_get_type (), CamelIndexCursor) +#define CAMEL_INDEX_CURSOR_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_index_cursor_get_type (), CamelIndexCursorClass) +#define CAMEL_IS_INDEX_CURSOR(obj) CAMEL_CHECK_TYPE (obj, camel_index_cursor_get_type ()) + +typedef struct _CamelIndexCursor CamelIndexCursor; +typedef struct _CamelIndexCursorClass CamelIndexCursorClass; + +typedef char * (*CamelIndexNorm)(CamelIndex *idx, const char *word, void *data); + +/* ********************************************************************** */ + +struct _CamelIndexCursor { + CamelObject parent; + + struct _CamelIndexCursorPrivate *priv; + + CamelIndex *index; +}; + +struct _CamelIndexCursorClass { + CamelObjectClass parent; + + const char * (*next) (CamelIndexCursor *idc); + void (*reset) (CamelIndexCursor *idc); +}; + +guint camel_index_cursor_get_type(void); + +CamelIndexCursor *camel_index_cursor_new(CamelIndex *idx, const char *name); + +const char *camel_index_cursor_next(CamelIndexCursor *idc); +void camel_index_cursor_reset(CamelIndexCursor *idc); + +/* ********************************************************************** */ + +struct _CamelIndexName { + CamelObject parent; + + struct _CamelIndexNamePrivate *priv; + + CamelIndex *index; + + char *name; /* name being indexed */ + + GByteArray *buffer; /* used for normalisation */ + GHashTable *words; /* unique list of words */ +}; + +struct _CamelIndexNameClass { + CamelObjectClass parent; + + int (*sync)(CamelIndexName *name); + void (*add_word)(CamelIndexName *name, const char *word); + size_t (*add_buffer)(CamelIndexName *name, const char *buffer, size_t len); +}; + +guint camel_index_name_get_type (void); + +CamelIndexName *camel_index_name_new(CamelIndex *idx, const char *name); + +void camel_index_name_add_word(CamelIndexName *name, const char *word); +size_t camel_index_name_add_buffer(CamelIndexName *name, const char *buffer, size_t len); + +/* ********************************************************************** */ + +struct _CamelIndex { + CamelObject parent; + + struct _CamelIndexPrivate *priv; + + char *path; + guint32 version; + guint32 flags; + + CamelIndexNorm normalise; + void *normalise_data; +}; + +struct _CamelIndexClass { + CamelObjectClass parent_class; + + int (*sync)(CamelIndex *idx); + int (*compress)(CamelIndex *idx); + + int (*rename)(CamelIndex *idx, const char *path); + + int (*has_name)(CamelIndex *idx, const char *name); + CamelIndexName * (*add_name)(CamelIndex *idx, const char *name); + int (*write_name)(CamelIndex *idx, CamelIndexName *idn); + CamelIndexCursor * (*find_name)(CamelIndex *idx, const char *name); + void (*delete_name)(CamelIndex *idx, const char *name); + CamelIndexCursor * (*find)(CamelIndex *idx, const char *word); + + CamelIndexCursor * (*words)(CamelIndex *idx); + CamelIndexCursor * (*names)(CamelIndex *idx); +}; + +guint camel_index_get_type (void); + +CamelIndex *camel_index_new(const char *path, int flags); +void camel_index_construct(CamelIndex *, const char *path, int flags); +int camel_index_rename(CamelIndex *, const char *path); + +void camel_index_set_normalise(CamelIndex *idx, CamelIndexNorm func, void *data); + +int camel_index_sync(CamelIndex *idx); +int camel_index_compress(CamelIndex *idx); + +int camel_index_has_name(CamelIndex *idx, const char *name); +CamelIndexName *camel_index_add_name(CamelIndex *idx, const char *name); +int camel_index_write_name(CamelIndex *idx, CamelIndexName *idn); +CamelIndexCursor *camel_index_find_name(CamelIndex *idx, const char *name); +void camel_index_delete_name(CamelIndex *idx, const char *name); +CamelIndexCursor *camel_index_find(CamelIndex *idx, const char *word); + +CamelIndexCursor *camel_index_words(CamelIndex *idx); +CamelIndexCursor *camel_index_names(CamelIndex *idx); + +#endif /* ! _CAMEL_INDEX_H */ diff --git a/camel/camel-mime-filter-index.c b/camel/camel-mime-filter-index.c index 5332bd4df2..e9df08072b 100644 --- a/camel/camel-mime-filter-index.c +++ b/camel/camel-mime-filter-index.c @@ -20,6 +20,7 @@ #include "camel-mime-filter-index.h" +#include "camel-text-index.h" static void camel_mime_filter_index_class_init (CamelMimeFilterIndexClass *klass); static void camel_mime_filter_index_finalize (CamelObject *o); @@ -49,8 +50,9 @@ camel_mime_filter_index_finalize(CamelObject *o) { CamelMimeFilterIndex *f = (CamelMimeFilterIndex *)o; - g_free(f->name); - f->index = NULL; /* ibex's need refcounting? */ + if (f->name) + camel_object_unref((CamelObject *)f->name); + camel_object_unref((CamelObject *)f->index); } static void @@ -62,7 +64,8 @@ complete(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out, goto donothing; } - ibex_index_buffer(f->index, f->name, in, len, NULL); + camel_index_name_add_buffer(f->name, in, len); + camel_index_name_add_buffer(f->name, NULL, 0); donothing: *out = in; @@ -74,22 +77,12 @@ static void filter(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out, size_t *outlenptr, size_t *outprespace) { CamelMimeFilterIndex *f = (CamelMimeFilterIndex *)mf; - int inleft = 0; if (f->index == NULL || f->name==NULL) { goto donothing; } - ibex_index_buffer(f->index, f->name, in, len, &inleft); - - if (inleft>0) { - camel_mime_filter_backup(mf, in+(len-inleft), inleft); - } - - *out = in; - *outlenptr = len-inleft; - *outprespace = prespace; - return; + camel_index_name_add_buffer(f->name, in, len); donothing: *out = in; @@ -123,25 +116,29 @@ camel_mime_filter_index_new (void) return new; } -CamelMimeFilterIndex *camel_mime_filter_index_new_ibex (ibex *index) +CamelMimeFilterIndex *camel_mime_filter_index_new_index (struct _CamelIndex *index) { CamelMimeFilterIndex *new = camel_mime_filter_index_new(); if (new) { new->index = index; - new->name = g_strdup(""); + if (index) + camel_object_ref((CamelObject *)index); } return new; } /* Set the match name for any indexed words */ -void camel_mime_filter_index_set_name (CamelMimeFilterIndex *mf, char *name) +void camel_mime_filter_index_set_name (CamelMimeFilterIndex *mf, struct _CamelIndexName *name) { - g_free(mf->name); - mf->name = g_strdup(name); + if (mf->name) + camel_object_unref((CamelObject *)mf->name); + mf->name = name; + if (name) + camel_object_ref((CamelObject *)name); } -void camel_mime_filter_index_set_ibex (CamelMimeFilterIndex *mf, ibex *index) +void camel_mime_filter_index_set_index (CamelMimeFilterIndex *mf, CamelIndex *index) { if (mf->index) { char *out; @@ -149,7 +146,10 @@ void camel_mime_filter_index_set_ibex (CamelMimeFilterIndex *mf, ibex *index) camel_mime_filter_complete((CamelMimeFilter *)mf, "", 0, 0, &out, &outlen, &outspace); } + mf->index = index; + if (index) + camel_object_ref((CamelObject *)index); } diff --git a/camel/camel-mime-filter-index.h b/camel/camel-mime-filter-index.h index ae8ef60801..fa1f49e830 100644 --- a/camel/camel-mime-filter-index.h +++ b/camel/camel-mime-filter-index.h @@ -29,7 +29,6 @@ extern "C" { #endif /* __cplusplus */ #include -#include #define CAMEL_MIME_FILTER_INDEX(obj) CAMEL_CHECK_CAST (obj, camel_mime_filter_index_get_type (), CamelMimeFilterIndex) #define CAMEL_MIME_FILTER_INDEX_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_mime_filter_index_get_type (), CamelMimeFilterIndexClass) @@ -42,8 +41,8 @@ struct _CamelMimeFilterIndex { struct _CamelMimeFilterIndexPrivate *priv; - ibex *index; - char *name; + struct _CamelIndex *index; + struct _CamelIndexName *name; }; struct _CamelMimeFilterIndexClass { @@ -53,11 +52,11 @@ struct _CamelMimeFilterIndexClass { guint camel_mime_filter_index_get_type (void); CamelMimeFilterIndex *camel_mime_filter_index_new (void); -CamelMimeFilterIndex *camel_mime_filter_index_new_ibex (ibex *); +CamelMimeFilterIndex *camel_mime_filter_index_new_index(struct _CamelIndex *); /* Set the match name for any indexed words */ -void camel_mime_filter_index_set_name (CamelMimeFilterIndex *, char *); -void camel_mime_filter_index_set_ibex (CamelMimeFilterIndex *mf, ibex *index); +void camel_mime_filter_index_set_name (CamelMimeFilterIndex *, struct _CamelIndexName *name); +void camel_mime_filter_index_set_index (CamelMimeFilterIndex *mf, struct _CamelIndex *index); #ifdef __cplusplus } diff --git a/camel/camel-partition-table.c b/camel/camel-partition-table.c new file mode 100644 index 0000000000..48936796dc --- /dev/null +++ b/camel/camel-partition-table.c @@ -0,0 +1,955 @@ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "e-util/e-msgport.h" + +#include "camel-block-file.h" +#include "camel-partition-table.h" + +/* Do we synchronously write table updates - makes the + tables consistent after program crash without sync */ +#define SYNC_UPDATES + +#ifdef ENABLE_THREADS +#include +#endif + +#define d(x) /*(printf("%s(%d):%s: ", __FILE__, __LINE__, __PRETTY_FUNCTION__),(x))*/ +/* key index debug */ +#define k(x) /*(printf("%s(%d):%s: ", __FILE__, __LINE__, __PRETTY_FUNCTION__),(x))*/ + +#ifdef ENABLE_THREADS + +struct _CamelPartitionTablePrivate { + pthread_mutex_t lock; /* for locking partition */ +}; + +#define CAMEL_PARTITION_TABLE_LOCK(kf, lock) (pthread_mutex_lock(&(kf)->priv->lock)) +#define CAMEL_PARTITION_TABLE_UNLOCK(kf, lock) (pthread_mutex_unlock(&(kf)->priv->lock)) +#else +#define CAMEL_PARTITION_TABLE_LOCK(kf, lock) +#define CAMEL_PARTITION_TABLE_UNLOCK(kf, lock) +#endif + +static void +camel_partition_table_class_init(CamelPartitionTableClass *klass) +{ +} + +static void +camel_partition_table_init(CamelPartitionTable *cpi) +{ + struct _CamelPartitionTablePrivate *p; + + e_dlist_init(&cpi->partition); + + p = cpi->priv = g_malloc0(sizeof(*cpi->priv)); +#ifdef ENABLE_THREADS + pthread_mutex_init(&p->lock, NULL); +#endif +} + +static void +camel_partition_table_finalise(CamelPartitionTable *cpi) +{ + CamelBlock *bl; + struct _CamelPartitionTablePrivate *p; + + p = cpi->priv; + + if (cpi->blocks) { + camel_block_file_sync(cpi->blocks); + while ((bl = (CamelBlock *)e_dlist_remhead(&cpi->partition))) { + camel_block_file_sync_block(cpi->blocks, bl); + camel_block_file_unref_block(cpi->blocks, bl); + } + + camel_object_unref((CamelObject *)cpi->blocks); + } + +#ifdef ENABLE_THREADS + pthread_mutex_destroy(&p->lock); +#endif + g_free(p); + +} + +CamelType +camel_partition_table_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelPartitionTable", + sizeof (CamelPartitionTable), + sizeof (CamelPartitionTableClass), + (CamelObjectClassInitFunc) camel_partition_table_class_init, + NULL, + (CamelObjectInitFunc) camel_partition_table_init, + (CamelObjectFinalizeFunc) camel_partition_table_finalise); + } + + return type; +} + +/* ********************************************************************** */ + +/* + Have 2 hashes: + Name -> nameid + Word -> wordid + +nameid is pointer to name file, includes a bit to say if name is deleted +wordid is a pointer to word file, includes pointer to start of word entries + +delete a name -> set it as deleted, do nothing else though + +lookup word, if nameid is deleted, mark it in wordlist as unused and mark for write (?) +*/ + +/* ********************************************************************** */ + +/* This simple hash seems to work quite well */ +static camel_hash_t hash_key(const char *key) +{ + camel_hash_t hash = 0xABADF00D; + + while (*key) { + hash = hash * (*key) ^ (*key); + key++; + } + + return hash; +} + +/* Call with lock held */ +static CamelBlock *find_partition(CamelPartitionTable *cpi, camel_hash_t id, int *indexp) +{ + int index, jump; + CamelBlock *bl; + CamelPartitionMapBlock *ptb; + CamelPartitionMap *part; + + /* first, find the block this key might be in, then binary search the block */ + bl = (CamelBlock *)cpi->partition.head; + while (bl->next) { + ptb = (CamelPartitionMapBlock *)&bl->data; + part = ptb->partition; + if (ptb->used > 0 && id <= part[ptb->used-1].hashid) { + index = ptb->used/2; + jump = ptb->used/4; + + if (jump == 0) + jump = 1; + + while (1) { + if (id <= part[index].hashid) { + if (index == 0 || id > part[index-1].hashid) + break; + index -= jump; + } else { + if (index >= ptb->used-1) + break; + index += jump; + } + jump = jump/2; + if (jump == 0) + jump = 1; + } + *indexp = index; + + return bl; + } + bl = bl->next; + } + + g_warning("could not find a partition that could fit ! partition table corrupt!"); + + /* This should never be reached */ + + return NULL; +} + +CamelPartitionTable *camel_partition_table_new(struct _CamelBlockFile *bs, camel_block_t root) +{ + CamelPartitionTable *cpi; + CamelPartitionMapBlock *ptb; + CamelPartitionKeyBlock *kb; + CamelBlock *block, *pblock; + + cpi = (CamelPartitionTable *)camel_object_new(camel_partition_table_get_type()); + cpi->rootid = root; + cpi->blocks = bs; + camel_object_ref((CamelObject *)bs); + + /* read the partition table into memory */ + do { + block = camel_block_file_get_block(bs, root); + if (block == NULL) + goto fail; + + ptb = (CamelPartitionMapBlock *)&block->data; + + d(printf("Adding partition block, used = %d, hashid = %08x\n", ptb->used, ptb->partition[0].hashid)); + + /* if we have no data, prime initial block */ + if (ptb->used == 0 && e_dlist_empty(&cpi->partition) && ptb->next == 0) { + pblock = camel_block_file_new_block(bs); + if (pblock == NULL) { + camel_block_file_unref_block(bs, block); + goto fail; + } + kb = (CamelPartitionKeyBlock *)&pblock->data; + kb->used = 0; + ptb->used = 1; + ptb->partition[0].hashid = 0xffffffff; + ptb->partition[0].blockid = pblock->id; + camel_block_file_touch_block(bs, pblock); + camel_block_file_unref_block(bs, pblock); + camel_block_file_touch_block(bs, block); +#ifdef SYNC_UPDATES + camel_block_file_sync_block(bs, block); +#endif + } + + root = ptb->next; + camel_block_file_detach_block(bs, block); + e_dlist_addtail(&cpi->partition, (EDListNode *)block); + } while (root); + + return cpi; + +fail: + camel_object_unref((CamelObject *)cpi); + return NULL; +} + +camel_key_t camel_partition_table_lookup(CamelPartitionTable *cpi, const char *key) +{ + CamelPartitionKeyBlock *pkb; + CamelPartitionMapBlock *ptb; + CamelBlock *block, *ptblock; + camel_hash_t hashid; + camel_key_t keyid = 0; + int index, i; + + hashid = hash_key(key); + + CAMEL_PARTITION_TABLE_LOCK(cpi, lock); + + ptblock = find_partition(cpi, hashid, &index); + if (ptblock == NULL) { + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + return 0; + } + ptb = (CamelPartitionMapBlock *)&ptblock->data; + block = camel_block_file_get_block(cpi->blocks, ptb->partition[index].blockid); + if (block == NULL) { + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + return 0; + } + + pkb = (CamelPartitionKeyBlock *)&block->data; + + /* What to do about duplicate hash's? */ + for (i=0;iused;i++) { + if (pkb->keys[i].hashid == hashid) { + /* !! need to: lookup and compare string value */ + /* get_key() if key == key ... */ + keyid = pkb->keys[i].keyid; + break; + } + } + + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + + camel_block_file_unref_block(cpi->blocks, block); + + return keyid; +} + +void camel_partition_table_remove(CamelPartitionTable *cpi, const char *key) +{ + CamelPartitionKeyBlock *pkb; + CamelPartitionMapBlock *ptb; + CamelBlock *block, *ptblock; + camel_hash_t hashid; + camel_key_t keyid = 0; + int index, i; + + hashid = hash_key(key); + + CAMEL_PARTITION_TABLE_LOCK(cpi, lock); + + ptblock = find_partition(cpi, hashid, &index); + if (ptblock == NULL) { + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + return; + } + ptb = (CamelPartitionMapBlock *)&ptblock->data; + block = camel_block_file_get_block(cpi->blocks, ptb->partition[index].blockid); + if (block == NULL) { + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + return; + } + pkb = (CamelPartitionKeyBlock *)&block->data; + + /* What to do about duplicate hash's? */ + for (i=0;iused;i++) { + if (pkb->keys[i].hashid == hashid) { + /* !! need to: lookup and compare string value */ + /* get_key() if key == key ... */ + keyid = pkb->keys[i].keyid; + + /* remove this key */ + pkb->used--; + for (;iused;i++) { + pkb->keys[i].keyid = pkb->keys[i+1].keyid; + pkb->keys[i].hashid = pkb->keys[i+1].hashid; + } + camel_block_file_touch_block(cpi->blocks, block); + break; + } + } + + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + + camel_block_file_unref_block(cpi->blocks, block); +} + +static int +keys_cmp(const void *ap, const void *bp) +{ + const CamelPartitionKey *a = ap; + const CamelPartitionKey *b = bp; + + if (a->hashid < b->hashid) + return -1; + else if (a->hashid > b->hashid) + return 1; + + return 0; +} + +int +camel_partition_table_add(CamelPartitionTable *cpi, const char *key, camel_key_t keyid) +{ + camel_hash_t hashid, partid; + int index, newindex = 0; /* initialisation of this and pkb/nkb is just to silence compiler */ + CamelPartitionMapBlock *ptb, *ptn; + CamelPartitionKeyBlock *kb, *newkb, *nkb = NULL, *pkb = NULL; + CamelBlock *block, *ptblock, *ptnblock; + int i, half, len; + struct _CamelPartitionKey keys[CAMEL_BLOCK_SIZE/4]; + int ret = -1; + +#define KEY_SIZE (sizeof(kb->keys)/sizeof(kb->keys[0])) + + hashid = hash_key(key); + + CAMEL_PARTITION_TABLE_LOCK(cpi, lock); + ptblock = find_partition(cpi, hashid, &index); + if (ptblock == NULL) { + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + return -1; + } + ptb = (CamelPartitionMapBlock *)&ptblock->data; + block = camel_block_file_get_block(cpi->blocks, ptb->partition[index].blockid); + if (block == NULL) { + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + return -1; + } + kb = (CamelPartitionKeyBlock *)&block->data; + + /* TODO: Keep the key array in sorted order, cheaper lookups and split operation */ + + if (kb->used < sizeof(kb->keys)/sizeof(kb->keys[0])) { + /* Have room, just put it in */ + kb->keys[kb->used].hashid = hashid; + kb->keys[kb->used].keyid = keyid; + kb->used++; + } else { + CamelBlock *newblock = NULL, *nblock = NULL, *pblock = NULL; + + /* Need to split? See if previous or next has room, then split across that instead */ + + /* TODO: Should look at next/previous partition table block as well ... */ + + if (index > 0) { + pblock = camel_block_file_get_block(cpi->blocks, ptb->partition[index-1].blockid); + if (pblock == NULL) + goto fail; + pkb = (CamelPartitionKeyBlock *)&pblock->data; + } + if (index < (ptb->used-1)) { + nblock = camel_block_file_get_block(cpi->blocks, ptb->partition[index+1].blockid); + if (nblock == NULL) { + if (pblock) + camel_block_file_unref_block(cpi->blocks, pblock); + goto fail; + } + nkb = (CamelPartitionKeyBlock *)&nblock->data; + } + + if (pblock && pkb->used < KEY_SIZE) { + if (nblock && nkb->used < KEY_SIZE) { + if (pkb->used < nkb->used) { + newindex = index+1; + newblock = nblock; + } else { + newindex = index-1; + newblock = pblock; + } + } else { + newindex = index-1; + newblock = pblock; + } + } else { + if (nblock && nkb->used < KEY_SIZE) { + newindex = index+1; + newblock = nblock; + } + } + + /* We had no room, need to split across another block */ + if (newblock == NULL) { + /* See if we have room in the partition table for this block or need to split that too */ + if (ptb->used >= sizeof(ptb->partition)/sizeof(ptb->partition[0])) { + /* TODO: Could check next block to see if it'll fit there first */ + ptnblock = camel_block_file_new_block(cpi->blocks); + if (ptnblock == NULL) { + if (nblock) + camel_block_file_unref_block(cpi->blocks, nblock); + if (pblock) + camel_block_file_unref_block(cpi->blocks, pblock); + goto fail; + } + camel_block_file_detach_block(cpi->blocks, ptnblock); + + /* split block and link on-disk, always sorted */ + ptn = (CamelPartitionMapBlock *)&ptnblock->data; + ptn->next = ptb->next; + ptb->next = ptnblock->id; + len = ptb->used / 2; + ptn->used = ptb->used - len; + ptb->used = len; + memcpy(ptn->partition, &ptb->partition[len], ptn->used * sizeof(ptb->partition[0])); + + /* link in-memory */ + ptnblock->next = ptblock->next; + ptblock->next->prev = ptblock; + ptblock->next = ptnblock; + ptnblock->prev = ptblock; + + /* write in right order to ensure structure */ + camel_block_file_touch_block(cpi->blocks, ptnblock); +#ifdef SYNC_UPDATES + camel_block_file_sync_block(cpi->blocks, ptnblock); +#endif + if (index > len) { + camel_block_file_touch_block(cpi->blocks, ptblock); +#ifdef SYNC_UPDATES + camel_block_file_sync_block(cpi->blocks, ptblock); +#endif + index -= len; + ptb = ptn; + ptblock = ptnblock; + } + } + + /* try get newblock before modifying existing */ + newblock = camel_block_file_new_block(cpi->blocks); + if (newblock == NULL) { + if (nblock) + camel_block_file_unref_block(cpi->blocks, nblock); + if (pblock) + camel_block_file_unref_block(cpi->blocks, pblock); + goto fail; + } + + for (i=ptb->used-1;i>index;i--) { + ptb->partition[i+1].hashid = ptb->partition[i].hashid; + ptb->partition[i+1].blockid = ptb->partition[i].blockid; + } + ptb->used++; + + newkb = (CamelPartitionKeyBlock *)&newblock->data; + newkb->used = 0; + newindex = index+1; + + ptb->partition[newindex].hashid = ptb->partition[index].hashid; + ptb->partition[newindex].blockid = newblock->id; + + if (nblock) + camel_block_file_unref_block(cpi->blocks, nblock); + if (pblock) + camel_block_file_unref_block(cpi->blocks, pblock); + } else { + newkb = (CamelPartitionKeyBlock *)&newblock->data; + + if (newblock == pblock) { + if (nblock) + camel_block_file_unref_block(cpi->blocks, nblock); + } else { + if (pblock) + camel_block_file_unref_block(cpi->blocks, pblock); + } + } + + /* sort keys to find midpoint */ + len = kb->used; + memcpy(keys, kb->keys, sizeof(kb->keys[0])*len); + memcpy(keys+len, newkb->keys, sizeof(newkb->keys[0])*newkb->used); + len += newkb->used; + keys[len].hashid = hashid; + keys[len].keyid = keyid; + len++; + qsort(keys, len, sizeof(keys[0]), keys_cmp); + + /* Split keys, fix partition table */ + half = len/2; + partid = keys[half-1].hashid; + + if (index < newindex) { + memcpy(kb->keys, keys, sizeof(keys[0])*half); + kb->used = half; + memcpy(newkb->keys, keys+half, sizeof(keys[0])*(len-half)); + newkb->used = len-half; + ptb->partition[index].hashid = partid; + } else { + memcpy(newkb->keys, keys, sizeof(keys[0])*half); + newkb->used = half; + memcpy(kb->keys, keys+half, sizeof(keys[0])*(len-half)); + kb->used = len-half; + ptb->partition[newindex].hashid = partid; + } + + camel_block_file_touch_block(cpi->blocks, ptblock); +#ifdef SYNC_UPDATES + camel_block_file_sync_block(cpi->blocks, ptblock); +#endif + camel_block_file_touch_block(cpi->blocks, newblock); + camel_block_file_unref_block(cpi->blocks, newblock); + } + + camel_block_file_touch_block(cpi->blocks, block); + camel_block_file_unref_block(cpi->blocks, block); + + ret = 0; +fail: + CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock); + + return ret; +} + +/* ********************************************************************** */ + + +#ifdef ENABLE_THREADS + +struct _CamelKeyTablePrivate { + pthread_mutex_t lock; /* for locking key */ +}; + +#define CAMEL_KEY_TABLE_LOCK(kf, lock) (pthread_mutex_lock(&(kf)->priv->lock)) +#define CAMEL_KEY_TABLE_UNLOCK(kf, lock) (pthread_mutex_unlock(&(kf)->priv->lock)) +#else +#define CAMEL_KEY_TABLE_LOCK(kf, lock) +#define CAMEL_KEY_TABLE_UNLOCK(kf, lock) +#endif + +static void +camel_key_table_class_init(CamelKeyTableClass *klass) +{ +} + +static void +camel_key_table_init(CamelKeyTable *ki) +{ + struct _CamelKeyTablePrivate *p; + + p = ki->priv = g_malloc0(sizeof(*ki->priv)); +#ifdef ENABLE_THREADS + pthread_mutex_init(&p->lock, NULL); +#endif +} + +static void +camel_key_table_finalise(CamelKeyTable *ki) +{ + struct _CamelKeyTablePrivate *p; + + p = ki->priv; + + if (ki->blocks) { + if (ki->root_block) + camel_block_file_unref_block(ki->blocks, ki->root_block); + camel_block_file_sync(ki->blocks); + camel_object_unref((CamelObject *)ki->blocks); + } + +#ifdef ENABLE_THREADS + pthread_mutex_destroy(&p->lock); +#endif + g_free(p); + +} + +CamelType +camel_key_table_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelKeyTable", + sizeof (CamelKeyTable), + sizeof (CamelKeyTableClass), + (CamelObjectClassInitFunc) camel_key_table_class_init, + NULL, + (CamelObjectInitFunc) camel_key_table_init, + (CamelObjectFinalizeFunc) camel_key_table_finalise); + } + + return type; +} + + +CamelKeyTable * +camel_key_table_new(CamelBlockFile *bs, camel_block_t root) +{ + CamelKeyTable *ki; + + ki = (CamelKeyTable *)camel_object_new(camel_key_table_get_type()); + + ki->blocks = bs; + camel_object_ref((CamelObject *)bs); + ki->rootid = root; + + ki->root_block = camel_block_file_get_block(bs, ki->rootid); + if (ki->root_block == NULL) { + camel_object_unref((CamelObject *)ki); + ki = NULL; + } else { + camel_block_file_detach_block(bs, ki->root_block); + ki->root = (CamelKeyRootBlock *)&ki->root_block->data; + + k(printf("Opening key index\n")); + k(printf(" first %u\n last %u\n free %u\n", ki->root->first, ki->root->last, ki->root->free)); + } + + return ki; +} + +camel_key_t +camel_key_table_add(CamelKeyTable *ki, const char *key, camel_block_t data, unsigned int flags) +{ + CamelBlock *last, *next; + CamelKeyBlock *kblast, *kbnext; + int len, left; + unsigned int offset; + camel_key_t keyid = 0; + + /* Maximum key size = 128 chars */ + len = strlen(key); + if (len > CAMEL_KEY_TABLE_MAX_KEY) + len = 128; + + CAMEL_KEY_TABLE_LOCK(ki, lock); + + if (ki->root->last == 0) { + last = camel_block_file_new_block(ki->blocks); + if (last == NULL) + goto fail; + ki->root->last = ki->root->first = last->id; + camel_block_file_touch_block(ki->blocks, ki->root_block); + k(printf("adding first block, first = %u\n", ki->root->first)); + } else { + last = camel_block_file_get_block(ki->blocks, ki->root->last); + if (last == NULL) + goto fail; + } + + kblast = (CamelKeyBlock *)&last->data; + + if (kblast->used >= 127) + goto fail; + + if (kblast->used > 0) { + left = &kblast->u.keydata[kblast->u.keys[kblast->used-1].offset] - (char *)(&kblast->u.keys[kblast->used+1]); + d(printf("used = %d (%d), filled = %d, left = %d len = %d?\n", + kblast->used, kblast->used * sizeof(kblast->u.keys[0]), + sizeof(kblast->u.keydata) - kblast->u.keys[kblast->used-1].offset, + left, len)); + if (left < len) { + next = camel_block_file_new_block(ki->blocks); + if (next == NULL) { + camel_block_file_unref_block(ki->blocks, last); + goto fail; + } + kbnext = (CamelKeyBlock *)&next->data; + kblast->next = next->id; + ki->root->last = next->id; + k(printf("adding new block, first = %u, last = %u\n", ki->root->first, ki->root->last)); + camel_block_file_touch_block(ki->blocks, ki->root_block); + camel_block_file_touch_block(ki->blocks, last); + camel_block_file_unref_block(ki->blocks, last); + kblast = kbnext; + last = next; + } + } + + if (kblast->used > 0) + offset = kblast->u.keys[kblast->used-1].offset - len; + else + offset = sizeof(kblast->u.keydata)-len; + + kblast->u.keys[kblast->used].flags = flags; + kblast->u.keys[kblast->used].data = data; + kblast->u.keys[kblast->used].offset = offset; + memcpy(kblast->u.keydata + offset, key, len); + + keyid = (last->id & (~(CAMEL_BLOCK_SIZE-1))) | kblast->used; + + kblast->used++; + + g_assert(kblast->used < 127); + + camel_block_file_touch_block(ki->blocks, last); + camel_block_file_unref_block(ki->blocks, last); + +#ifdef SYNC_UPDATES + camel_block_file_sync_block(ki->blocks, ki->root_block); +#endif +fail: + CAMEL_KEY_TABLE_UNLOCK(ki, lock); + + return keyid; +} + +void +camel_key_table_set_data(CamelKeyTable *ki, camel_key_t keyid, camel_block_t data) +{ + CamelBlock *bl; + camel_block_t blockid; + int index; + CamelKeyBlock *kb; + + if (keyid == 0) + return; + + blockid = keyid & (~(CAMEL_BLOCK_SIZE-1)); + index = keyid & (CAMEL_BLOCK_SIZE-1); + + bl = camel_block_file_get_block(ki->blocks, blockid); + if (bl == NULL) + return; + kb = (CamelKeyBlock *)&bl->data; + + CAMEL_KEY_TABLE_LOCK(ki, lock); + + if (kb->u.keys[index].data != data) { + kb->u.keys[index].data = data; + camel_block_file_touch_block(ki->blocks, bl); + } + + CAMEL_KEY_TABLE_UNLOCK(ki, lock); + + camel_block_file_unref_block(ki->blocks, bl); +} + +void +camel_key_table_set_flags(CamelKeyTable *ki, camel_key_t keyid, unsigned int flags, unsigned int set) +{ + CamelBlock *bl; + camel_block_t blockid; + int index; + CamelKeyBlock *kb; + unsigned int old; + + if (keyid == 0) + return; + + blockid = keyid & (~(CAMEL_BLOCK_SIZE-1)); + index = keyid & (CAMEL_BLOCK_SIZE-1); + + bl = camel_block_file_get_block(ki->blocks, blockid); + if (bl == NULL) + return; + kb = (CamelKeyBlock *)&bl->data; + + g_assert(kb->used < 127); + g_assert(index < kb->used); + + CAMEL_KEY_TABLE_LOCK(ki, lock); + + old = kb->u.keys[index].flags; + if ((old & set) != (flags & set)) { + kb->u.keys[index].flags = (old & (~set)) | (flags & set); + camel_block_file_touch_block(ki->blocks, bl); + } + + CAMEL_KEY_TABLE_UNLOCK(ki, lock); + + camel_block_file_unref_block(ki->blocks, bl); +} + +camel_block_t +camel_key_table_lookup(CamelKeyTable *ki, camel_key_t keyid, char **keyp, unsigned int *flags) +{ + CamelBlock *bl; + camel_block_t blockid; + int index, len, off; + char *key; + CamelKeyBlock *kb; + + if (keyp) + *keyp = 0; + if (flags) + *flags = 0; + if (keyid == 0) + return 0; + + blockid = keyid & (~(CAMEL_BLOCK_SIZE-1)); + index = keyid & (CAMEL_BLOCK_SIZE-1); + + bl = camel_block_file_get_block(ki->blocks, blockid); + if (bl == NULL) + return 0; + + kb = (CamelKeyBlock *)&bl->data; + + g_assert(kb->used < 127); + g_assert(index < kb->used); + + CAMEL_KEY_TABLE_LOCK(ki, lock); + + blockid = kb->u.keys[index].data; + if (flags) + *flags = kb->u.keys[index].flags; + + if (keyp) { + off = kb->u.keys[index].offset; + if (index == 0) + len = sizeof(kb->u.keydata) - off; + else + len = kb->u.keys[index-1].offset - off; + *keyp = key = g_malloc(len+1); + memcpy(key, kb->u.keydata + off, len); + key[len] = 0; + } + + CAMEL_KEY_TABLE_UNLOCK(ki, lock); + + camel_block_file_unref_block(ki->blocks, bl); + + return blockid; +} + +/* iterate through all keys */ +camel_key_t +camel_key_table_next(CamelKeyTable *ki, camel_key_t next, char **keyp, unsigned int *flagsp, camel_block_t *datap) +{ + CamelBlock *bl; + CamelKeyBlock *kb; + camel_block_t blockid; + int index; + + if (next == 0) { + next = ki->root->first; + if (next == 0) + return 0; + } else + next++; + + do { + blockid = next & (~(CAMEL_BLOCK_SIZE-1)); + index = next & (CAMEL_BLOCK_SIZE-1); + + bl = camel_block_file_get_block(ki->blocks, blockid); + if (bl == NULL) + return 0; + + kb = (CamelKeyBlock *)&bl->data; + + /* see if we need to goto the next block */ + if (index >= kb->used) { + /* FIXME: check for loops */ + next = kb->next; + camel_block_file_unref_block(ki->blocks, bl); + bl = NULL; + } + } while (bl == NULL); + + CAMEL_KEY_TABLE_LOCK(ki, lock); + + /* invalid block data */ + if ((kb->u.keys[index].offset >= sizeof(kb->u.keydata) + || kb->u.keys[index].offset < kb->u.keydata - (char *)&kb->u.keys[kb->used]) + || (index > 0 && + (kb->u.keys[index-1].offset >= sizeof(kb->u.keydata) + || kb->u.keys[index-1].offset < kb->u.keydata - (char *)&kb->u.keys[kb->used]))) { + g_warning("Block %u invalid scanning keys", bl->id); + camel_block_file_unref_block(ki->blocks, bl); + CAMEL_KEY_TABLE_UNLOCK(ki, lock); + return 0; + } + + if (datap) + *datap = kb->u.keys[index].data; + + if (flagsp) + *flagsp = kb->u.keys[index].flags; + + if (keyp) { + int len, off = kb->u.keys[index].offset; + char *key; + + if (index == 0) + len = sizeof(kb->u.keydata) - off; + else + len = kb->u.keys[index-1].offset - off; + *keyp = key = g_malloc(len+1); + memcpy(key, kb->u.keydata + off, len); + key[len] = 0; + } + + CAMEL_KEY_TABLE_UNLOCK(ki, lock); + + camel_block_file_unref_block(ki->blocks, bl); + + return next; +} + +/* ********************************************************************** */ diff --git a/camel/camel-partition-table.h b/camel/camel-partition-table.h new file mode 100644 index 0000000000..ab6ac8c204 --- /dev/null +++ b/camel/camel-partition-table.h @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef _CAMEL_PARTITION_TABLE_H +#define _CAMEL_PARTITION_TABLE_H + +#include +#include +#include + +#include "camel-block-file.h" + +/* ********************************************************************** */ + +/* CamelPartitionTable - index of key to keyid */ + +typedef guint32 camel_hash_t; /* a hashed key */ + +typedef struct _CamelPartitionKey CamelPartitionKey; +typedef struct _CamelPartitionKeyBlock CamelPartitionKeyBlock; +typedef struct _CamelPartitionMap CamelPartitionMap; +typedef struct _CamelPartitionMapBlock CamelPartitionMapBlock; + +typedef struct _CamelPartitionTable CamelPartitionTable; +typedef struct _CamelPartitionTable CamelPartitionTableClass; + +struct _CamelPartitionKey { + camel_hash_t hashid; + camel_key_t keyid; +}; + +struct _CamelPartitionKeyBlock { + guint32 used; + struct _CamelPartitionKey keys[(CAMEL_BLOCK_SIZE-4)/sizeof(struct _CamelPartitionKey)]; +}; + +struct _CamelPartitionMap { + camel_hash_t hashid; + camel_block_t blockid; +}; + +struct _CamelPartitionMapBlock { + camel_block_t next; + guint32 used; + struct _CamelPartitionMap partition[(CAMEL_BLOCK_SIZE-8)/sizeof(struct _CamelPartitionMap)]; +}; + +struct _CamelPartitionTable { + CamelObject parent; + + struct _CamelPartitionTablePrivate *priv; + + CamelBlockFile *blocks; + camel_block_t rootid; + + int (*is_key)(CamelPartitionTable *cpi, const char *key, camel_key_t keyid, void *data); + void *is_key_data; + + /* we keep a list of partition blocks active at all times */ + EDList partition; +}; + +struct _CamelPartitionTableClass { + CamelObjectClass parent; +}; + +CamelType camel_partition_table_get_type(void); + +CamelPartitionTable *camel_partition_table_new(struct _CamelBlockFile *bs, camel_block_t root); +int camel_partition_table_add(CamelPartitionTable *cpi, const char *key, camel_key_t keyid); +camel_key_t camel_partition_table_lookup(CamelPartitionTable *cpi, const char *key); +void camel_partition_table_remove(CamelPartitionTable *cpi, const char *key); + +/* ********************************************************************** */ + +/* CamelKeyTable - index of keyid to key and flag and data mapping */ + +typedef struct _CamelKeyBlock CamelKeyBlock; +typedef struct _CamelKeyRootBlock CamelKeyRootBlock; + +typedef struct _CamelKeyTable CamelKeyTable; +typedef struct _CamelKeyTable CamelKeyTableClass; + +struct _CamelKeyRootBlock { + camel_block_t first; + camel_block_t last; + camel_key_t free; /* free list */ +}; + +struct _CamelKeyKey { + camel_block_t data; + unsigned int offset:10; + unsigned int flags:22; +}; + +struct _CamelKeyBlock { + camel_block_t next; + guint32 used; + union { + struct _CamelKeyKey keys[(CAMEL_BLOCK_SIZE-8)/sizeof(struct _CamelKeyKey)]; + char keydata[CAMEL_BLOCK_SIZE-8]; + } u; +}; + +#define CAMEL_KEY_TABLE_MAX_KEY (128) /* max size of any key */ + +struct _CamelKeyTable { + CamelObject parent; + + struct _CamelKeyTablePrivate *priv; + + CamelBlockFile *blocks; + + camel_block_t rootid; + + CamelKeyRootBlock *root; + CamelBlock *root_block; +}; + +struct _CamelKeyTableClass { + CamelObjectClass parent; +}; + +CamelType camel_key_table_get_type(void); + +CamelKeyTable * camel_key_table_new(CamelBlockFile *bs, camel_block_t root); +camel_key_t camel_key_table_add(CamelKeyTable *ki, const char *key, camel_block_t data, unsigned int flags); +void camel_key_table_set_data(CamelKeyTable *ki, camel_key_t keyid, camel_block_t data); +void camel_key_table_set_flags(CamelKeyTable *ki, camel_key_t keyid, unsigned int flags, unsigned int set); +camel_block_t camel_key_table_lookup(CamelKeyTable *ki, camel_key_t keyid, char **key, unsigned int *flags); +camel_key_t camel_key_table_next(CamelKeyTable *ki, camel_key_t next, char **keyp, unsigned int *flagsp, camel_block_t *datap); + +#endif /* ! _CAMEL_PARTITION_TABLE_H */ diff --git a/camel/camel-private.h b/camel/camel-private.h index 22771b3625..78af7814b7 100644 --- a/camel/camel-private.h +++ b/camel/camel-private.h @@ -156,7 +156,9 @@ struct _CamelFolderSummaryPrivate { struct _CamelMimeFilterSave *filter_save; struct _CamelMimeFilterHTML *filter_html; - struct ibex *index; + struct _CamelStreamFilter *filter_stream; + + struct _CamelIndex *index; #ifdef ENABLE_THREADS GMutex *summary_lock; /* for the summary hashtable/array */ diff --git a/camel/camel-text-index.c b/camel/camel-text-index.c new file mode 100644 index 0000000000..7ad069d90c --- /dev/null +++ b/camel/camel-text-index.c @@ -0,0 +1,1687 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*- */ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "e-util/e-msgport.h" +#include "e-util/e-memory.h" + +#include "camel/camel-object.h" + +#include "camel-text-index.h" +#include "camel-block-file.h" +#include "camel-partition-table.h" + +#include + +#include + +#define w(x) +#define io(x) +#define d(x) /*(printf("%s(%d):%s: ", __FILE__, __LINE__, __PRETTY_FUNCTION__),(x))*/ + +/* cursor debug */ +#define c(x) + +#ifdef ENABLE_THREADS +#define CAMEL_TEXT_INDEX_LOCK(kf, lock) (e_mutex_lock(((CamelTextIndex *)kf)->priv->lock)) +#define CAMEL_TEXT_INDEX_UNLOCK(kf, lock) (e_mutex_unlock(((CamelTextIndex *)kf)->priv->lock)) +#else +#define CAMEL_TEXT_INDEX_LOCK(kf, lock) +#define CAMEL_TEXT_INDEX_UNLOCK(kf, lock) +#endif + +static int text_index_compress_nosync(CamelIndex *idx); + +void camel_text_index_dump(CamelTextIndex *idx); +void camel_text_index_info(CamelTextIndex *idx); +void camel_text_index_validate(CamelTextIndex *idx); + +/* ********************************************************************** */ + +/* "private" data, shared between index/cursor/name classes */ +typedef struct _CamelTextIndexNamePrivate CamelTextIndexNamePrivate; + +struct _CamelTextIndexNamePrivate { + GString *buffer; + camel_key_t nameid; + EMemPool *pool; +}; + +CamelTextIndexName *camel_text_index_name_new(CamelTextIndex *idx, const char *name, camel_key_t nameid); + +/* ****************************** */ + +typedef struct _CamelTextIndexCursorPrivate CamelTextIndexCursorPrivate; + +struct _CamelTextIndexCursorPrivate { + camel_block_t first; + camel_block_t next; + + int record_index; + + size_t record_count; + camel_key_t *records; + + char *current; +}; + +CamelTextIndexCursor *camel_text_index_cursor_new(CamelTextIndex *idx, camel_block_t data); + +/* ****************************** */ +typedef struct _CamelTextIndexKeyCursorPrivate CamelTextIndexKeyCursorPrivate; + +struct _CamelTextIndexKeyCursorPrivate { + CamelKeyTable *table; + + camel_key_t keyid; + unsigned int flags; + camel_block_t data; + char *current; +}; + +CamelTextIndexKeyCursor *camel_text_index_key_cursor_new(CamelTextIndex *idx, CamelKeyTable *table); + +/* ********************************************************************** */ + +#define CAMEL_TEXT_INDEX_VERSION "TEXT.000" +#define CAMEL_TEXT_INDEX_KEY_VERSION "KEYS.000" + +struct _CamelTextIndexPrivate { + CamelBlockFile *blocks; + CamelKeyFile *links; + + CamelKeyTable *word_index; + CamelPartitionTable *word_hash; + + CamelKeyTable *name_index; + CamelPartitionTable *name_hash; + + /* Cache of words to write */ + int word_cache_limit; + int word_cache_count; + EDList word_cache; + GHashTable *words; +#ifdef ENABLE_THREADS + EMutex *lock; +#endif +}; + +/* Root block of text index */ +struct _CamelTextIndexRoot { + struct _CamelBlockRoot root; + + /* FIXME: the index root could contain a pointer to the hash root */ + camel_block_t word_index_root; /* a keyindex containing the keyid -> word mapping */ + camel_block_t word_hash_root; /* a partitionindex containing word -> keyid mapping */ + + camel_block_t name_index_root; /* same, for names */ + camel_block_t name_hash_root; + + guint32 words; /* total words */ + guint32 names; /* total names */ + guint32 deleted; /* deleted names */ + guint32 keys; /* total key 'chunks' written, used with deleted to determine fragmentation */ +}; + +struct _CamelTextIndexWord { + struct _CamelTextIndexWord *next; + struct _CamelTextIndexWord *prev; + + camel_block_t data; /* where the data starts */ + camel_key_t wordid; + char *word; + unsigned int used; + camel_key_t names[32]; +}; + +#define CTI_PRIVATE(o) (((CamelTextIndex *)(o))->priv) + +#define CI_CLASS(o) ((CamelTextIndexClass *)(((CamelObject *)o)->classfuncs)) + +/* ********************************************************************** */ +/* CamelTextIndex */ +/* ********************************************************************** */ + +static CamelObjectClass *camel_text_index_parent; + +/* call locked */ +static void +text_index_add_name_to_word(CamelIndex *idx, const char *word, camel_key_t nameid) +{ + struct _CamelTextIndexWord *w, *wp, *ww; + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + camel_key_t wordid; + camel_block_t data; + struct _CamelTextIndexRoot *rb = (struct _CamelTextIndexRoot *)p->blocks->root; + + w = g_hash_table_lookup(p->words, word); + if (w == NULL) { + wordid = camel_partition_table_lookup(p->word_hash, word); + if (wordid == 0) { + data = 0; + wordid = camel_key_table_add(p->word_index, word, 0, 0); + if (wordid == 0) + return; + camel_partition_table_add(p->word_hash, word, wordid); + rb->words++; + } else { + data = camel_key_table_lookup(p->word_index, wordid, NULL, 0); + if (data == 0) + return; + } + + w = g_malloc0(sizeof(*w)); + w->word = g_strdup(word); + w->wordid = wordid; + w->used = 1; + w->data = data; + + w->names[0] = nameid; + g_hash_table_insert(p->words, w->word, w); + e_dlist_addhead(&p->word_cache, (EDListNode *)w); + p->word_cache_count++; + ww = (struct _CamelTextIndexWord *)p->word_cache.tailpred; + wp = ww->prev; + while (wp && p->word_cache_count > p->word_cache_limit) { + io(printf("writing key file entry '%s' [%x]\n", ww->word, ww->data)); + if (camel_key_file_write(p->links, &ww->data, ww->used, ww->names) != -1) { + io(printf(" new data [%x]\n", ww->data)); + rb->keys++; + camel_key_table_set_data(p->word_index, ww->wordid, ww->data); + e_dlist_remove((EDListNode *)ww); + g_hash_table_remove(p->words, ww->word); + g_free(ww->word); + g_free(ww); + p->word_cache_count--; + } + ww = wp; + wp = wp->prev; + } + } else { + e_dlist_remove((EDListNode *)w); + e_dlist_addhead(&p->word_cache, (EDListNode *)w); + w->names[w->used] = nameid; + w->used++; + if (w->used == sizeof(w->names)/sizeof(w->names[0])) { + io(printf("writing key file entry '%s' [%x]\n", w->word, w->data)); + if (camel_key_file_write(p->links, &w->data, w->used, w->names) != -1) { + rb->keys++; + camel_key_table_set_data(p->word_index, w->wordid, w->data); + } + /* FIXME: what to on error? lost data? */ + w->used = 0; + } + } +} + +static int +text_index_sync(CamelIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + struct _CamelTextIndexWord *ww; + struct _CamelTextIndexRoot *rb; + int ret = 0, wfrag, nfrag, work = FALSE; + + if (p->blocks == NULL) + return 0; + + rb = (struct _CamelTextIndexRoot *)p->blocks->root; + + /* sync/flush word cache */ + + CAMEL_TEXT_INDEX_LOCK(idx, lock); + + /* we sync, bump down the cahce limits since we dont need them for reading */ + p->blocks->block_cache_limit = 128; + /* this doesn't really need to be dropped, its only used in updates anyway */ + p->word_cache_limit = 1024; + + work = !e_dlist_empty(&p->word_cache); + + while ( (ww = (struct _CamelTextIndexWord *)e_dlist_remhead(&p->word_cache)) ) { + if (ww->used > 0) { + io(printf("writing key file entry '%s' [%x]\n", ww->word, ww->data)); + if (camel_key_file_write(p->links, &ww->data, ww->used, ww->names) != -1) { + io(printf(" new data [%x]\n", ww->data)); + rb->keys++; + camel_key_table_set_data(p->word_index, ww->wordid, ww->data); + } else { + ret = -1; + } + ww->used = 0; + } + g_hash_table_remove(p->words, ww->word); + g_free(ww->word); + g_free(ww); + } + + /* only do the frag/compress check if we did some new writes on this index */ + if (ret == 0 && work) { + wfrag = rb->words ? (((rb->keys - rb->words) * 100)/ rb->words) : 0; + nfrag = rb->names ? ((rb->deleted * 100) / rb->names) : 0; + printf("wfrag = %d, nfrag = %d\n", wfrag, nfrag); + printf(" words = %d, keys = %d\n", rb->words, rb->keys); + if (wfrag > 30 || nfrag > 20) + ret = text_index_compress_nosync(idx); + } + + CAMEL_TEXT_INDEX_UNLOCK(idx, lock); + + return camel_block_file_sync(p->blocks); +} + +static void tmp_name(const char *in, char *o) +{ + char *s; + + s = strrchr(in, '/'); + if (s) { + memcpy(o, in, s-in+1); + memcpy(o+(s-in+1), ".#", 2); + strcpy(o+(s-in+3), s+1); + } else { + sprintf(o, ".#%s", in); + } +} + +static int +text_index_compress(CamelIndex *idx) +{ + int ret; + + CAMEL_TEXT_INDEX_LOCK(idx, lock); + + ret = camel_index_sync(idx); + if (ret != -1) + ret = text_index_compress_nosync(idx); + + CAMEL_TEXT_INDEX_UNLOCK(idx, lock); + + return ret; +} + +/* Attempt to recover index space by compressing the indices */ +static int +text_index_compress_nosync(CamelIndex *idx) +{ + CamelTextIndex *newidx; + struct _CamelTextIndexPrivate *newp, *oldp; + camel_key_t oldkeyid, newkeyid; + GHashTable *remap; + unsigned int deleted; + camel_block_t data, newdata; + int i, ret = -1; + char *name = NULL; + unsigned int flags; + char *newpath, *savepath, *oldpath; + size_t count, newcount; + camel_key_t *records, newrecords[256]; + struct _CamelTextIndexRoot *rb; + + newpath = alloca(strlen(idx->path)+5); + tmp_name(idx->path, newpath); + savepath = alloca(strlen(idx->path)+2); + sprintf(savepath, "%s~", idx->path); + oldpath = alloca(strlen(idx->path)+1); + strcpy(oldpath, idx->path); + + newidx = camel_text_index_new(newpath, O_RDWR|O_CREAT); + if (newidx == NULL) + return -1; + + newp = CTI_PRIVATE(newidx); + oldp = CTI_PRIVATE(idx); + + rb = (struct _CamelTextIndexRoot *)newp->blocks->root; + + rb->words = 0; + rb->names = 0; + rb->deleted = 0; + rb->keys = 0; + + /* Process: + For each name we still have: + Add it to the new index & setup remap table + + For each word: + Copy word's data to a new file + Add new word to index(*) (can we just copy blocks?) */ + + /* Copy undeleted names to new index file, creating new indices */ + io(printf("Copying undeleted names to new file\n")); + remap = g_hash_table_new(NULL, NULL); + oldkeyid = 0; + deleted = 0; + while ( (oldkeyid = camel_key_table_next(oldp->name_index, oldkeyid, &name, &flags, &data)) ) { + if ((flags&1) == 0) { + io(printf("copying name '%s'\n", name)); + newkeyid = camel_key_table_add(newp->name_index, name, data, flags); + if (newkeyid == 0) + goto fail; + rb->names++; + camel_partition_table_add(newp->name_hash, name, newkeyid); + g_hash_table_insert(remap, (void *)oldkeyid, (void *)newkeyid); + } else + io(printf("deleted name '%s'\n", name)); + g_free(name); + name = NULL; + deleted |= flags; + } while (oldkeyid); + + /* Copy word data across, remapping/deleting and create new index for it */ + /* We re-block the data into 256 entry lots while we're at it, since we only + have to do 1 at a time and its cheap */ + oldkeyid = 0; + while ( (oldkeyid = camel_key_table_next(oldp->word_index, oldkeyid, &name, &flags, &data)) ) { + io(printf("copying word '%s'\n", name)); + newdata = 0; + newcount = 0; + if (data) { + rb->words++; + rb->keys++; + } + while (data) { + if (camel_key_file_read(oldp->links, &data, &count, &records) == -1) { + io(printf("could not read from old keys at %d for word '%s'\n", (int)data, name)); + goto fail; + } + for (i=0;ilinks, &newdata, newcount, newrecords) == -1) + goto fail; + newcount = 0; + } + } + } + } + + if (newcount > 0) { + if (camel_key_file_write(newp->links, &newdata, newcount, newrecords) == -1) + goto fail; + } + + newkeyid = camel_key_table_add(newp->word_index, name, newdata, flags); + if (newkeyid == 0) + goto fail; + camel_partition_table_add(newp->word_hash, name, newkeyid); + g_free(name); + name = NULL; + } while (oldkeyid); + + if (camel_index_sync((CamelIndex *)newidx) == -1) + goto fail; + + /* Rename underlying files to match */ + ret = camel_index_rename(idx, savepath); + if (ret == -1) + goto fail; + + /* If this fails, we'll pick up something during restart? */ + ret = camel_index_rename((CamelIndex *)newidx, oldpath); + +#define myswap(a, b) { void *tmp = a; a = b; b = tmp; } + /* Poke the private data across to the new object */ + /* And change the fd's over, etc? */ + myswap(newp->blocks, oldp->blocks); + myswap(newp->links, oldp->links); + myswap(newp->word_index, oldp->word_index); + myswap(newp->word_hash, oldp->word_hash); + myswap(newp->name_index, oldp->name_index); + myswap(newp->name_hash, oldp->name_hash); +#undef myswap + + ret = 0; +fail: + camel_object_unref((CamelObject *)newidx); + g_free(name); + g_hash_table_destroy(remap); + + /* clean up temp files always */ + camel_text_index_remove(newpath); + unlink(savepath); + newpath = alloca(strlen(savepath)+6); + sprintf(newpath, "%s.data", savepath); + unlink(newpath); + + return ret; +} + +static int +text_index_rename(CamelIndex *idx, const char *path) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + char *newlink; + int err, ret; + + /* TODO: Needs some lock? */ + + ret = camel_block_file_rename(p->blocks, path); + if (ret == -1) + return -1; + + newlink = alloca(strlen(path)+8); + sprintf(newlink, "%s.data", path); + ret = camel_key_file_rename(p->links, newlink); + if (ret == -1) { + err = errno; + camel_block_file_rename(p->blocks, path); + errno = err; + return -1; + } + + g_free(idx->path); + idx->path = g_strdup(path); + + return 0; +} + +static int +text_index_has_name(CamelIndex *idx, const char *name) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + + return camel_partition_table_lookup(p->name_hash, name) != 0; +} + +static CamelIndexName * +text_index_add_name(CamelIndex *idx, const char *name) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + camel_key_t keyid; + CamelIndexName *idn; + struct _CamelTextIndexRoot *rb = (struct _CamelTextIndexRoot *)p->blocks->root; + + CAMEL_TEXT_INDEX_LOCK(idx, lock); + + /* if we're adding words, up the cache limits a lot */ + if (p->blocks) { + p->blocks->block_cache_limit = 1024; + p->word_cache_limit = 8192; + } + + /* If we have it already replace it */ + keyid = camel_partition_table_lookup(p->name_hash, name); + if (keyid != 0) { + rb->deleted++; + camel_key_table_set_flags(p->name_index, keyid, 1, 1); + camel_partition_table_remove(p->name_hash, name); + } + + keyid = camel_key_table_add(p->name_index, name, 0, 0); + if (keyid != 0) { + camel_partition_table_add(p->name_hash, name, keyid); + rb->names++; + } + + /* TODO: if keyid == 0, we had a failure, we shoudl somehow flag that, but for + now just return a valid object but discard its results, see text_index_write_name */ + + CAMEL_TEXT_INDEX_UNLOCK(idx, lock); + + idn = (CamelIndexName *)camel_text_index_name_new((CamelTextIndex *)idx, name, keyid); + + return idn; +} + +/* call locked */ +static void +hash_write_word(char *word, void *data, CamelIndexName *idn) +{ + CamelTextIndexName *tin = (CamelTextIndexName *)idn; + + text_index_add_name_to_word(idn->index, word, tin->priv->nameid); +} + +static int +text_index_write_name(CamelIndex *idx, CamelIndexName *idn) +{ + /* force 'flush' of any outstanding data */ + camel_index_name_add_buffer(idn, NULL, 0); + + /* see text_index_add_name for when this can be 0 */ + if (((CamelTextIndexName *)idn)->priv->nameid != 0) { + CAMEL_TEXT_INDEX_LOCK(idx, lock); + + g_hash_table_foreach(idn->words, (GHFunc)hash_write_word, idn); + + CAMEL_TEXT_INDEX_UNLOCK(idx, lock); + } + + return 0; +} + +static CamelIndexCursor * +text_index_find_name(CamelIndex *idx, const char *name) +{ + /* what was this for, umm */ + return NULL; +} + +static void +text_index_delete_name(CamelIndex *idx, const char *name) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + camel_key_t keyid; + struct _CamelTextIndexRoot *rb = (struct _CamelTextIndexRoot *)p->blocks->root; + + /* probably doesn't really need locking, but oh well */ + CAMEL_TEXT_INDEX_LOCK(idx, lock); + + /* We just mark the key deleted, and remove it from the hash table */ + keyid = camel_partition_table_lookup(p->name_hash, name); + if (keyid != 0) { + rb->deleted++; + camel_key_table_set_flags(p->name_index, keyid, 1, 1); + camel_partition_table_remove(p->name_hash, name); + } + + CAMEL_TEXT_INDEX_UNLOCK(idx, lock); +} + +static CamelIndexCursor * +text_index_find(CamelIndex *idx, const char *word) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + camel_key_t keyid; + camel_block_t data = 0; + unsigned int flags; + CamelIndexCursor *idc; + + CAMEL_TEXT_INDEX_LOCK(idx, lock); + + keyid = camel_partition_table_lookup(p->word_hash, word); + if (keyid != 0) { + data = camel_key_table_lookup(p->word_index, keyid, NULL, &flags); + if (flags & 1) + data = 0; + } + + CAMEL_TEXT_INDEX_UNLOCK(idx, lock); + + idc = (CamelIndexCursor *)camel_text_index_cursor_new((CamelTextIndex *)idx, data); + + return idc; +} + +static CamelIndexCursor * +text_index_words(CamelIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + + return (CamelIndexCursor *)camel_text_index_key_cursor_new((CamelTextIndex *)idx, p->word_index); +} + +static CamelIndexCursor * +text_index_names(CamelIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + + return (CamelIndexCursor *)camel_text_index_key_cursor_new((CamelTextIndex *)idx, p->name_index); +} + +static void +camel_text_index_class_init(CamelTextIndexClass *klass) +{ + CamelIndexClass *iklass = (CamelIndexClass *)klass; + + camel_text_index_parent = CAMEL_OBJECT_CLASS(camel_type_get_global_classfuncs(camel_object_get_type())); + + iklass->sync = text_index_sync; + iklass->compress = text_index_compress; + + iklass->rename = text_index_rename; + + iklass->has_name = text_index_has_name; + iklass->add_name = text_index_add_name; + iklass->write_name = text_index_write_name; + iklass->find_name = text_index_find_name; + iklass->delete_name = text_index_delete_name; + iklass->find = text_index_find; + + iklass->words = text_index_words; + iklass->names = text_index_names; +} + +static void +camel_text_index_init(CamelTextIndex *idx) +{ + struct _CamelTextIndexPrivate *p; + + p = CTI_PRIVATE(idx) = g_malloc0(sizeof(*p)); + + e_dlist_init(&p->word_cache); + p->words = g_hash_table_new(g_str_hash, g_str_equal); + p->word_cache_count = 0; + /* this cache size and the block cache size have been tuned for about the best + with moderate memory usage. Doubling the memory usage barely affects performance. */ + p->word_cache_limit = 4096; /* 1024 = 128K */ + +#ifdef ENABLE_THREADS + p->lock = e_mutex_new(E_MUTEX_REC); +#endif +} + +static void +camel_text_index_finalise(CamelTextIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + + camel_index_sync((CamelIndex *)idx); + + g_assert(e_dlist_empty(&p->word_cache)); + g_assert(g_hash_table_size(p->words) == 0); + + g_hash_table_destroy(p->words); + + if (p->word_index) + camel_object_unref((CamelObject *)p->word_index); + if (p->word_hash) + camel_object_unref((CamelObject *)p->word_hash); + if (p->name_index) + camel_object_unref((CamelObject *)p->name_index); + if (p->name_hash) + camel_object_unref((CamelObject *)p->name_hash); + + if (p->blocks) + camel_object_unref((CamelObject *)p->blocks); + if (p->links) + camel_object_unref((CamelObject *)p->links); + +#ifdef ENABLE_THREADS + e_mutex_destroy(p->lock); +#endif + + g_free(p); +} + +CamelType +camel_text_index_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelTextIndex", + sizeof (CamelTextIndex), + sizeof (CamelTextIndexClass), + (CamelObjectClassInitFunc) camel_text_index_class_init, + NULL, + (CamelObjectInitFunc) camel_text_index_init, + (CamelObjectFinalizeFunc) camel_text_index_finalise); + } + + return type; +} + +static char * +text_index_normalise(CamelIndex *idx, const char *in, void *data) +{ + char *word; + + /* Sigh, this is really epensive */ + word = g_utf8_normalize(in, strlen(in), G_NORMALIZE_ALL); + g_utf8_strdown(word); + + return word; +} + +/* Need flags? */ +CamelTextIndex * +camel_text_index_new(const char *path, int flags) +{ + CamelTextIndex *idx = (CamelTextIndex *)camel_object_new(camel_text_index_get_type()); + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + struct _CamelTextIndexRoot *rb; + char *link; + CamelBlock *bl; + + camel_index_construct((CamelIndex *)idx, path, flags); + camel_index_set_normalise((CamelIndex *)idx, text_index_normalise, NULL); + + p->blocks = camel_block_file_new(idx->parent.path, flags, CAMEL_TEXT_INDEX_VERSION, CAMEL_BLOCK_SIZE); + link = alloca(strlen(idx->parent.path)+7); + sprintf(link, "%s.data", idx->parent.path); + p->links = camel_key_file_new(link, flags, CAMEL_TEXT_INDEX_KEY_VERSION); + + if (p->blocks == NULL || p->links == NULL) { + camel_object_unref((CamelObject *)idx); + return NULL; + } + + rb = (struct _CamelTextIndexRoot *)p->blocks->root; + + if (rb->word_index_root == 0) { + bl = camel_block_file_new_block(p->blocks); + rb->word_index_root = bl->id; + camel_block_file_unref_block(p->blocks, bl); + } + + if (rb->word_hash_root == 0) { + bl = camel_block_file_new_block(p->blocks); + rb->word_hash_root = bl->id; + camel_block_file_unref_block(p->blocks, bl); + } + + if (rb->name_index_root == 0) { + bl = camel_block_file_new_block(p->blocks); + rb->name_index_root = bl->id; + camel_block_file_unref_block(p->blocks, bl); + } + + if (rb->name_hash_root == 0) { + bl = camel_block_file_new_block(p->blocks); + rb->name_hash_root = bl->id; + camel_block_file_unref_block(p->blocks, bl); + } + + p->word_index = camel_key_table_new(p->blocks, rb->word_index_root); + p->word_hash = camel_partition_table_new(p->blocks, rb->word_hash_root); + p->name_index = camel_key_table_new(p->blocks, rb->name_index_root); + p->name_hash = camel_partition_table_new(p->blocks, rb->name_hash_root); + + if (p->word_index == NULL || p->word_hash == NULL + || p->name_index == NULL || p->name_hash == NULL) { + camel_object_unref((CamelObject *)idx); + idx = NULL; + } + + return idx; +} + +/* returns 0 if the index exists, is valid, and synced, -1 otherwise */ +int +camel_text_index_check(const char *path) +{ + char *block, *key; + CamelBlockFile *blocks; + CamelKeyFile *keys; + + block = alloca(strlen(path)+7); + sprintf(block, "%s.index", path); + blocks = camel_block_file_new(block, O_RDONLY, CAMEL_TEXT_INDEX_VERSION, CAMEL_BLOCK_SIZE); + if (blocks == NULL) { + io(printf("Check failed: No block file: %s\n", strerror(errno))); + return -1; + } + key = alloca(strlen(path)+12); + sprintf(key, "%s.index.data", path); + keys = camel_key_file_new(key, O_RDONLY, CAMEL_TEXT_INDEX_KEY_VERSION); + if (keys == NULL) { + io(printf("Check failed: No key file: %s\n", strerror(errno))); + camel_object_unref((CamelObject *)blocks); + return -1; + } + + camel_object_unref((CamelObject *)keys); + camel_object_unref((CamelObject *)blocks); + + return 0; +} + +int +camel_text_index_rename(const char *old, const char *new) +{ + char *oldname, *newname; + int err; + + /* TODO: camel_text_index_rename should find out if we have an active index and use that instead */ + + oldname = alloca(strlen(old)+12); + newname = alloca(strlen(new)+12); + sprintf(oldname, "%s.index", old); + sprintf(newname, "%s.index", new); + + if (rename(oldname, newname) == -1) + return -1; + + sprintf(oldname, "%s.index.data", old); + sprintf(newname, "%s.index.data", new); + + if (rename(oldname, newname) == -1) { + err = errno; + sprintf(oldname, "%s.index", old); + sprintf(newname, "%s.index", new); + rename(newname, oldname); + errno = err; + return -1; + } + + return 0; +} + +int +camel_text_index_remove(const char *old) +{ + char *block, *key; + int ret = 0; + + /* TODO: needs to poke any active indices to remain unlinked */ + + block = alloca(strlen(old)+12); + key = alloca(strlen(old)+12); + sprintf(block, "%s.index", old); + sprintf(key, "%s.index.data", old); + + if (unlink(block) == -1 && errno != ENOENT) + ret = -1; + if (unlink(key) == -1 && errno != ENOENT) + ret = -1; + + return ret; +} + +/* Debug */ +void +camel_text_index_info(CamelTextIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + struct _CamelTextIndexRoot *rb = (struct _CamelTextIndexRoot *)p->blocks->root; + int frag; + + printf("Path: '%s'\n", idx->parent.path); + printf("Version: %d\n", idx->parent.version); + printf("Flags: %08x\n", idx->parent.flags); + printf("Total words: %d\n", rb->words); + printf("Total names: %d\n", rb->names); + printf("Total deleted: %d\n", rb->deleted); + printf("Total key blocks: %d\n", rb->keys); + + if (rb->words > 0) { + frag = ((rb->keys - rb->words) * 100)/ rb->words; + printf("Word fragmentation: %d%%\n", frag); + } + + if (rb->names > 0) { + frag = (rb->deleted * 100)/ rb->names; + printf("Name fragmentation: %d%%\n", frag); + } +} + +/* Debug */ +void +camel_text_index_dump(CamelTextIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + camel_key_t keyid; + char *word; + const char *name; + unsigned int flags; + camel_block_t data; + + /* Iterate over all names in the file first */ + + printf("UID's in index\n"); + + keyid = 0; + while ( (keyid = camel_key_table_next(p->name_index, keyid, &word, &flags, &data)) ) { + if ((flags & 1) == 0) + printf(" %s\n", word); + else + printf(" %s (deleted)\n", word); + g_free(word); + } + + printf("Word's in index\n"); + + keyid = 0; + while ( (keyid = camel_key_table_next(p->word_index, keyid, &word, &flags, &data)) ) { + CamelIndexCursor *idc; + + printf("Word: '%s':\n", word); + + idc = camel_index_find((CamelIndex *)idx, word); + while ( (name = camel_index_cursor_next(idc)) ) { + printf(" %s", name); + } + printf("\n"); + camel_object_unref((CamelObject *)idc); + g_free(word); + } +} + +/* more debug stuff */ +void +camel_text_index_validate(CamelTextIndex *idx) +{ + struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx); + camel_key_t keyid; + char *word; + const char *name; + unsigned int flags; + camel_block_t data; + char *oldword; + camel_key_t *records; + size_t count; + + GHashTable *names, *deleted, *words, *keys, *name_word, *word_word; + + names = g_hash_table_new(NULL, NULL); + deleted = g_hash_table_new(NULL, NULL); + + name_word = g_hash_table_new(g_str_hash, g_str_equal); + + words = g_hash_table_new(NULL, NULL); + keys = g_hash_table_new(NULL, NULL); + + word_word = g_hash_table_new(g_str_hash, g_str_equal); + + /* Iterate over all names in the file first */ + + printf("Checking UID consistency\n"); + + keyid = 0; + while ( (keyid = camel_key_table_next(p->name_index, keyid, &word, &flags, &data)) ) { + if ((oldword = g_hash_table_lookup(names, (void *)keyid)) != NULL + || (oldword = g_hash_table_lookup(deleted, (void *)keyid)) != NULL) { + printf("Warning, name '%s' duplicates key (%x) with name '%s'\n", word, keyid, oldword); + g_free(word); + } else { + g_hash_table_insert(name_word, word, (void *)1); + if ((flags & 1) == 0) { + g_hash_table_insert(names, (void *)keyid, word); + } else { + g_hash_table_insert(deleted, (void *)keyid, word); + } + } + } + + printf("Checking WORD member consistency\n"); + + keyid = 0; + while ( (keyid = camel_key_table_next(p->word_index, keyid, &word, &flags, &data)) ) { + CamelIndexCursor *idc; + GHashTable *used; + + /* first, check for duplicates of keyid, and data */ + if ((oldword = g_hash_table_lookup(words, (void *)keyid)) != NULL) { + printf("Warning, word '%s' duplicates key (%x) with name '%s'\n", word, keyid, oldword); + g_free(word); + word = oldword; + } else { + g_hash_table_insert(words, (void *)keyid, word); + } + + if (data == 0) { + /* This may not be an issue if things have been removed over time, + though it is a problem if its a fresh index */ + printf("Word '%s' has no data associated with it\n", word); + } else { + if ((oldword = g_hash_table_lookup(keys, (void *)data)) != NULL) { + printf("Warning, word '%s' duplicates data (%x) with name '%s'\n", word, data, oldword); + } else { + g_hash_table_insert(keys, (void *)data, word); + } + } + + if ((oldword = g_hash_table_lookup(word_word, word)) != NULL) { + printf("Warning, word '%s' occurs more than once\n", word); + } else { + g_hash_table_insert(word_word, word, word); + } + + used = g_hash_table_new(g_str_hash, g_str_equal); + + idc = camel_index_find((CamelIndex *)idx, word); + while ( (name = camel_index_cursor_next(idc)) ) { + if (g_hash_table_lookup(name_word, name) == NULL) { + printf("word '%s' references non-existant name '%s'\n", word, name); + } + if (g_hash_table_lookup(used, name) != NULL) { + printf("word '%s' uses word '%s' more than once\n", word, name); + } else { + g_hash_table_insert(used, g_strdup(name), (void *)1); + } + } + camel_object_unref((CamelObject *)idc); + + g_hash_table_foreach(used, (GHFunc)g_free, NULL); + g_hash_table_destroy(used); + + printf("word '%s'\n", word); + + while (data) { + printf(" data %x ", data); + if (camel_key_file_read(p->links, &data, &count, &records) == -1) { + printf("Warning, read failed for word '%s', at data '%d'\n", word, data); + data = 0; + } else { + printf("(%d)\n", count); + g_free(records); + } + } + } + + g_hash_table_destroy(names); + g_hash_table_destroy(deleted); + g_hash_table_destroy(words); + g_hash_table_destroy(keys); + + g_hash_table_foreach(name_word, (GHFunc)g_free, NULL); + g_hash_table_destroy(name_word); + + g_hash_table_foreach(word_word, (GHFunc)g_free, NULL); + g_hash_table_destroy(word_word); +} + +/* ********************************************************************** */ +/* CamelTextIndexName */ +/* ********************************************************************** */ + +static CamelIndexNameClass *camel_text_index_name_parent; + +#define CIN_CLASS(o) ((CamelTextIndexNameClass *)(((CamelObject *)o)->classfuncs)) +#define CIN_PRIVATE(o) (((CamelTextIndexName *)(o))->priv) + +static void +text_index_name_add_word(CamelIndexName *idn, const char *word) +{ + struct _CamelTextIndexNamePrivate *p = ((CamelTextIndexName *)idn)->priv; + + if (g_hash_table_lookup(idn->words, word) == NULL) { + char *w = e_mempool_strdup(p->pool, word); + + g_hash_table_insert(idn->words, w, w); + } +} + +/* Why? + Because it doesn't hang/loop forever on bad data + Used to clean up utf8 before it gets further */ + +static __inline__ guint32 +camel_utf8_next(const unsigned char **ptr, const unsigned char *ptrend) +{ + register unsigned char *p = (unsigned char *)*ptr; + register unsigned int c; + register guint32 v; + int l; + + if (p == ptrend) + return 0; + + while ( (c = *p++) ) { + if (c < 0x80) { + *ptr = p; + return c; + } else if ((c&0xe0) == 0xc0) { + v = c & 0x1f; + l = 1; + } else if ((c&0xf0) == 0xe0) { + v = c & 0x0f; + l = 2; + } else if ((c&0xf8) == 0xf0) { + v = c & 0x07; + l = 3; + } else if ((c&0xfc) == 0xf8) { + v = c & 0x03; + l = 4; + } else if ((c&0xfe) == 0xfc) { + v = c & 0x01; + l = 5; + } else + /* Invalid, ignore and look for next start char if room */ + if (p == ptrend) { + return 0; + } else { + continue; + } + + /* bad data or truncated buffer */ + if (p + l > ptrend) + return 0; + + while (l && ((c = *p) & 0xc0) == 0x80) { + p++; + l--; + v = (v << 6) | (c & 0x3f); + } + + /* valid char */ + if (l == 0) { + *ptr = p; + return v; + } + + /* else look for a start char again */ + } + + return 0; +} + +static size_t +text_index_name_add_buffer(CamelIndexName *idn, const char *buffer, size_t len) +{ + CamelTextIndexNamePrivate *p = CIN_PRIVATE(idn); + const unsigned char *ptr, *ptrend; + guint32 c; + unsigned char utf8[8]; + size_t utf8len; + + if (buffer == NULL) { + if (p->buffer->len) { + camel_index_name_add_word(idn, p->buffer->str); + g_string_truncate(p->buffer, 0); + } + return 0; + } + + ptr = buffer; + ptrend = buffer+len; + while ((c = camel_utf8_next(&ptr, ptrend))) { + if (g_unichar_isalnum(c)) { + c = g_unichar_tolower(c); + utf8len = g_unichar_to_utf8(c, utf8); + utf8[utf8len] = 0; + g_string_append(p->buffer, utf8); + } else { + if (p->buffer->len) { + text_index_name_add_word(idn, p->buffer->str); + /*camel_index_name_add_word(idn, p->buffer->str);*/ + g_string_truncate(p->buffer, 0); + } + } + } + + return 0; +} + +static void +camel_text_index_name_class_init(CamelTextIndexNameClass *klass) +{ + CamelIndexNameClass *nklass = (CamelIndexNameClass *)klass; + + camel_text_index_name_parent = CAMEL_INDEX_NAME_CLASS(camel_type_get_global_classfuncs(camel_index_name_get_type())); + + nklass->add_word = text_index_name_add_word; + nklass->add_buffer = text_index_name_add_buffer; +} + +static void +camel_text_index_name_init(CamelTextIndexName *idn) +{ + CamelTextIndexNamePrivate *p; + + idn->parent.words = g_hash_table_new(g_str_hash, g_str_equal); + + p = idn->priv = g_malloc0(sizeof(*idn->priv)); + p->buffer = g_string_new(""); + p->pool = e_mempool_new(256, 128, E_MEMPOOL_ALIGN_BYTE); +} + +static void +camel_text_index_name_finalise(CamelTextIndexName *idn) +{ + CamelTextIndexNamePrivate *p = CIN_PRIVATE(idn); + + g_hash_table_destroy(idn->parent.words); + + g_string_free(p->buffer, TRUE); + e_mempool_destroy(p->pool); + + g_free(p); +} + +CamelType +camel_text_index_name_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelTextIndexName", + sizeof (CamelTextIndexName), + sizeof (CamelTextIndexNameClass), + (CamelObjectClassInitFunc) camel_text_index_name_class_init, + NULL, + (CamelObjectInitFunc) camel_text_index_name_init, + (CamelObjectFinalizeFunc) camel_text_index_name_finalise); + } + + return type; +} + +CamelTextIndexName * +camel_text_index_name_new(CamelTextIndex *idx, const char *name, camel_key_t nameid) +{ + CamelTextIndexName *idn = (CamelTextIndexName *)camel_object_new(camel_text_index_name_get_type()); + CamelIndexName *cin = &idn->parent; + CamelTextIndexNamePrivate *p = CIN_PRIVATE(idn); + + cin->index = (CamelIndex *)idx; + camel_object_ref((CamelObject *)idx); + cin->name = g_strdup(name); + p->nameid = nameid; + + return idn; +} + +/* ********************************************************************** */ +/* CamelTextIndexCursor */ +/* ********************************************************************** */ + +static CamelIndexCursorClass *camel_text_index_cursor_parent; + +#define CIC_CLASS(o) ((CamelTextIndexCursorClass *)(((CamelObject *)o)->classfuncs)) +#define CIC_PRIVATE(o) (((CamelTextIndexCursor *)(o))->priv) + +static const char * +text_index_cursor_next(CamelIndexCursor *idc) +{ + struct _CamelTextIndexCursorPrivate *p = CIC_PRIVATE(idc); + struct _CamelTextIndexPrivate *tip = CTI_PRIVATE(idc->index); + unsigned int flags; + + c(printf("Going to next cursor for word with data '%08x' next %08x\n", p->first, p->next)); + + do { + while (p->record_index >= p->record_count) { + g_free(p->records); + p->records = NULL; + p->record_index = 0; + p->record_count = 0; + if (p->next == 0) + return NULL; + if (camel_key_file_read(tip->links, &p->next, &p->record_count, &p->records) == -1) + return NULL; + } + + g_free(p->current); + camel_key_table_lookup(tip->name_index, p->records[p->record_index], &p->current, &flags); + if (flags & 1) { + g_free(p->current); + p->current = NULL; + } + p->record_index++; + } while (p->current == NULL); + + return p->current; +} + +static void +text_index_cursor_reset(CamelIndexCursor *idc) +{ + struct _CamelTextIndexCursorPrivate *p = CIC_PRIVATE(idc); + + g_free(p->records); + g_free(p->current); + p->record_count = 0; + p->record_index = 0; + p->next = p->first; +} + +static void +camel_text_index_cursor_class_init(CamelTextIndexCursorClass *klass) +{ + CamelIndexCursorClass *cklass = (CamelIndexCursorClass *)klass; + + camel_text_index_cursor_parent = CAMEL_INDEX_CURSOR_CLASS(camel_type_get_global_classfuncs(camel_index_cursor_get_type())); + + cklass->next = text_index_cursor_next; + cklass->reset = text_index_cursor_reset; +} + +static void +camel_text_index_cursor_init(CamelTextIndexCursor *idc) +{ + CIC_PRIVATE(idc) = g_malloc0(sizeof(struct _CamelTextIndexCursorPrivate)); +} + +static void +camel_text_index_cursor_finalise(CamelTextIndexCursor *idc) +{ + struct _CamelTextIndexCursorPrivate *p = CIC_PRIVATE(idc); + + g_free(p->records); + g_free(p->current); + g_free(p); +} + +CamelType +camel_text_index_cursor_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelTextIndexCursor", + sizeof (CamelTextIndexCursor), + sizeof (CamelTextIndexCursorClass), + (CamelObjectClassInitFunc) camel_text_index_cursor_class_init, + NULL, + (CamelObjectInitFunc) camel_text_index_cursor_init, + (CamelObjectFinalizeFunc) camel_text_index_cursor_finalise); + } + + return type; +} + +CamelTextIndexCursor * +camel_text_index_cursor_new(CamelTextIndex *idx, camel_block_t data) +{ + CamelTextIndexCursor *idc = (CamelTextIndexCursor *)camel_object_new(camel_text_index_cursor_get_type()); + CamelIndexCursor *cic = &idc->parent; + struct _CamelTextIndexCursorPrivate *p = CIC_PRIVATE(idc); + + cic->index = (CamelIndex *)idx; + camel_object_ref((CamelObject *)idx); + p->first = data; + p->next = data; + p->record_count = 0; + p->record_index = 0; + + return idc; +} + +/* ********************************************************************** */ +/* CamelTextIndexKeyCursor */ +/* ********************************************************************** */ + +static CamelIndexCursorClass *camel_text_index_key_cursor_parent; + +#define CIKC_CLASS(o) ((CamelTextIndexKeyCursorClass *)(((CamelObject *)o)->classfuncs)) +#define CIKC_PRIVATE(o) (((CamelTextIndexKeyCursor *)(o))->priv) + +static const char * +text_index_key_cursor_next(CamelIndexCursor *idc) +{ + struct _CamelTextIndexKeyCursorPrivate *p = CIKC_PRIVATE(idc); + + c(printf("Going to next cursor for keyid %08x\n", p->keyid)); + + g_free(p->current); + p->current = NULL; + + while ( (p->keyid = camel_key_table_next(p->table, p->keyid, &p->current, &p->flags, &p->data)) ) { + if ((p->flags & 1) == 0) { + return p->current; + } else { + g_free(p->current); + p->current = NULL; + } + } + + return NULL; +} + +static void +text_index_key_cursor_reset(CamelIndexCursor *idc) +{ + struct _CamelTextIndexKeyCursorPrivate *p = CIKC_PRIVATE(idc); + + p->keyid = 0; + p->flags = 0; + p->data = 0; + g_free(p->current); + p->current = NULL; +} + +static void +camel_text_index_key_cursor_class_init(CamelTextIndexKeyCursorClass *klass) +{ + CamelIndexCursorClass *cklass = (CamelIndexCursorClass *)klass; + + camel_text_index_key_cursor_parent = CAMEL_INDEX_CURSOR_CLASS(camel_type_get_global_classfuncs(camel_index_cursor_get_type())); + + cklass->next = text_index_key_cursor_next; + cklass->reset = text_index_key_cursor_reset; +} + +static void +camel_text_index_key_cursor_init(CamelTextIndexKeyCursor *idc) +{ + struct _CamelTextIndexKeyCursorPrivate *p; + + p = idc->priv = g_malloc0(sizeof(struct _CamelTextIndexKeyCursorPrivate)); + p->keyid = 0; + p->flags = 0; + p->data = 0; + p->current = NULL; +} + +static void +camel_text_index_key_cursor_finalise(CamelTextIndexKeyCursor *idc) +{ + struct _CamelTextIndexKeyCursorPrivate *p = CIKC_PRIVATE(idc); + + g_free(p->current); + if (p->table) + camel_object_unref((CamelObject *)p->table); + g_free(p); +} + +CamelType +camel_text_index_key_cursor_get_type(void) +{ + static CamelType type = CAMEL_INVALID_TYPE; + + if (type == CAMEL_INVALID_TYPE) { + type = camel_type_register(camel_object_get_type(), "CamelTextIndexKeyCursor", + sizeof (CamelTextIndexKeyCursor), + sizeof (CamelTextIndexKeyCursorClass), + (CamelObjectClassInitFunc) camel_text_index_key_cursor_class_init, + NULL, + (CamelObjectInitFunc) camel_text_index_key_cursor_init, + (CamelObjectFinalizeFunc) camel_text_index_key_cursor_finalise); + } + + return type; +} + +CamelTextIndexKeyCursor * +camel_text_index_key_cursor_new(CamelTextIndex *idx, CamelKeyTable *table) +{ + CamelTextIndexKeyCursor *idc = (CamelTextIndexKeyCursor *)camel_object_new(camel_text_index_key_cursor_get_type()); + CamelIndexCursor *cic = &idc->parent; + struct _CamelTextIndexKeyCursorPrivate *p = CIKC_PRIVATE(idc); + + cic->index = (CamelIndex *)idx; + camel_object_ref((CamelObject *)idx); + p->table = table; + camel_object_ref((CamelObject *)table); + + return idc; +} + +/* ********************************************************************** */ + +#define m(x) + +#if 0 + +struct _CamelIndexRoot { + struct _CamelBlockRoot root; + + camel_block_t word_root; /* a keyindex containing the keyid -> word mapping */ + camel_block_t word_hash_root; /* a partitionindex containing word -> keyid mapping */ + + camel_block_t name_root; /* same, for names */ + camel_block_t name_hash_root; +}; + +char wordbuffer[] = "This is a buffer of multiple words. Some of the words are duplicates" +" while other words are the same, some are in difFerenT Different different case cAsE casE," +" with,with:with;with-with'with\"'\"various punctuation as well. So much for those Words. and 10" +" numbers in a row too 1,2,3,4,5,6,7,8,9,10! Yay!."; + +int main(int argc, char **argv) +{ +#if 0 + CamelBlockFile *bs; + CamelKeyTable *ki; + CamelPartitionTable *cpi; + CamelBlock *keyroot, *partroot; + struct _CamelIndexRoot *root; + FILE *fp; + char line[256], *key; + camel_key_t keyid; + int index = 0, flags, data; +#endif + CamelIndex *idx; + CamelIndexName *idn; + CamelIndexCursor *idc; + const char *word; + int i; + + printf("Camel text index tester!\n"); + + g_thread_init(NULL); + camel_init(NULL, 0); + + idx = (CamelIndex *)camel_text_index_new("textindex", O_CREAT|O_RDWR|O_TRUNC); + + +#if 1 + camel_index_compress(idx); + + return 0; +#endif + + for (i=0;i<100;i++) { + char name[16]; + + sprintf(name, "%d", i); + printf("Adding words to name '%s'\n", name); + idn = camel_index_add_name(idx, name); + camel_index_name_add_buffer(idn, wordbuffer, sizeof(wordbuffer)-1); + camel_index_write_name(idx, idn); + camel_object_unref((CamelObject *)idn); + } + + printf("Looking up which names contain word 'word'\n"); + idc = camel_index_find(idx, "words"); + while ( (word = camel_index_cursor_next(idc)) != NULL ) { + printf(" name is '%s'\n", word); + } + camel_object_unref((CamelObject *)idc); + printf("done.\n"); + + printf("Looking up which names contain word 'truncate'\n"); + idc = camel_index_find(idx, "truncate"); + while ( (word = camel_index_cursor_next(idc)) != NULL ) { + printf(" name is '%s'\n", word); + } + camel_object_unref((CamelObject *)idc); + printf("done.\n"); + + camel_index_sync(idx); + camel_object_unref((CamelObject *)idx); + +#if 0 + bs = camel_block_file_new("blocks", "TESTINDX", CAMEL_BLOCK_SIZE); + + root = (struct _CamelIndexRoot *)bs->root; + if (root->word_root == 0) { + keyroot = camel_block_file_new_block(bs); + root->word_root = keyroot->id; + camel_block_file_touch_block(bs, bs->root_block); + } + if (root->word_hash_root == 0) { + partroot = camel_block_file_new_block(bs); + root->word_hash_root = partroot->id; + camel_block_file_touch_block(bs, bs->root_block); + } + + ki = camel_key_table_new(bs, root->word_root); + cpi = camel_partition_table_new(bs, root->word_hash_root); + + fp = fopen("/usr/dict/words", "r"); + if (fp == NULL) { + perror("fopen"); + return 1; + } + + while (fgets(line, sizeof(line), fp) != NULL) { + line[strlen(line)-1] = 0; + + /* see if its already there */ + keyid = camel_partition_table_lookup(cpi, line); + if (keyid == 0) { + m(printf("Adding word '%s' %d\n", line, index)); + + keyid = camel_key_table_add(ki, line, index, 0); + m(printf(" key = %08x\n", keyid)); + + camel_partition_table_add(cpi, line, keyid); + + m(printf("Lookup word '%s'\n", line)); + keyid = camel_partition_table_lookup(cpi, line); + m(printf(" key = %08x\n", keyid)); + } + + m(printf("Lookup key %08x\n", keyid)); + + camel_key_table_set_flags(ki, keyid, index, 1); + + data = camel_key_table_lookup(ki, keyid, &key, &flags); + m(printf(" word = '%s' %d %04x\n", key, data, flags)); + + g_assert(data == index && strcmp(key, line) == 0); + + g_free(key); + + index++; + } + + printf("Scanning again\n"); + fseek(fp, SEEK_SET, 0); + index = 0; + while (fgets(line, sizeof(line), fp) != NULL) { + line[strlen(line)-1] = 0; + m(printf("Lookup word '%s' %d\n", line, index)); + keyid = camel_partition_table_lookup(cpi, line); + m(printf(" key = %08d\n", keyid)); + + m(printf("Lookup key %08x\n", keyid)); + data = camel_key_table_lookup(ki, keyid, &key, &flags); + m(printf(" word = '%s' %d\n", key, data)); + + g_assert(data == index && strcmp(key, line) == 0); + + g_free(key); + + index++; + } + + printf("Freeing partition index\n"); + camel_partition_table_free(cpi); + + printf("Syncing block file\n"); + camel_block_file_sync(bs); +#endif + return 0; +} + +#endif diff --git a/camel/camel-text-index.h b/camel/camel-text-index.h new file mode 100644 index 0000000000..1982321502 --- /dev/null +++ b/camel/camel-text-index.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2001 Ximian Inc. + * + * Authors: Michael Zucchi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef _CAMEL_TEXT_INDEX_H +#define _CAMEL_TEXT_INDEX_H + +#include +#include +#include "camel-index.h" + +#define CAMEL_TEXT_INDEX(obj) CAMEL_CHECK_CAST (obj, camel_text_index_get_type (), CamelTextIndex) +#define CAMEL_TEXT_INDEX_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_text_index_get_type (), CamelTextIndexClass) +#define CAMEL_IS_TEXT_INDEX(obj) CAMEL_CHECK_TYPE (obj, camel_text_index_get_type ()) + +typedef struct _CamelTextIndex CamelTextIndex; +typedef struct _CamelTextIndexClass CamelTextIndexClass; + +typedef struct _CamelTextIndexName CamelTextIndexName; +typedef struct _CamelTextIndexNameClass CamelTextIndexNameClass; + +typedef struct _CamelTextIndexCursor CamelTextIndexCursor; +typedef struct _CamelTextIndexCursorClass CamelTextIndexCursorClass; + +typedef struct _CamelTextIndexKeyCursor CamelTextIndexKeyCursor; +typedef struct _CamelTextIndexKeyCursorClass CamelTextIndexKeyCursorClass; + +typedef void (*CamelTextIndexFunc)(CamelTextIndex *idx, const char *word, char *buffer); + +/* ********************************************************************** */ + +struct _CamelTextIndexCursor { + CamelIndexCursor parent; + + struct _CamelTextIndexCursorPrivate *priv; +}; + +struct _CamelTextIndexCursorClass { + CamelIndexCursorClass parent; +}; + +CamelType camel_text_index_cursor_get_type(void); + +/* ********************************************************************** */ + +struct _CamelTextIndexKeyCursor { + CamelIndexCursor parent; + + struct _CamelTextIndexKeyCursorPrivate *priv; +}; + +struct _CamelTextIndexKeyCursorClass { + CamelIndexCursorClass parent; +}; + +CamelType camel_text_index_key_cursor_get_type(void); + +/* ********************************************************************** */ + +struct _CamelTextIndexName { + CamelIndexName parent; + + struct _CamelTextIndexNamePrivate *priv; +}; + +struct _CamelTextIndexNameClass { + CamelIndexNameClass parent; +}; + +CamelType camel_text_index_name_get_type(void); + +/* ********************************************************************** */ + +struct _CamelTextIndex { + CamelIndex parent; + + struct _CamelTextIndexPrivate *priv; +}; + +struct _CamelTextIndexClass { + CamelIndexClass parent_class; +}; + +guint camel_text_index_get_type (void); +CamelTextIndex *camel_text_index_new(const char *path, int flags); + +/* static utility functions */ +int camel_text_index_check(const char *path); +int camel_text_index_rename(const char *old, const char *new); +int camel_text_index_remove(const char *old); + +#endif /* ! _CAMEL_TEXT_INDEX_H */ + diff --git a/camel/providers/local/camel-local-folder.c b/camel/providers/local/camel-local-folder.c index d84041025c..0344fc8952 100644 --- a/camel/providers/local/camel-local-folder.c +++ b/camel/providers/local/camel-local-folder.c @@ -50,6 +50,8 @@ #include "camel-local-private.h" +#include "camel-text-index.h" + #define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ #ifndef PATH_MAX @@ -138,9 +140,8 @@ local_finalize(CamelObject * object) camel_object_unref((CamelObject *)local_folder->search); } - /* must free index after summary, since it isn't refcounted */ if (local_folder->index) - ibex_close(local_folder->index); + camel_object_unref((CamelObject *)local_folder->index); while (local_folder->locked> 0) camel_local_folder_unlock(local_folder); @@ -211,10 +212,17 @@ camel_local_folder_construct(CamelLocalFolder *lf, CamelStore *parent_store, con lf->changes = camel_folder_change_info_new(); - /* if we have no index file, force it */ - forceindex = stat(lf->index_path, &st) == -1; + /* TODO: Remove the following line, it is a temporary workaround to remove + the old-format 'ibex' files that might be lying around */ + unlink(lf->index_path); + + /* if we have no/invalid index file, force it */ + forceindex = camel_text_index_check(lf->index_path) == -1; if (flags & CAMEL_STORE_FOLDER_BODY_INDEX) { - lf->index = ibex_open(lf->index_path, O_CREAT | O_RDWR, 0600); + int flag = O_RDWR|O_CREAT; + if (forceindex) + flag |= O_TRUNC; + lf->index = (CamelIndex *)camel_text_index_new(lf->index_path, flag); if (lf->index == NULL) { /* yes, this isn't fatal at all */ g_warning("Could not open/create index file: %s: indexing not performed", strerror(errno)); @@ -223,10 +231,9 @@ camel_local_folder_construct(CamelLocalFolder *lf, CamelStore *parent_store, con flags &= ~CAMEL_STORE_FOLDER_BODY_INDEX; } } else { - /* if we do have an index file, remove it */ - if (forceindex == FALSE) { - unlink(lf->index_path); - } + /* if we do have an index file, remove it (?) */ + if (forceindex == FALSE) + camel_text_index_remove(lf->index_path); forceindex = FALSE; } @@ -237,7 +244,9 @@ camel_local_folder_construct(CamelLocalFolder *lf, CamelStore *parent_store, con camel_exception_clear(ex); } - if (camel_local_summary_check((CamelLocalSummary *)folder->summary, lf->changes, ex) == -1) { + /*if (camel_local_summary_check((CamelLocalSummary *)folder->summary, lf->changes, ex) == -1) {*/ + /* we sync here so that any hard work setting up the folder isn't lost */ + if (camel_local_summary_sync((CamelLocalSummary *)folder->summary, FALSE, lf->changes, ex) == -1) { camel_object_unref (CAMEL_OBJECT (folder)); return NULL; } diff --git a/camel/providers/local/camel-local-folder.h b/camel/providers/local/camel-local-folder.h index 3eb4b02c32..c958bde835 100644 --- a/camel/providers/local/camel-local-folder.h +++ b/camel/providers/local/camel-local-folder.h @@ -29,7 +29,7 @@ extern "C" { #include #include -#include +#include #include "camel-local-summary.h" #include "camel-lock.h" @@ -54,7 +54,7 @@ typedef struct { char *summary_path; /* where the summary lives */ char *index_path; /* where the index file lives */ - ibex *index; /* index for this folder */ + CamelIndex *index; /* index for this folder */ CamelFolderSearch *search; /* used to run searches, we just use the real thing (tm) */ CamelFolderChangeInfo *changes; /* used to store changes to the folder during processing */ } CamelLocalFolder; @@ -65,7 +65,7 @@ typedef struct { /* Virtual methods */ /* summary factory, only used at init */ - CamelLocalSummary *(*create_summary)(const char *path, const char *folder, ibex *index); + CamelLocalSummary *(*create_summary)(const char *path, const char *folder, CamelIndex *index); /* Lock the folder for my operations */ int (*lock)(CamelLocalFolder *, CamelLockType type, CamelException *ex); diff --git a/camel/providers/local/camel-local-store.c b/camel/providers/local/camel-local-store.c index daa70979dc..0e7be5f48b 100644 --- a/camel/providers/local/camel-local-store.c +++ b/camel/providers/local/camel-local-store.c @@ -38,6 +38,7 @@ #include "camel-url.h" #include "camel-local-folder.h" +#include #define d(x) @@ -321,11 +322,12 @@ rename_folder(CamelStore *store, const char *old, const char *new, CamelExceptio CAMEL_STORE_LOCK(store, cache_lock); folder = g_hash_table_lookup(store->folders, old); - if (folder) { - if (folder->index && ibex_move(folder->index, newibex) == -1) + if (folder && folder->index) { + if (folder->index && camel_index_rename(folder->index, newibex) == -1) goto ibex_failed; } else { - if (xrename(old, new, path, ".ibex", TRUE, ex)) + /* TODO: camel_text_index_rename should find out if we have an active index itself? */ + if (camel_text_index_rename(oldibex, newibex) == -1) goto ibex_failed; } @@ -348,10 +350,9 @@ base_failed: summary_failed: if (folder) { if (folder->index) - ibex_move(folder->index, oldibex); + camel_index_rename(folder->index, oldibex); } else - xrename(new, old, path, ".ibex", TRUE, ex); - + camel_text_index_rename(newibex, oldibex); ibex_failed: CAMEL_STORE_UNLOCK(store, cache_lock); g_free(newibex); @@ -379,7 +380,7 @@ delete_folder(CamelStore *store, const char *folder_name, CamelException *ex) } g_free(str); str = g_strdup_printf("%s.ibex", name); - if (unlink(str) == -1 && errno != ENOENT) { + if (camel_text_index_remove(str) == -1 && errno != ENOENT) { camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Could not delete folder index file `%s': %s"), str, strerror(errno)); diff --git a/camel/providers/local/camel-local-summary.c b/camel/providers/local/camel-local-summary.c index 508c7e0831..eae265d5a6 100644 --- a/camel/providers/local/camel-local-summary.c +++ b/camel/providers/local/camel-local-summary.c @@ -118,16 +118,20 @@ camel_local_summary_finalise(CamelObject *obj) { CamelLocalSummary *mbs = CAMEL_LOCAL_SUMMARY(obj); + if (mbs->index) + camel_object_unref((CamelObject *)mbs->index); g_free(mbs->folder_path); } void -camel_local_summary_construct(CamelLocalSummary *new, const char *filename, const char *local_name, ibex *index) +camel_local_summary_construct(CamelLocalSummary *new, const char *filename, const char *local_name, CamelIndex *index) { camel_folder_summary_set_build_content(CAMEL_FOLDER_SUMMARY(new), FALSE); camel_folder_summary_set_filename(CAMEL_FOLDER_SUMMARY(new), filename); new->folder_path = g_strdup(local_name); new->index = index; + if (index) + camel_object_ref((CamelObject *)index); } static int @@ -368,7 +372,7 @@ local_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeIn g_warning("Could not save summary for %s: %s", cls->folder_path, strerror(errno)); } - if (cls->index && ibex_save(cls->index) == -1) + if (cls->index && camel_index_sync(cls->index) == -1) g_warning("Could not sync index for %s: %s", cls->folder_path, strerror(errno)); return ret; @@ -566,7 +570,7 @@ message_info_new(CamelFolderSummary *s, struct _header_raw *h) if (cls->index && (doindex || cls->index_force - || !ibex_contains_name(cls->index, (char *)camel_message_info_uid(mi)))) { + || !camel_index_has_name(cls->index, camel_message_info_uid(mi)))) { d(printf("Am indexing message %s\n", camel_message_info_uid(mi))); camel_folder_summary_set_index(s, cls->index); } else { diff --git a/camel/providers/local/camel-local-summary.h b/camel/providers/local/camel-local-summary.h index e9c570deb1..4a4db952d2 100644 --- a/camel/providers/local/camel-local-summary.h +++ b/camel/providers/local/camel-local-summary.h @@ -24,7 +24,7 @@ #include #include #include -#include +#include #define CAMEL_LOCAL_SUMMARY(obj) CAMEL_CHECK_CAST (obj, camel_local_summary_get_type (), CamelLocalSummary) #define CAMEL_LOCAL_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_local_summary_get_type (), CamelLocalSummaryClass) @@ -46,7 +46,7 @@ struct _CamelLocalSummary { char *folder_path; /* name of matching folder */ - ibex *index; + CamelIndex *index; int index_force; /* do we force index during creation? */ }; @@ -63,7 +63,7 @@ struct _CamelLocalSummaryClass { }; guint camel_local_summary_get_type (void); -void camel_local_summary_construct (CamelLocalSummary *new, const char *filename, const char *local_name, ibex *index); +void camel_local_summary_construct (CamelLocalSummary *new, const char *filename, const char *local_name, CamelIndex *index); /* load/check the summary */ int camel_local_summary_load(CamelLocalSummary *cls, int forceindex, CamelException *ex); diff --git a/camel/providers/local/camel-maildir-folder.c b/camel/providers/local/camel-maildir-folder.c index 5336cb39c3..7bf998bff9 100644 --- a/camel/providers/local/camel-maildir-folder.c +++ b/camel/providers/local/camel-maildir-folder.c @@ -50,7 +50,7 @@ static CamelLocalFolderClass *parent_class = NULL; #define CF_CLASS(so) CAMEL_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(so)) #define CMAILDIRS_CLASS(so) CAMEL_STORE_CLASS (CAMEL_OBJECT_GET_CLASS(so)) -static CamelLocalSummary *maildir_create_summary(const char *path, const char *folder, ibex *index); +static CamelLocalSummary *maildir_create_summary(const char *path, const char *folder, CamelIndex *index); static void maildir_append_message(CamelFolder * folder, CamelMimeMessage * message, const CamelMessageInfo *info, CamelException * ex); static CamelMimeMessage *maildir_get_message(CamelFolder * folder, const gchar * uid, CamelException * ex); @@ -120,7 +120,7 @@ camel_maildir_folder_new(CamelStore *parent_store, const char *full_name, guint3 return folder; } -static CamelLocalSummary *maildir_create_summary(const char *path, const char *folder, ibex *index) +static CamelLocalSummary *maildir_create_summary(const char *path, const char *folder, CamelIndex *index) { return (CamelLocalSummary *)camel_maildir_summary_new(path, folder, index); } diff --git a/camel/providers/local/camel-maildir-summary.c b/camel/providers/local/camel-maildir-summary.c index 1cfcf50a88..b0bd172257 100644 --- a/camel/providers/local/camel-maildir-summary.c +++ b/camel/providers/local/camel-maildir-summary.c @@ -22,7 +22,9 @@ #include #endif +#include #include +#include #include #include #include @@ -36,6 +38,7 @@ #include "camel-maildir-summary.h" #include +#include #include "camel-private.h" #include "e-util/e-memory.h" @@ -152,7 +155,7 @@ camel_maildir_summary_finalise(CamelObject *obj) * * Return value: A new #CamelMaildirSummary object. **/ -CamelMaildirSummary *camel_maildir_summary_new (const char *filename, const char *maildirdir, ibex *index) +CamelMaildirSummary *camel_maildir_summary_new (const char *filename, const char *maildirdir, CamelIndex *index) { CamelMaildirSummary *o = (CamelMaildirSummary *)camel_object_new(camel_maildir_summary_get_type ()); @@ -457,7 +460,7 @@ static int camel_maildir_summary_add(CamelLocalSummary *cls, const char *name, i mp = camel_mime_parser_new(); camel_mime_parser_scan_from(mp, FALSE); camel_mime_parser_init_with_fd(mp, fd); - if (cls->index && (forceindex || !ibex_contains_name(cls->index, (char *)name))) { + if (cls->index && (forceindex || !camel_index_has_name(cls->index, name))) { d(printf("forcing indexing of message content\n")); camel_folder_summary_set_index((CamelFolderSummary *)maildirs, cls->index); } else { @@ -477,7 +480,7 @@ remove_summary(char *key, CamelMessageInfo *info, CamelLocalSummary *cls) { d(printf("removing message %s from summary\n", key)); if (cls->index) - ibex_unindex(cls->index, (char *)camel_message_info_uid(info)); + camel_index_delete_name(cls->index, camel_message_info_uid(info)); camel_folder_summary_remove((CamelFolderSummary *)cls, info); camel_folder_summary_info_free((CamelFolderSummary *)cls, info); } @@ -507,7 +510,7 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca CamelMaildirMessageInfo *mdi; CamelFolderSummary *s = (CamelFolderSummary *)cls; GHashTable *left; - int i, count; + int i, count, total; int forceindex; char *new, *cur; char *uid; @@ -519,6 +522,8 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca d(printf("checking summary ...\n")); + camel_operation_start(NULL, _("Checking folder consistency")); + /* scan the directory, check for mail files not in the index, or index entries that no longer exist */ dir = opendir(cur); @@ -526,6 +531,7 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca camel_exception_setv(ex, 1, _("Cannot open maildir directory path: %s: %s"), cls->folder_path, strerror(errno)); g_free(cur); g_free(new); + camel_operation_end(NULL); return -1; } @@ -540,7 +546,19 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca } } + /* joy, use this to pre-count the total, so we can report progress meaningfully */ + total = 0; + count = 0; + while ( (d = readdir(dir)) ) + total++; + rewinddir(dir); + while ( (d = readdir(dir)) ) { + int pc = count * 100 / total; + + camel_operation_progress(NULL, pc); + count++; + /* FIXME: also run stat to check for regular file */ p = d->d_name; if (p[0] == '.') @@ -554,7 +572,7 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca uid = g_strdup(d->d_name); info = camel_folder_summary_uid((CamelFolderSummary *)cls, uid); - if (info == NULL || (cls->index && (!ibex_contains_name(cls->index, uid)))) { + if (info == NULL || (cls->index && (!camel_index_has_name(cls->index, uid)))) { /* need to add this file to the summary */ if (info != NULL) { CamelMessageInfo *old = g_hash_table_lookup(left, camel_message_info_uid(info)); @@ -606,12 +624,26 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca g_hash_table_foreach(left, (GHFunc)remove_summary, cls); g_hash_table_destroy(left); + camel_operation_end(NULL); + + camel_operation_start(NULL, _("Checking for new messages")); + /* now, scan new for new messages, and copy them to cur, and so forth */ dir = opendir(new); if (dir != NULL) { + total = 0; + count = 0; + while ( (d = readdir(dir)) ) + total++; + rewinddir(dir); + while ( (d = readdir(dir)) ) { char *name, *newname, *destname, *destfilename; char *src, *dest; + int pc = count * 100 / total; + + camel_operation_progress(NULL, pc); + count++; name = d->d_name; if (name[0] == '.') @@ -650,6 +682,7 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca g_free(src); g_free(dest); } + camel_operation_end(NULL); } closedir(dir); @@ -682,8 +715,12 @@ maildir_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChange if (camel_local_summary_check(cls, changes, ex) == -1) return -1; + camel_operation_start(NULL, _("Storing folder")); + count = camel_folder_summary_count((CamelFolderSummary *)cls); for (i=count-1;i>=0;i--) { + camel_operation_progress(NULL, (count-i)*100/count); + info = camel_folder_summary_index((CamelFolderSummary *)cls, i); mdi = (CamelMaildirMessageInfo *)info; if (info && (info->flags & CAMEL_MESSAGE_DELETED) && expunge) { @@ -693,7 +730,7 @@ maildir_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChange /* FIXME: put this in folder_summary::remove()? */ if (cls->index) - ibex_unindex(cls->index, (char *)camel_message_info_uid(info)); + camel_index_delete_name(cls->index, camel_message_info_uid(info)); camel_folder_change_info_remove_uid(changes, camel_message_info_uid(info)); camel_folder_summary_remove((CamelFolderSummary *)cls, info); @@ -748,6 +785,8 @@ maildir_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChange camel_folder_summary_info_free((CamelFolderSummary *)cls, info); } + camel_operation_end(NULL); + return ((CamelLocalSummaryClass *)parent_class)->sync(cls, expunge, changes, ex); } diff --git a/camel/providers/local/camel-maildir-summary.h b/camel/providers/local/camel-maildir-summary.h index 5362795ab2..0cae785c6c 100644 --- a/camel/providers/local/camel-maildir-summary.h +++ b/camel/providers/local/camel-maildir-summary.h @@ -24,7 +24,7 @@ #include "camel-local-summary.h" #include #include -#include +#include #define CAMEL_MAILDIR_SUMMARY(obj) CAMEL_CHECK_CAST (obj, camel_maildir_summary_get_type (), CamelMaildirSummary) #define CAMEL_MAILDIR_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_maildir_summary_get_type (), CamelMaildirSummaryClass) @@ -66,7 +66,7 @@ struct _CamelMaildirSummaryClass { }; CamelType camel_maildir_summary_get_type (void); -CamelMaildirSummary *camel_maildir_summary_new (const char *filename, const char *maildirdir, ibex *index); +CamelMaildirSummary *camel_maildir_summary_new (const char *filename, const char *maildirdir, CamelIndex *index); /* convert some info->flags to/from the messageinfo */ char *camel_maildir_summary_info_to_name(const CamelMessageInfo *info); diff --git a/camel/providers/local/camel-mbox-folder.c b/camel/providers/local/camel-mbox-folder.c index c2b98c9aa3..bef6b64436 100644 --- a/camel/providers/local/camel-mbox-folder.c +++ b/camel/providers/local/camel-mbox-folder.c @@ -60,7 +60,7 @@ static void mbox_set_message_user_tag(CamelFolder *folder, const char *uid, cons static void mbox_append_message(CamelFolder *folder, CamelMimeMessage * message, const CamelMessageInfo * info, CamelException *ex); static CamelMimeMessage *mbox_get_message(CamelFolder *folder, const gchar * uid, CamelException *ex); -static CamelLocalSummary *mbox_create_summary(const char *path, const char *folder, ibex *index); +static CamelLocalSummary *mbox_create_summary(const char *path, const char *folder, CamelIndex *index); static void mbox_finalise(CamelObject * object); @@ -134,7 +134,7 @@ camel_mbox_folder_new(CamelStore *parent_store, const char *full_name, guint32 f return folder; } -static CamelLocalSummary *mbox_create_summary(const char *path, const char *folder, ibex *index) +static CamelLocalSummary *mbox_create_summary(const char *path, const char *folder, CamelIndex *index) { return (CamelLocalSummary *)camel_mbox_summary_new(path, folder, index); } diff --git a/camel/providers/local/camel-mbox-store.c b/camel/providers/local/camel-mbox-store.c index f401632b8d..3333580766 100644 --- a/camel/providers/local/camel-mbox-store.c +++ b/camel/providers/local/camel-mbox-store.c @@ -23,7 +23,9 @@ #include #endif +#include #include +#include #include #include #include diff --git a/camel/providers/local/camel-mbox-summary.c b/camel/providers/local/camel-mbox-summary.c index 697e806e53..2d06d29a38 100644 --- a/camel/providers/local/camel-mbox-summary.c +++ b/camel/providers/local/camel-mbox-summary.c @@ -27,7 +27,9 @@ #include "camel/camel-mime-message.h" #include "camel/camel-operation.h" +#include #include +#include #include #include #include @@ -136,7 +138,7 @@ camel_mbox_summary_finalise(CamelObject *obj) * Return value: A new CamelMboxSummary widget. **/ CamelMboxSummary * -camel_mbox_summary_new(const char *filename, const char *mbox_name, ibex *index) +camel_mbox_summary_new(const char *filename, const char *mbox_name, CamelIndex *index) { CamelMboxSummary *new = (CamelMboxSummary *)camel_object_new(camel_mbox_summary_get_type()); @@ -289,7 +291,7 @@ summary_rebuild(CamelMboxSummary *mbs, off_t offset, CamelException *ex) off_t pc = camel_mime_parser_tell_start_from (mp) + 1; camel_operation_progress (NULL, (int) (((float) pc / size) * 100)); - + info = camel_folder_summary_add_from_parser(s, mp); if (info == NULL) { camel_exception_setv(ex, 1, _("Fatal mail parser error near position %ld in folder %s"), @@ -560,7 +562,7 @@ mbox_summary_sync_full(CamelLocalSummary *cls, gboolean expunge, CamelFolderChan d(printf("Deleting %s\n", uid)); if (cls->index) - ibex_unindex(cls->index, (char *)uid); + camel_index_delete_name(cls->index, uid); /* remove it from the change list */ camel_folder_change_info_remove_uid(changeinfo, uid); diff --git a/camel/providers/local/camel-mbox-summary.h b/camel/providers/local/camel-mbox-summary.h index 82d200c854..ec4d015489 100644 --- a/camel/providers/local/camel-mbox-summary.h +++ b/camel/providers/local/camel-mbox-summary.h @@ -53,7 +53,7 @@ struct _CamelMboxSummaryClass { }; guint camel_mbox_summary_get_type (void); -CamelMboxSummary *camel_mbox_summary_new (const char *filename, const char *mbox_name, ibex *index); +CamelMboxSummary *camel_mbox_summary_new (const char *filename, const char *mbox_name, CamelIndex *index); /* generate a From line from headers */ char *camel_mbox_summary_build_from(struct _header_raw *header); diff --git a/camel/providers/local/camel-mh-folder.c b/camel/providers/local/camel-mh-folder.c index 6795ab3fbf..3d08bdb60a 100644 --- a/camel/providers/local/camel-mh-folder.c +++ b/camel/providers/local/camel-mh-folder.c @@ -50,7 +50,7 @@ static CamelLocalFolderClass *parent_class = NULL; #define CF_CLASS(so) CAMEL_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(so)) #define CMHS_CLASS(so) CAMEL_STORE_CLASS (CAMEL_OBJECT_GET_CLASS(so)) -static CamelLocalSummary *mh_create_summary(const char *path, const char *folder, ibex *index); +static CamelLocalSummary *mh_create_summary(const char *path, const char *folder, CamelIndex *index); static void mh_append_message(CamelFolder * folder, CamelMimeMessage * message, const CamelMessageInfo *info, CamelException * ex); static CamelMimeMessage *mh_get_message(CamelFolder * folder, const gchar * uid, CamelException * ex); @@ -115,7 +115,7 @@ camel_mh_folder_new(CamelStore *parent_store, const char *full_name, guint32 fla return folder; } -static CamelLocalSummary *mh_create_summary(const char *path, const char *folder, ibex *index) +static CamelLocalSummary *mh_create_summary(const char *path, const char *folder, CamelIndex *index) { return (CamelLocalSummary *)camel_mh_summary_new(path, folder, index); } diff --git a/camel/providers/local/camel-mh-summary.c b/camel/providers/local/camel-mh-summary.c index 06c5715d0d..530baa3caf 100644 --- a/camel/providers/local/camel-mh-summary.c +++ b/camel/providers/local/camel-mh-summary.c @@ -22,14 +22,16 @@ #include #endif +#include #include +#include + #include #include #include #include #include -#include #include #include @@ -118,7 +120,7 @@ camel_mh_summary_finalise(CamelObject *obj) * * Return value: A new #CamelMhSummary object. **/ -CamelMhSummary *camel_mh_summary_new (const char *filename, const char *mhdir, ibex *index) +CamelMhSummary *camel_mh_summary_new (const char *filename, const char *mhdir, CamelIndex *index) { CamelMhSummary *o = (CamelMhSummary *)camel_object_new(camel_mh_summary_get_type ()); @@ -171,7 +173,7 @@ static int camel_mh_summary_add(CamelLocalSummary *cls, const char *name, int fo mp = camel_mime_parser_new(); camel_mime_parser_scan_from(mp, FALSE); camel_mime_parser_init_with_fd(mp, fd); - if (cls->index && (forceindex || !ibex_contains_name(cls->index, (char *)name))) { + if (cls->index && (forceindex || !camel_index_has_name(cls->index, name))) { d(printf("forcing indexing of message content\n")); camel_folder_summary_set_index((CamelFolderSummary *)mhs, cls->index); } else { @@ -191,7 +193,7 @@ remove_summary(char *key, CamelMessageInfo *info, CamelLocalSummary *cls) { d(printf("removing message %s from summary\n", key)); if (cls->index) - ibex_unindex(cls->index, (char *)camel_message_info_uid(info)); + camel_index_delete_name(cls->index, camel_message_info_uid(info)); camel_folder_summary_remove((CamelFolderSummary *)cls, info); camel_folder_summary_info_free((CamelFolderSummary *)cls, info); } @@ -239,7 +241,7 @@ mh_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changeinfo, Came } if (c==0) { info = camel_folder_summary_uid((CamelFolderSummary *)cls, d->d_name); - if (info == NULL || (cls->index && (!ibex_contains_name(cls->index, d->d_name)))) { + if (info == NULL || (cls->index && (!camel_index_has_name(cls->index, d->d_name)))) { /* need to add this file to the summary */ if (info != NULL) { g_hash_table_remove(left, camel_message_info_uid(info)); @@ -368,7 +370,7 @@ mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo /* FIXME: put this in folder_summary::remove()? */ if (cls->index) - ibex_unindex(cls->index, (char *)uid); + camel_index_delete_name(cls->index, (char *)uid); camel_folder_change_info_remove_uid(changes, uid); camel_folder_summary_remove((CamelFolderSummary *)cls, info); diff --git a/camel/providers/local/camel-mh-summary.h b/camel/providers/local/camel-mh-summary.h index b24efbbfbf..4ee30df63b 100644 --- a/camel/providers/local/camel-mh-summary.h +++ b/camel/providers/local/camel-mh-summary.h @@ -24,7 +24,7 @@ #include "camel-local-summary.h" #include #include -#include +#include #define CAMEL_MH_SUMMARY(obj) CAMEL_CHECK_CAST (obj, camel_mh_summary_get_type (), CamelMhSummary) #define CAMEL_MH_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_mh_summary_get_type (), CamelMhSummaryClass) @@ -47,7 +47,7 @@ struct _CamelMhSummaryClass { }; CamelType camel_mh_summary_get_type (void); -CamelMhSummary *camel_mh_summary_new (const char *filename, const char *mhdir, ibex *index); +CamelMhSummary *camel_mh_summary_new (const char *filename, const char *mhdir, CamelIndex *index); #endif /* ! _CAMEL_MH_SUMMARY_H */ diff --git a/camel/providers/local/camel-spool-folder.h b/camel/providers/local/camel-spool-folder.h index 79f3e9ec88..af43e0dca1 100644 --- a/camel/providers/local/camel-spool-folder.h +++ b/camel/providers/local/camel-spool-folder.h @@ -29,7 +29,7 @@ extern "C" { #include #include -#include +#include #include "camel-spool-summary.h" #include "camel-lock.h" @@ -69,7 +69,7 @@ typedef struct { /* Virtual methods */ /* summary factory, only used at init */ - CamelSpoolSummary *(*create_summary)(const char *path, const char *folder, ibex *index); + CamelSpoolSummary *(*create_summary)(const char *path, const char *folder, CamelIndex *index); /* Lock the folder for my operations */ int (*lock)(CamelSpoolFolder *, CamelLockType type, CamelException *ex); diff --git a/camel/providers/local/camel-spool-summary.c b/camel/providers/local/camel-spool-summary.c index 823b26fc77..d8c85b9e6d 100644 --- a/camel/providers/local/camel-spool-summary.c +++ b/camel/providers/local/camel-spool-summary.c @@ -23,14 +23,17 @@ #include #endif -#include +#include #include +#include #include #include #include #include #include +#include + #include "camel-spool-summary.h" #include "camel-mime-message.h" #include "camel-file-utils.h" diff --git a/camel/providers/local/camel-spool-summary.h b/camel/providers/local/camel-spool-summary.h index e0b63a1b6b..5b20e1dfbe 100644 --- a/camel/providers/local/camel-spool-summary.h +++ b/camel/providers/local/camel-spool-summary.h @@ -24,7 +24,7 @@ #include #include #include -#include +#include #define CAMEL_SPOOL_SUMMARY(obj) CAMEL_CHECK_CAST (obj, camel_spool_summary_get_type (), CamelSpoolSummary) #define CAMEL_SPOOL_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_spool_summary_get_type (), CamelSpoolSummaryClass) @@ -68,7 +68,7 @@ struct _CamelSpoolSummaryClass { }; guint camel_spool_summary_get_type (void); -void camel_spool_summary_construct (CamelSpoolSummary *new, const char *filename, const char *spool_name, ibex *index); +void camel_spool_summary_construct (CamelSpoolSummary *new, const char *filename, const char *spool_name, CamelIndex *index); /* create the summary, in-memory only */ CamelSpoolSummary *camel_spool_summary_new(const char *filename); diff --git a/camel/providers/local/camel-spoold-store.c b/camel/providers/local/camel-spoold-store.c index addd727214..aba994cc91 100644 --- a/camel/providers/local/camel-spoold-store.c +++ b/camel/providers/local/camel-spoold-store.c @@ -259,7 +259,7 @@ static int scan_dir(CamelStore *store, GHashTable *visited, char *root, const ch uri = g_strdup_printf("spoold:%s;noselect=yes#%s", root, path); tmp = strrchr(path, '/'); if (tmp == NULL) - tmp = path; + tmp = (char *)path; else tmp++; fi = camel_folder_info_new(uri, path, tmp, -1); -- cgit v1.2.3