NOOP if writing to a failed file. (word_index_pre): NOOP if failed file.

2001-08-10  Not Zed  <NotZed@Ximian.com>

        * wordindexmem.c (sync_cache_entry): NOOP if writing to a failed
        file.
        (word_index_pre): NOOP if failed file.
        (ibex_create_word_index_mem): Setup blocks value.

        ** Added internal exception handling to libibex, in the case of
        errors with on-disk data, exceptions are returned.

        * block.c (ibex_block_cache_open): Detect fatal errors below us
        and clean up appropriately.
        (ibex_block_cache_fail): New function to handle the failure, and
        keep track of it.
        (ibex_block_cache_sync): Dont do anything if we've failed on this
        file.

        * disktail.c (tail_compress): Add blocks param so we can assert
        for exceptions.

        * hash.c, block.c disktail.c: g_assert->ibex_block_cache_assert
        where dealing with external data.

        * hash.c (hash_info): Add index param so we can assert for exceptions.

        * ibex_block.c (ibex_index_buffer): Setjmp before calling into
        internal routines.
        (ibex_save): "
        (ibex_unindex): "
        (ibex_find): "
        (ibex_find_name): "
        (ibex_contains_name): "
        (ibex_reset): Function to reset the index file if we have an
        error, call when we have an error.

        * block.h (ibex_block_cache_assert): Create assertion/exception
        macros, and include a setjmp buffer for returning it.

2001-08-09  Not Zed  <NotZed@Ximian.com>

        * Makefile.am (libibex_la_SOURCES): Remove wordindex.c,
        wordindexmem is what's used.

svn path=/trunk/; revision=11864

10 files changed, 240 insertions, 60 deletions

diff --git a/libibex/ChangeLog b/libibex/ChangeLog
index ca122c3298..80340c7af8 100644
--- a/libibex/ChangeLog
+++ b/libibex/ChangeLog
@@ -1,3 +1,46 @@
+2001-08-10  Not Zed  <NotZed@Ximian.com>
+
+	* wordindexmem.c (sync_cache_entry): NOOP if writing to a failed
+	file.
+	(word_index_pre): NOOP if failed file.
+	(ibex_create_word_index_mem): Setup blocks value.
+
+	** Added internal exception handling to libibex, in the case of
+	errors with on-disk data, exceptions are returned.
+
+	* block.c (ibex_block_cache_open): Detect fatal errors below us
+	and clean up appropriately.
+	(ibex_block_cache_fail): New function to handle the failure, and
+	keep track of it.
+	(ibex_block_cache_sync): Dont do anything if we've failed on this
+	file.
+
+	* disktail.c (tail_compress): Add blocks param so we can assert
+	for exceptions.
+
+	* hash.c, block.c disktail.c: g_assert->ibex_block_cache_assert
+	where dealing with external data.
+
+	* hash.c (hash_info): Add index param so we can assert for exceptions.
+
+	* ibex_block.c (ibex_index_buffer): Setjmp before calling into
+	internal routines.
+	(ibex_save): "
+	(ibex_unindex): "
+	(ibex_find): "
+	(ibex_find_name): "
+	(ibex_contains_name): "
+	(ibex_reset): Function to reset the index file if we have an
+	error, call when we have an error.
+
+	* block.h (ibex_block_cache_assert): Create assertion/exception
+	macros, and include a setjmp buffer for returning it.
+
+2001-08-09  Not Zed  <NotZed@Ximian.com>
+
+	* Makefile.am (libibex_la_SOURCES): Remove wordindex.c,
+	wordindexmem is what's used.
+
 2001-06-01  Peter Williams  <peterw@ximian.com>
 
 	* Makefile.am (dumpindex_LDADD): Add GAL_LIBS here too.
diff --git a/libibex/Makefile.am b/libibex/Makefile.am
index 88c4d13f06..52dc07fb54 100644
--- a/libibex/Makefile.am
+++ b/libibex/Makefile.am
@@ -3,7 +3,7 @@
 noinst_LTLIBRARIES = libibex.la
 
 libibex_la_SOURCES = \
-	wordindex.c wordindexmem.c \
+	wordindexmem.c \
 	block.c ibex.h \
 	hash.c \
 	disktail.c \
diff --git a/libibex/block.c b/libibex/block.c
index 6e035aebc9..ef7a524d71 100644
--- a/libibex/block.c
+++ b/libibex/block.c
@@ -277,6 +277,9 @@ ibex_block_cache_sync(struct _memcache *block_cache)
 {
 	struct _memblock *memblock;
 
+	if (block_cache->failed)
+		return;
+
 	memblock = (struct _memblock *)block_cache->nodes.head;
 	while (memblock->next) {
 #ifdef MALLOC_CHECK
@@ -365,8 +368,8 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid)
 #endif
 
 	/* nothing can read the root block directly */
-	g_assert(blockid != 0);
-	g_assert(blockid < block_cache->root.roof);
+	ibex_block_cache_assert(block_cache, blockid != 0);
+	ibex_block_cache_assert(block_cache, blockid < block_cache->root.roof);
 
 	memblock = g_hash_table_lookup(block_cache->index, (void *)blockid);
 
@@ -443,6 +446,19 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid)
 	return &memblock->data;
 }
 
+void
+ibex_block_cache_fail(struct _memcache *block_cache, char *where, int line, char *why)
+{
+	block_cache->failed = TRUE;
+	block_cache->root.flags &= ~IBEX_ROOT_SYNCF;
+	/* and blow it away, we can do nothing better yet */
+	ftruncate(block_cache->fd, 0);
+
+	g_warning("%s(%d): Integrity assertion failed: '%s' on file '%s'", where, line, why, block_cache->name);
+
+	longjmp(block_cache->failenv, 1);
+}
+
 /**
  * ibex_block_cache_open:
  * @name: 
@@ -469,6 +485,8 @@ ibex_block_cache_open(const char *name, int flags, int mode)
 	block_cache->count = 0;
 	block_cache->index = g_hash_table_new(g_direct_hash, g_direct_equal);
 	block_cache->fd = open(name, flags, mode);
+	block_cache->failed = FALSE;
+	block_cache->name = g_strdup(name);
 
 	if (block_cache->fd == -1) {
 		g_hash_table_destroy(block_cache->index);
@@ -476,6 +494,13 @@ ibex_block_cache_open(const char *name, int flags, int mode)
 		return NULL;
 	}
 
+	if (ibex_block_cache_setjmp(block_cache) != 0) {
+		close(block_cache->fd);
+		g_hash_table_destroy(block_cache->index);
+		g_free(block_cache);
+		return NULL;		
+	}
+
 	ibex_block_read_root(block_cache);
 	if (block_cache->root.roof == 0
 	    || memcmp(block_cache->root.version, IBEX_VERSION, 4)
@@ -526,6 +551,7 @@ ibex_block_cache_close(struct _memcache *block_cache)
 
 	ibex_block_cache_sync(block_cache);
 	close(block_cache->fd);
+	g_free(block_cache->name);
 
 	mw = (struct _memblock *)block_cache->nodes.head;
 	mn = mw->next;
@@ -585,7 +611,7 @@ ibex_block_get(struct _memcache *block_cache)
 		block = ibex_block_read(block_cache, head);
 	}
 
-	g_assert(head != 0);
+	ibex_block_cache_assert(block_cache, head != 0);
 
 	d(printf("new block = %d\n", head));
 	block->next = 0;
diff --git a/libibex/block.h b/libibex/block.h
index dbc8fd5ad8..74221836e1 100644
--- a/libibex/block.h
+++ b/libibex/block.h
@@ -7,6 +7,7 @@
 /*#define IBEX_STATS*/		/* define to get/dump block access stats */
 
 #include <glib.h>
+#include <setjmp.h>
 
 /* version of file format */
 #define IBEX_VERSION "ibx6"
@@ -78,7 +79,12 @@ struct _memcache {
 	int count;		/* nodes in cache */
 
 	GHashTable *index;	/* blockid->memblock mapping */
+
 	int fd;			/* file fd */
+	char *name;		/* file name */
+
+	jmp_buf failenv;	/* for exception failure */
+	int failed;		/* indicates the file failed */
 
 #ifdef IBEX_STATS
 	GHashTable *stats;
@@ -103,6 +109,11 @@ void ibex_block_cache_close(struct _memcache *block_cache);
 void ibex_block_cache_sync(struct _memcache *block_cache);
 void ibex_block_cache_flush(struct _memcache *block_cache);
 
+#define ibex_block_cache_setjmp(bc) (((bc)==NULL)?1:setjmp((bc)->failenv))
+#define ibex_block_cache_assert(bc, cond) { if (!(cond)) { ibex_block_cache_fail(bc, __FILE__, __LINE__, # cond); } }
+
+void ibex_block_cache_fail(struct _memcache *block_cache, char *file, int line, char *why);
+
 blockid_t ibex_block_get(struct _memcache *block_cache);
 void ibex_block_free(struct _memcache *block_cache, blockid_t blockid);
 void ibex_block_dirty(struct _block *block);
diff --git a/libibex/disktail.c b/libibex/disktail.c
index b7f0091243..50c21afa9c 100644
--- a/libibex/disktail.c
+++ b/libibex/disktail.c
@@ -105,7 +105,7 @@ tail_info(struct _tailblock *bucket, nameid_t tailid, blockid_t **startptr)
 
 /* compresses (or expand) the bucket entry, to the new size */
 static void
-tail_compress(struct _tailblock *bucket, int index, int newsize)
+tail_compress(struct _memcache *blocks, struct _tailblock *bucket, int index, int newsize)
 {
 	int i;
 	blockid_t *start, *end, *newstart;
@@ -170,10 +170,10 @@ tail_compress(struct _tailblock *bucket, int index, int newsize)
 	/* fixup data */
 	newstart = &bucket->tb_data[bucket->tb_offset[bucket->used-1]];
 
-	g_assert(newstart+(end-start)-newsize <= &bucket->tb_data[sizeof(bucket->tb_data)/sizeof(bucket->tb_data[0])]);
-	g_assert(newstart + (start-newstart) + MIN(end-start, newsize) <= &bucket->tb_data[sizeof(bucket->tb_data)/sizeof(bucket->tb_data[0])]);
-	g_assert(newstart+(end-start)-newsize >= (blockid_t *) &bucket->tb_offset[bucket->used]);
-	g_assert(newstart + (start-newstart) + MIN(end-start, newsize) >= (blockid_t *) &bucket->tb_offset[bucket->used]);
+	ibex_block_cache_assert(blocks, newstart+(end-start)-newsize <= &bucket->tb_data[sizeof(bucket->tb_data)/sizeof(bucket->tb_data[0])]);
+	ibex_block_cache_assert(blocks, newstart + (start-newstart) + MIN(end-start, newsize) <= &bucket->tb_data[sizeof(bucket->tb_data)/sizeof(bucket->tb_data[0])]);
+	ibex_block_cache_assert(blocks, newstart+(end-start)-newsize >= (blockid_t *) &bucket->tb_offset[bucket->used]);
+	ibex_block_cache_assert(blocks, newstart + (start-newstart) + MIN(end-start, newsize) >= (blockid_t *) &bucket->tb_offset[bucket->used]);
 
 	memmove(newstart+(end-start)-newsize, newstart, ((start-newstart)+MIN(end-start, newsize)) * sizeof(blockid_t));
 
@@ -240,14 +240,14 @@ tail_get(struct _memcache *blocks, int size)
 			d(printf("allocated %d (%d), used %d\n", tailid, tailid, tail->used));
 			ibex_block_dirty((struct _block *)tail);
 
-			g_assert(&tail->tb_offset[tail->used-1]
-				 < (unsigned char *) &tail->tb_data[tail->tb_offset[tail->used-1]]);
+			ibex_block_cache_assert(blocks, &tail->tb_offset[tail->used-1]
+						< (unsigned char *) &tail->tb_data[tail->tb_offset[tail->used-1]]);
 
 			return tailid;
 		}
 
-		g_assert(&tail->tb_offset[tail->used-1]
-			 < (unsigned char *) &tail->tb_data[tail->tb_offset[tail->used-1]]);
+		ibex_block_cache_assert(blocks, &tail->tb_offset[tail->used-1]
+					< (unsigned char *) &tail->tb_data[tail->tb_offset[tail->used-1]]);
 
 		/* see if we have a free slot first */
 		freeindex = -1;
@@ -278,7 +278,7 @@ tail_get(struct _memcache *blocks, int size)
 				tail->tb_offset[tail->used] = tail->tb_offset[tail->used-1];
 				tail->used++;
 			}
-			tail_compress(tail, freeindex, size);
+			tail_compress(blocks, tail, freeindex, size);
 			ibex_block_dirty((struct _block *)tail);
 			d(printf("allocated %d (%d), used %d\n", tailid, TAIL_KEY(tailid, freeindex), tail->used));
 			return TAIL_KEY(tailid, freeindex);
@@ -315,11 +315,11 @@ tail_free(struct _memcache *blocks, blockid_t tailid)
 
 	tail = (struct _tailblock *)ibex_block_read(blocks, TAIL_BLOCK(tailid));
 	d(printf("  tail %d used %d\n", tailid, tail->used));
-	g_assert(tail->used);
+	ibex_block_cache_assert(blocks, tail->used);
 	if (TAIL_INDEX(tailid)  == tail->used - 1) {
 		tail->used --;
 	} else {
-		tail_compress(tail, TAIL_INDEX(tailid), 0);
+		tail_compress(blocks, tail, TAIL_INDEX(tailid), 0);
 	}
 	ibex_block_dirty((struct _block *)tail);
 }
@@ -391,7 +391,7 @@ disk_add_blocks_internal(struct _IBEXStore *store, blockid_t *headptr, blockid_t
 	int space, copied = 0, left;
 
 	/* assumes this funciton is in control of any tail creation */ 
-	g_assert(tail == 0);
+	ibex_block_cache_assert(store->blocks, tail == 0);
 
 	d(printf("Adding %d items to block list\n", data->len));
 
@@ -519,7 +519,7 @@ disk_add_list(struct _IBEXStore *store, blockid_t *headptr, blockid_t *tailptr,
 				head = disk_add_blocks_internal(store, headptr, tailptr, tmpdata);
 			} else if (tail_space(tailblock) >= data->len) {
 				/* can we just expand this in our node, or do we need a new one? */
-				tail_compress(tailblock, TAIL_INDEX(tail), data->len + len);
+				tail_compress(store->blocks, tailblock, TAIL_INDEX(tail), data->len + len);
 				memcpy(&tailblock->tb_data[tailblock->tb_offset[TAIL_INDEX(tail)] + len],
 				       data->data, data->len * sizeof(blockid_t));
 				ibex_block_dirty((struct _block *)tailblock);
@@ -625,7 +625,7 @@ disk_remove(struct _IBEXStore *store, blockid_t *headptr, blockid_t *tailptr, na
 					ibex_block_dirty((struct _block *)tailblock);
 					*tailptr = 0;
 				} else {
-					tail_compress(tailblock, TAIL_INDEX(tail), len-1);
+					tail_compress(store->blocks, tailblock, TAIL_INDEX(tail), len-1);
 					ibex_block_dirty((struct _block *)tailblock);
 				}
 			}
@@ -654,7 +654,7 @@ static void disk_free(struct _IBEXStore *store, blockid_t head, blockid_t tail)
 	if (tail) {
 		struct _tailblock *tailblock = (struct _tailblock *)ibex_block_read(store->blocks, TAIL_BLOCK(tail));
 		d(printf("freeing tail block %d (%d)\n", TAIL_BLOCK(tail), tail));
-		tail_compress(tailblock, TAIL_INDEX(tail), 0);
+		tail_compress(store->blocks, tailblock, TAIL_INDEX(tail), 0);
 		ibex_block_dirty((struct _block *)tailblock);
 	}
 }
diff --git a/libibex/hash.c b/libibex/hash.c
index ff3ff71e55..e0ad09ac2f 100644
--- a/libibex/hash.c
+++ b/libibex/hash.c
@@ -244,7 +244,7 @@ hash_get_key(struct _IBEXIndex *index, hashid_t hashbucket, int *len)
 	bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashbucket));
 	ind = HASH_INDEX(hashbucket);
 
-	g_assert(ind < bucket->used);
+	ibex_block_cache_assert(index->blocks, ind < bucket->used);
 
 	start = &bucket->hb_keydata[bucket->hb_keys[ind].keyoffset];
 	if (ind == 0) {
@@ -286,7 +286,7 @@ hash_find(struct _IBEXIndex *index, const char *key, int keylen)
 	/* find the table containing this entry */
 	hash = hash_key(key, keylen) % hashroot->size;
 	hashtable = hashroot->table[hash / (BLOCK_SIZE/sizeof(blockid_t))];
-	g_assert(hashtable != 0);
+	ibex_block_cache_assert(index->blocks, hashtable != 0);
 	table = (struct _hashtableblock *)ibex_block_read(index->blocks, hashtable);
 	hashentry = hash % (BLOCK_SIZE/sizeof(blockid_t));
 	/* and its bucket */
@@ -312,7 +312,7 @@ hash_find(struct _IBEXIndex *index, const char *key, int keylen)
 		/* and get the key number within the block */
 		ind = HASH_INDEX(hashbucket);
 
-		g_assert(ind < bucket->used);
+		ibex_block_cache_assert(index->blocks, ind < bucket->used);
 
 		start = &bucket->hb_keydata[bucket->hb_keys[ind].keyoffset];
 		if (ind == 0) {
@@ -330,11 +330,11 @@ hash_find(struct _IBEXIndex *index, const char *key, int keylen)
 }
 
 static int
-hash_info(struct _hashblock *bucket, int index)
+hash_info(struct _IBEXIndex *idex, struct _hashblock *bucket, int index)
 {
 	char *start, *end;
 
-	g_assert(index < bucket->used);
+	ibex_block_cache_assert(idex->blocks, index < bucket->used);
 
 	start = &bucket->hb_keydata[bucket->hb_keys[index].keyoffset];
 	if (index == 0) {
@@ -448,7 +448,7 @@ hash_remove(struct _IBEXIndex *index, const char *key, int keylen)
 		/* and get the key number within the block */
 		ind = HASH_INDEX(hashbucket);
 
-		g_assert(ind < bucket->used);
+		ibex_block_cache_assert(index->blocks, ind < bucket->used);
 
 		start = &bucket->hb_keydata[bucket->hb_keys[ind].keyoffset];
 		if (ind == 0) {
@@ -473,8 +473,7 @@ hash_remove(struct _IBEXIndex *index, const char *key, int keylen)
 				ibex_block_dirty((struct _block *)hashroot);
 			} else {
 				prevbucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashprev));
-				prevbucket->hb_keys[HASH_INDEX(hashprev)].next =
-					bucket->hb_keys[ind].next;
+				prevbucket->hb_keys[HASH_INDEX(hashprev)].next = bucket->hb_keys[ind].next;
 				/* link into free list */
 				bucket->hb_keys[ind].next = hashroot->free;
 				hashroot->free = hashbucket;
@@ -500,7 +499,7 @@ hash_set_data_block(struct _IBEXIndex *index, hashid_t keyid, blockid_t blockid,
 	d(printf("setting data block hash %d to %d tail %d\n", keyid, blockid, tail));
 
 	/* map to a block number */
-	g_assert((blockid & (BLOCK_SIZE-1)) == 0);
+	ibex_block_cache_assert(index->blocks, (blockid & (BLOCK_SIZE-1)) == 0);
 	blockid >>= BLOCK_BITS;
 
 	bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(keyid));
@@ -581,8 +580,7 @@ hash_insert(struct _IBEXIndex *index, const char *key, int keylen)
 			- (char *)&bucket->hb_keys[bucket->used];
 		if (space >= keylen) {
 			hash_expand(bucket, HASH_INDEX(keybucket), keylen);
-			memcpy(&bucket->hb_keydata[bucket->hb_keys[HASH_INDEX(keybucket)].keyoffset],
-			       key, keylen);
+			memcpy(&bucket->hb_keydata[bucket->hb_keys[HASH_INDEX(keybucket)].keyoffset], key, keylen);
 
 			/* check if there is free space still in this node, and there are no other empty blocks */
 			keyfree = bucket->hb_keys[HASH_INDEX(keybucket)].next;
@@ -709,7 +707,7 @@ hash_cursor_next(struct _IBEXCursor *idc)
 	while (hc->block != 0) {
 		bucket = (struct _hashblock *)ibex_block_read(idc->index->blocks, hc->block);
 		while (hc->index < bucket->used) {
-			if (hash_info(bucket, hc->index) > 0) {
+			if (hash_info(idc->index, bucket, hc->index) > 0) {
 				hc->key = HASH_KEY(hc->block, hc->index);
 				hc->index++;
 				if (hc->index == bucket->used) {
diff --git a/libibex/ibex_block.c b/libibex/ibex_block.c
index c1aad5e78a..bbd13f2308 100644
--- a/libibex/ibex_block.c
+++ b/libibex/ibex_block.c
@@ -11,6 +11,8 @@
 
 #define d(x)
 
+static void ibex_reset(ibex *ib);
+
 static signed char utf8_trans[] = {
 	'A', 'A', 'A', 'A', 'A', 'A', -1, 'C', 'E', 'E', 'E', 'E', 'I', 'I',
 	'I', 'I', -2, 'N', 'O', 'O', 'O', 'O', 'O', '*', 'O', 'U', 'U', 'U',
@@ -129,7 +131,8 @@ utf8_category (char *p, char **np, char *end)
 int
 ibex_index_buffer (ibex *ib, char *name, char *buffer, size_t len, size_t *unread)
 {
-	char *p, *q, *nq, *end, *word;
+	char *p, *q, *nq, *end;
+	char *word;
 	int wordsiz, cat = 0;
 	GHashTable *words = g_hash_table_new(g_str_hash, g_str_equal);
 	GPtrArray *wordlist = g_ptr_array_new();
@@ -186,24 +189,36 @@ ibex_index_buffer (ibex *ib, char *name, char *buffer, size_t len, size_t *unrea
 		p = q;
 	}
 done:
+	/* this weird word=NULL shit is to get rid of compiler warnings about clobbering with longjmp */
+	g_free(word);
+	word = NULL;
+
 	IBEX_LOCK(ib);
 
-	d(printf("name %s count %d size %d\n", name, wordlist->len, len));
-	if (!ib->predone) {
-		ib->words->klass->index_pre(ib->words);
-		ib->predone = TRUE;
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		printf("Error in indexing\n");
+		ret = -1;
+		ibex_reset(ib);
+		word = NULL;	/* here too */
+	} else {
+		word = NULL;	/* ... and here */
+		d(printf("name %s count %d size %d\n", name, wordlist->len, len));
+		if (!ib->predone) {
+			ib->words->klass->index_pre(ib->words);
+			ib->predone = TRUE;
+		}
+		ib->words->klass->add_list(ib->words, name, wordlist);
+		ret = 0;
 	}
-	ib->words->klass->add_list(ib->words, name, wordlist);
 
 	IBEX_UNLOCK(ib);
 
-	ret = 0;
 error:
 	for (i=0;i<wordlist->len;i++)
 		g_free(wordlist->pdata[i]);
 	g_ptr_array_free(wordlist, TRUE);
 	g_hash_table_destroy(words);
-	g_free (word);
+	g_free(word);
 	return ret;
 }
 
@@ -219,6 +234,11 @@ ibex *ibex_open (char *file, int flags, int mode)
 		g_free(ib);
 		return NULL;
 	}
+
+	ib->name = g_strdup(file);
+	ib->flags = flags;
+	ib->mode = mode;
+
 	/* FIXME: the blockcache or the wordindex needs to manage the other one */
 	ib->words = ib->blocks->words;
 
@@ -230,48 +250,100 @@ ibex *ibex_open (char *file, int flags, int mode)
 
 int ibex_save (ibex *ib)
 {
+	int ret;
+
 	d(printf("syncing database\n"));
 
 	IBEX_LOCK(ib);
 
-	if (ib->predone) {
-		ib->words->klass->index_post(ib->words);
-		ib->predone = FALSE;
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		ibex_reset(ib);
+		printf("Error saving\n");
+		ret = -1;
+	} else {
+		if (ib->predone) {
+			ib->words->klass->index_post(ib->words);
+			ib->predone = FALSE;
+		}
+		ib->words->klass->sync(ib->words);
+		/* FIXME: some return */
+		ibex_block_cache_sync(ib->blocks);
+		ret = 0;
 	}
-	ib->words->klass->sync(ib->words);
-	/* FIXME: some return */
-	ibex_block_cache_sync(ib->blocks);
 
 	IBEX_UNLOCK(ib);
 
-	return 0;
+	return ret;
+}
+
+static int
+close_backend(ibex *ib)
+{
+	int ret;
+
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		printf("Error closing!\n");
+		ret = -1;
+	} else {
+		if (ib->predone) {
+			ib->words->klass->index_post(ib->words);
+			ib->predone = FALSE;
+		}
+
+		ib->words->klass->close(ib->words);
+		/* FIXME: return */
+		ibex_block_cache_close(ib->blocks);
+		ret = 0;
+	}
+
+	return ret;
+}
+
+/* close/reopen the ibex file, assumes we have lock */
+static void
+ibex_reset(ibex *ib)
+{
+	g_warning("resetting ibex file");
+
+	close_backend(ib);
+
+	ib->blocks = ibex_block_cache_open(ib->name, ib->flags, ib->mode);
+	if (ib->blocks == 0) {
+		g_warning("ibex_reset create: Error occured?: %s\n", strerror(errno));
+	}
+	/* FIXME: the blockcache or the wordindex needs to manage the other one */
+	ib->words = ib->blocks->words;
 }
 
 int ibex_close (ibex *ib)
 {
-	int ret = 0;
+	int ret;
 
 	d(printf("closing database\n"));
 
-	if (ib->predone) {
-		ib->words->klass->index_post(ib->words);
-		ib->predone = FALSE;
-	}
+	ret = close_backend(ib);
+
+	g_free(ib->name);
 
-	ib->words->klass->close(ib->words);
-	ibex_block_cache_close(ib->blocks);
 #ifdef ENABLE_THREADS
 	g_mutex_free(ib->lock);
 #endif
 	g_free(ib);
+
 	return ret;
 }
 
 void ibex_unindex (ibex *ib, char *name)
 {
 	d(printf("trying to unindex '%s'\n", name));
+
 	IBEX_LOCK(ib);
-	ib->words->klass->unindex_name(ib->words, name);
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		printf("Error unindexing!\n");
+		ibex_reset(ib);
+	} else {
+		ib->words->klass->unindex_name(ib->words, name);
+	}
 	IBEX_UNLOCK(ib);
 }
 
@@ -285,7 +357,12 @@ GPtrArray *ibex_find (ibex *ib, char *word)
 	normal = alloca(len+1);
 	ibex_normalise_word(word, word+len, normal);
 	IBEX_LOCK(ib);
-	ret = ib->words->klass->find(ib->words, normal);
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		ibex_reset(ib);
+		ret = NULL;
+	} else {
+		ret = ib->words->klass->find(ib->words, normal);
+	}
 	IBEX_UNLOCK(ib);
 	return ret;
 }
@@ -300,7 +377,12 @@ gboolean ibex_find_name (ibex *ib, char *name, char *word)
 	normal = alloca(len+1);
 	ibex_normalise_word(word, word+len, normal);
 	IBEX_LOCK(ib);
-	ret = ib->words->klass->find_name(ib->words, name, normal);
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		ibex_reset(ib);
+		ret = FALSE;
+	} else {
+		ret = ib->words->klass->find_name(ib->words, name, normal);
+	}
 	IBEX_UNLOCK(ib);
 	return ret;
 }
@@ -310,7 +392,12 @@ gboolean ibex_contains_name(ibex *ib, char *name)
 	gboolean ret;
 
 	IBEX_LOCK(ib);
-	ret = ib->words->klass->contains_name(ib->words, name);
+	if (ibex_block_cache_setjmp(ib->blocks) != 0) {
+		ibex_reset(ib);
+		ret = FALSE;
+	} else {
+		ret = ib->words->klass->contains_name(ib->words, name);
+	}
 	IBEX_UNLOCK(ib);
 	return ret;
 }
diff --git a/libibex/ibex_internal.h b/libibex/ibex_internal.h
index 889df90006..d8e5e28296 100644
--- a/libibex/ibex_internal.h
+++ b/libibex/ibex_internal.h
@@ -27,7 +27,9 @@
 #include "wordindex.h"
 
 struct ibex {
-	char *path;
+	char *name;
+	int flags;
+	int mode;
 	struct _memcache *blocks;
 	struct _IBEXWord *words;
 	int predone;
diff --git a/libibex/wordindex.h b/libibex/wordindex.h
index 9d685cce8f..e113f7e82c 100644
--- a/libibex/wordindex.h
+++ b/libibex/wordindex.h
@@ -56,6 +56,8 @@ struct _IBEXWord {
 	struct _IBEXStore *namestore;
 	struct _IBEXIndex *nameindex;
 
+	struct _memcache *blocks;
+
 	/* word caching info (should probably be modularised) */
 	GHashTable *wordcache;	/* word->struct _wordcache mapping */
 	struct _list wordnodes;	/* LRU list of wordcache structures */
@@ -66,7 +68,7 @@ struct _IBEXWord {
 };
 
 
-struct _IBEXWord *ibex_create_word_index(struct _memcache *bc, blockid_t *wordroot, blockid_t *nameroot);
+/*struct _IBEXWord *ibex_create_word_index(struct _memcache *bc, blockid_t *wordroot, blockid_t *nameroot);*/
 
 /* alternate implemenation */
 struct _IBEXWord *ibex_create_word_index_mem(struct _memcache *bc, blockid_t *wordroot, blockid_t *nameroot);
diff --git a/libibex/wordindexmem.c b/libibex/wordindexmem.c
index 55d2f5ef40..077b53e051 100644
--- a/libibex/wordindexmem.c
+++ b/libibex/wordindexmem.c
@@ -134,6 +134,7 @@ ibex_create_word_index_mem(struct _memcache *bc, blockid_t *wordroot, blockid_t
 	struct _IBEXWord *idx;
 
 	idx = g_malloc0(sizeof(*idx));
+	idx->blocks = bc;
 	idx->wordcache = g_hash_table_new(g_str_hash, g_str_equal);
 	ibex_list_new(&idx->wordnodes);
 	idx->wordcount = 0;
@@ -201,6 +202,12 @@ static void word_index_pre(struct _IBEXWord *idx)
 	if (idx->precount > 1)
 		return;
 
+	/* NOOP if failed */
+	if (idx->blocks->failed) {
+		g_warning("word_index_pre() on failed data file");
+		return;
+	}
+
 	/* want to load all words into the cache lookup table */
 	d(printf("pre-loading all word info into memory\n"));
 	idc = idx->wordindex->klass->get_cursor(idx->wordindex);
@@ -479,6 +486,10 @@ sync_cache_entry(struct _IBEXWord *idx, struct _wordcache *cache)
 {
 	GArray array; /* just use this as a header */
 	blockid_t oldblock, oldtail;
+
+	/* NOOP if failed */
+	if (idx->blocks->failed)
+		return;
 	
 	if (cache->filecount == 0)
 		return;