From 9aae808cd0b4cf8bd35f6c0205e30c79f62632ef Mon Sep 17 00:00:00 2001 From: Not Zed Date: Wed, 25 Oct 2000 13:59:44 +0000 Subject: Bugfixes, performance improvemnts. Should scale up much better than before, and be more bugfree than ever! 2000-10-25 Not Zed * ibex_internal.h (IBEX_VERSION): Bumped to another version. The file format hasn't changed, but earlier bugs may create invalid files. * block.c (ibex_block_read): Use the root data directly. (ibex_block_cache_open): As well. (ibex_block_get): And here too. (ibex_block_cache_sync): Sync the root block directly here. * block.h: Pad root block out to 1024 bytes. Added root block to struct _memcache. * disktail.c (tail_get): Dirty the root block. (tail_get): Fix for changes to root access. (disk_remove): And here too. * wordindexmem.c (sync_cache_entry): Handle the case of not having any files in the list, which can happen now. (word_index_pre): Make sure we set the wordid on the new cache entry. * ibex_block.c (ibex_save): Sigh. Pass the right argument to index_post. * block.c (ibex_block_cache_open): Create a word_index_mem for indexing the words, rather than a word_index. * ibex_block.c (ibex_index_buffer): If we haven't called index_pre yet, do it before indexing anything. (ibex_save): If wehave called index_pre previously, call index_post. (ibex_close): And same for here. * index.h: Added a cursor class, and cursor retrieval function for iterating through an index's keys. * wordindexmem.c (ibex_create_word_index_mem): New word class, similar to wordindex, but meant to be faster for updates. (word_index_pre): Implement. We load all keys into memory. (word_index_post): Implement. We sync and free all keys. (find): Remove lru code, its no longer a cache, but a lookup table. (add_index_cache): Remove lru code here too. (find_name): And here. (word_flush): Flush the hashtable direct. (word_close): Call flush to flush, rather than doing it ourselves. (add_index_cache): If we are in an index state, we can assume a cache miss == a new word. (word_index_post): Maintain whether or not we are in an index state, and the depth of the state. (word_index_pre): Likewise. Dont reread the index if we have already. (cache_sanity): Fixed for struct changes. * wordindex.h (IBEXWordClass): Added functions to prepare/cleanup for lots of indexing. i.e. can be used to optimise indexing speed at the cost of extra memory usage during the indexing process. * hash.c (hash_cursor_create): Create a new cursor for iterating through a hashtable. (hash_cursor_close): 'close' the cursor. It is upto the application to close any cursors it creates. (hash_cursor_next): Goto the next key id. (hash_cursor_next_key): Goto the next key, reutrn the key. (hash_get_cursor): Return a cursor object. * wordindex.c (word_index_post): (word_index_pre): Added (empty) callbacks for pre/post functions. svn path=/trunk/; revision=6165 --- libibex/block.c | 210 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 138 insertions(+), 72 deletions(-) (limited to 'libibex/block.c') diff --git a/libibex/block.c b/libibex/block.c index 8e10a116cb..52a79d27ae 100644 --- a/libibex/block.c +++ b/libibex/block.c @@ -30,11 +30,18 @@ #include #include #include - +#include +#include #include #include "block.h" +/*#define MALLOC_CHECK*/ + +#ifdef MALLOC_CHECK +#include +#endif + #define d(x) /*#define DEBUG*/ @@ -113,6 +120,27 @@ add_miss(struct _memcache *index, int id) } #endif /* IBEX_STATS */ +#ifdef MALLOC_CHECK +static void +checkmem(void *p) +{ + if (p) { + int status = mprobe(p); + + switch (status) { + case MCHECK_HEAD: + printf("Memory underrun at %p\n", p); + abort(); + case MCHECK_TAIL: + printf("Memory overrun at %p\n", p); + abort(); + case MCHECK_FREE: + printf("Double free %p\n", p); + abort(); + } + } +} +#endif /* simple list routines (for simplified memory management of cache/lists) */ @@ -187,6 +215,29 @@ memblock_addr(struct _block *block) return (struct _memblock *)(((char *)block) - G_STRUCT_OFFSET(struct _memblock, data)); } +/* read/sync the rootblock into the block_cache structure */ +static int +ibex_block_read_root(struct _memcache *block_cache) +{ + lseek(block_cache->fd, 0, SEEK_SET); + if (read(block_cache->fd, &block_cache->root, sizeof(block_cache->root)) != sizeof(block_cache->root)) { + + return -1; + } + return 0; +} + +static int +ibex_block_sync_root(struct _memcache *block_cache) +{ + lseek(block_cache->fd, 0, SEEK_SET); + if (write(block_cache->fd, &block_cache->root, sizeof(block_cache->root)) != sizeof(block_cache->root)) { + return -1; + } + return fsync(block_cache->fd); +} + + /** * ibex_block_dirty: * @block: @@ -224,27 +275,22 @@ sync_block(struct _memcache *block_cache, struct _memblock *memblock) void ibex_block_cache_sync(struct _memcache *block_cache) { - struct _memblock *memblock, *rootblock = 0; + struct _memblock *memblock; memblock = (struct _memblock *)block_cache->nodes.head; while (memblock->next) { - if (memblock->block == 0) { - rootblock = memblock; - } else { - if (memblock->flags & BLOCK_DIRTY) { - sync_block(block_cache, memblock); - } +#ifdef MALLOC_CHECK + checkmem(memblock); +#endif + if (memblock->flags & BLOCK_DIRTY) { + sync_block(block_cache, memblock); } memblock = memblock->next; } - if (rootblock) { - struct _root *root = (struct _root *)&rootblock->data; - root->flags |= IBEX_ROOT_SYNCF; - sync_block(block_cache, rootblock); - } - if (fsync(block_cache->fd) == 0) { - block_cache->flags |= IBEX_ROOT_SYNCF; + block_cache->root.flags |= IBEX_ROOT_SYNCF; + if (ibex_block_sync_root(block_cache) != 0) { + block_cache->root.flags &= ~IBEX_ROOT_SYNCF; } #ifdef IBEX_STATS @@ -253,6 +299,25 @@ ibex_block_cache_sync(struct _memcache *block_cache) } +#ifdef MALLOC_CHECK +static void +check_cache(struct _memcache *block_cache) +{ + struct _memblock *mw, *mn; + + checkmem(block_cache); + checkmem(block_cache->index); + + mw = (struct _memblock *)block_cache->nodes.head; + mn = mw->next; + while (mn) { + checkmem(mw); + mw = mn; + mn = mn->next; + } +} +#endif + /** * ibex_block_cache_flush: * @block_cache: @@ -278,7 +343,6 @@ ibex_block_cache_flush(struct _memcache *block_cache) ibex_list_new(&block_cache->nodes); } - /** * ibex_block_read: * @block_cache: @@ -296,32 +360,39 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid) { struct _memblock *memblock; - { - /* assert blockid 0) { - struct _root *root = (struct _root *)ibex_block_read(block_cache, 0); - g_assert(blockid < root->roof); - } - } +#ifdef MALLOC_CHECK + check_cache(block_cache); +#endif + + /* nothing can read the root block directly */ + g_assert(blockid != 0); + g_assert(blockid < block_cache->root.roof); memblock = g_hash_table_lookup(block_cache->index, (void *)blockid); + +#ifdef MALLOC_CHECK + check_cache(block_cache); +#endif + if (memblock) { d(printf("foudn blockid in cache %d = %p\n", blockid, &memblock->data)); -#if 0 - if (blockid == 0) { - struct _root *root = &memblock->data; - d(printf("superblock state:\n" - " roof = %d\n free = %d\n words = %d\n names = %d\n tail = %d", - root->roof, root->free, root->words, root->names, root->tail)); - - } -#endif + /* 'access' page */ ibex_list_remove((struct _listnode *)memblock); ibex_list_addtail(&block_cache->nodes, (struct _listnode *)memblock); + +#ifdef MALLOC_CHECK + check_cache(block_cache); +#endif + #ifdef IBEX_STATS add_hit(block_cache, memblock->block); #endif + +#ifdef MALLOC_CHECK + check_cache(block_cache); +#endif + return &memblock->data; } #ifdef IBEX_STATS @@ -338,6 +409,7 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid) lseek(block_cache->fd, blockid, SEEK_SET); memset(&memblock->data, 0, sizeof(memblock->data)); read(block_cache->fd, &memblock->data, sizeof(memblock->data)); + ibex_list_addtail(&block_cache->nodes, (struct _listnode *)memblock); g_hash_table_insert(block_cache->index, (void *)blockid, memblock); if (block_cache->count >= CACHE_SIZE) { @@ -347,18 +419,14 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid) ibex_list_remove((struct _listnode *)old); if (old->flags & BLOCK_DIRTY) { /* are we about to un-sync the file? update root and sync it */ - if (block_cache->flags & IBEX_ROOT_SYNCF) { - /* TODO: put the rootblock in the block_cache struct, not in the cache */ - struct _memblock *rootblock = g_hash_table_lookup(block_cache->index, (void *)0); - struct _root *root = (struct _root *)&rootblock->data; - + if (block_cache->root.flags & IBEX_ROOT_SYNCF) { printf("Unsyncing root block\n"); - g_assert(rootblock != NULL); - root->flags &= ~IBEX_ROOT_SYNCF; - sync_block(block_cache, rootblock); - if (fsync(block_cache->fd) == 0) - block_cache->flags &= ~IBEX_ROOT_SYNCF; + block_cache->root.flags &= ~IBEX_ROOT_SYNCF; + if (ibex_block_sync_root(block_cache) != 0) { + /* what do we do? i dont know! */ + g_warning("Could not sync root block of index: %s", strerror(errno)); + } } sync_block(block_cache, old); } @@ -369,6 +437,9 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid) d(printf(" --- cached blocks : %d\n", block_cache->count)); +#ifdef MALLOC_CHECK + check_cache(block_cache); +#endif return &memblock->data; } @@ -389,7 +460,6 @@ ibex_block_read(struct _memcache *block_cache, blockid_t blockid) struct _memcache * ibex_block_cache_open(const char *name, int flags, int mode) { - struct _root *root; struct _memcache *block_cache = g_malloc0(sizeof(*block_cache)); d(printf("opening ibex file: %s", name)); @@ -406,33 +476,33 @@ ibex_block_cache_open(const char *name, int flags, int mode) return NULL; } - root = (struct _root *)ibex_block_read(block_cache, 0); - if (root->roof == 0 - || memcmp(root->version, "ibx4", 4) - || ((root->flags & IBEX_ROOT_SYNCF) == 0)) { + ibex_block_read_root(block_cache); + if (block_cache->root.roof == 0 + || memcmp(block_cache->root.version, "ibx5", 4) + || ((block_cache->root.flags & IBEX_ROOT_SYNCF) == 0)) { (printf("Initialising superblock\n")); /* reset root data */ - memcpy(root->version, "ibx4", 4); - root->roof = 1024; - root->free = 0; - root->words = 0; - root->names = 0; - root->tail = 0; /* list of tail blocks */ - root->flags = 0; - ibex_block_dirty((struct _block *)root); + memcpy(block_cache->root.version, "ibx5", 4); + block_cache->root.roof = 1024; + block_cache->root.free = 0; + block_cache->root.words = 0; + block_cache->root.names = 0; + block_cache->root.tail = 0; /* list of tail blocks */ + block_cache->root.flags = 0; + ibex_block_sync_root(block_cache); /* reset the file contents */ ftruncate(block_cache->fd, 1024); } else { (printf("superblock already initialised:\n" - " roof = %d\n free = %d\n words = %d\n names = %d\n tail = %d", - root->roof, root->free, root->words, root->names, root->tail)); + " roof = %d\n free = %d\n words = %d\n names = %d\n tail = %d\n", + block_cache->root.roof, block_cache->root.free, + block_cache->root.words, block_cache->root.names, block_cache->root.tail)); } - block_cache->flags = root->flags; - /* this should be moved higher up */ + /* FIXME: this should be moved higher up in the object tree */ { - struct _IBEXWord *ibex_create_word_index(struct _memcache *bc, blockid_t *wordroot, blockid_t *nameroot); + struct _IBEXWord *ibex_create_word_index_mem(struct _memcache *bc, blockid_t *wordroot, blockid_t *nameroot); - block_cache->words = ibex_create_word_index(block_cache, &root->words, &root->names); + block_cache->words = ibex_create_word_index_mem(block_cache, &block_cache->root.words,&block_cache->root.names); } #ifdef IBEX_STATS @@ -480,12 +550,10 @@ ibex_block_cache_close(struct _memcache *block_cache) void ibex_block_free(struct _memcache *block_cache, blockid_t blockid) { - struct _root *root = (struct _root *)ibex_block_read(block_cache, 0); struct _block *block = ibex_block_read(block_cache, blockid); - block->next = block_number(root->free); - root->free = blockid; - ibex_block_dirty((struct _block *)root); + block->next = block_number(block_cache->root.free); + block_cache->root.free = blockid; ibex_block_dirty((struct _block *)block); } @@ -502,19 +570,18 @@ ibex_block_free(struct _memcache *block_cache, blockid_t blockid) blockid_t ibex_block_get(struct _memcache *block_cache) { - struct _root *root = (struct _root *)ibex_block_read(block_cache, 0); struct _block *block; blockid_t head; - if (root->free) { - head = root->free; + if (block_cache->root.free) { + head = block_cache->root.free; block = ibex_block_read(block_cache, head); - root->free = block_location(block->next); + block_cache->root.free = block_location(block->next); } else { /* TODO: check the block will fit first */ /* TODO: no need to read this block, can allocate it manually (saves a syscall/read) */ - head = root->roof; - root->roof += BLOCK_SIZE; + head = block_cache->root.roof; + block_cache->root.roof += BLOCK_SIZE; block = ibex_block_read(block_cache, head); } @@ -524,6 +591,5 @@ ibex_block_get(struct _memcache *block_cache) block->next = 0; block->used = 0; ibex_block_dirty(block); - ibex_block_dirty((struct _block *)root); return head; } -- cgit v1.2.3