From 854f94bc2016d4501aa7b6be1e78790a9ffb12ae Mon Sep 17 00:00:00 2001 From: Not Zed Date: Mon, 20 Nov 2000 23:54:48 +0000 Subject: Fixes for the summary messageid changes. Hash the messageid and store it. 2000-11-20 Not Zed * providers/nntp/camel-nntp-utils.c (get_XOVER_headers): Fixes for the summary messageid changes. Hash the messageid and store it. (get_XOVER_headers): Use camel_folder_summary_info_new() to create the summary item before adding it. * camel-folder-summary.h (CamelMessageInfo): Changed the messgae-id to be an 8 byte md5 hash, and the references list to be an array of these. * providers/local/camel-mh-summary.c (mh_summary_sync_message): New function, sync out the message info stuff. Only updates the X-Ev header if it can get away with it, otherwise writes out a whole new message. (mh_summary_sync): Added more functionality. All summary info is now written to the X-Ev header, etc, and new messages re-written if required during the sync process. * providers/local/camel-local-folder.c (local_set_message_user_flag): Set the XEVCHANGE flag. (local_set_message_user_tag): And here too. * providers/local/camel-local-summary.h: New flag CAMEL_MESSAGE_FOLDER_XEVCHANGE to indicate the XEV header has probably changed size and needs to be rewritten in whole. * camel-folder-summary.c (next_uid_string): Want this static, not const. (message_info_new): Store the references and message-id values as 64 bit, binary hashes. (message_info_load): fix for message-id/references changes. (message_info_save): Likewise. (camel_message_info_dup_to): And here. (camel_message_info_free): And here too. No longer free message_id, and simple free for references array. (CAMEL_FOLDER_SUMMARY_VERSION): Bumped file revision. (camel_folder_summary_init): Init memchunk allocators to empty. (camel_folder_summary_finalize): Free memchunk allocators if there. (message_info_new): Use the chunk allocator to allocate message info's. (camel_folder_summary_info_new): New helper to allocate the message info, and setup the memchunk if required. (content_info_alloc): Likewise for content info's. (message_info_load): Use summary_info_new_empty. (content_info_new): Use content_info_alloc. (content_info_load): " (content_info_free): Free the content info as a memchunk. (message_info_free): Free everything directly and the base as a memchunk, rather than calling camel_message_info_free(), which assumes a malloc'd array. * providers/local/camel-local-summary.c: Include ctype.h, kill a warning. (local_summary_decode_x_evolution): If we get a NULL message info, then dont try and set anything, just check for validity. (camel_local_summary_write_headers): New function to write a set of headers to an fd. (camel_local_summary_check): Added some statistic generation stuff for memory profiling. * providers/local/camel-mbox-summary.c (header_write): Changed to use stdoi functions to write out the header to a buffered stream, instead of using writev, which is apparently slow (and writing each line separately is slow anyway). (mbox_summary_sync_full): New implementation. Does things differently, doesn't use or require the content info stuff. (summary_rebuild): Dont return an error if we start scanning at the end of file. (mbox_summary_sync_full): If we are not writing out new headers, make sure we copy the From line as we go, and update frompos appropriately. (mbox_summary_sync_full): Always copy the From line from the existing one, rather than trying to make one up ourselves. (mbox_summary_sync): If we can get by with a quick-sync, then try it, if that fails, then try a full sync anyway. (mbox_summary_sync_quick): Quick sync. Only update system flags, etc. (mbox_summary_sync_full): Use the proper local summary encode_xev function. (header_evolution_decode): Removed, no longer needed. (header_evolution_encode): Same. (copy_block): No longer needed, removed. (header_write): Removed, replaced with camel_local_summary_write_headers. (mbox_summary_sync_full): Fixed for header_write change. * camel-mime-parser.c (folder_scan_step): Implement the new optional parser state HSCAN_PRE_FROM, that returns the (currently unfiltered) input data. (folder_scan_drop_step): Do the right thing for the PRE_FROM state. (camel_mime_parser_scan_from): Update the doco. (camel_mime_parser_scan_pre_from): Ok, make this behaviour optional, it simplifies a lot of loops that dont otherwise need to know about it. (folder_scan_step): Made the PRE_FROM state optional. (struct _header_scan_state): Made the bool vars 1 bit. (folder_pull_part): Free the from_line buffer if it is there. (folder_scan_skip_line): Added a new arg, can save the skpped data to a byte_array, as we go. (folder_scan_step): Fixed calls to skip_line approrpiately. Now we save the from line as we parse it. (camel_mime_parser_read): New function to read from the mime parser buffer directly. Useful if you use the parser to read the first/some headers, then need to scan the rest of the data, without needing to use a seek(), or allocate your own buffers. * camel-mime-parser.h (struct _header_state): Added a new parser state, pre-from which returns any data found before a from line during parsing (all other data can be retrieved by the caller except this). svn path=/trunk/; revision=6618 --- camel/ChangeLog | 119 ++++++ camel/camel-folder-summary.c | 171 ++++++-- camel/camel-folder-summary.h | 67 +++- camel/camel-lock.c | 2 +- camel/camel-mime-parser.c | 166 ++++++-- camel/camel-mime-parser.h | 10 + camel/providers/local/camel-local-folder.c | 6 +- camel/providers/local/camel-local-summary.c | 182 ++++++++- camel/providers/local/camel-local-summary.h | 4 + camel/providers/local/camel-mbox-folder.c | 28 +- camel/providers/local/camel-mbox-summary.c | 590 +++++++++++++--------------- camel/providers/local/camel-mh-folder.c | 2 +- camel/providers/local/camel-mh-summary.c | 90 ++++- camel/providers/nntp/camel-nntp-utils.c | 8 +- 14 files changed, 1038 insertions(+), 407 deletions(-) (limited to 'camel') diff --git a/camel/ChangeLog b/camel/ChangeLog index 734bbbe75f..f1919f4b6d 100644 --- a/camel/ChangeLog +++ b/camel/ChangeLog @@ -1,3 +1,117 @@ +2000-11-20 Not Zed + + * providers/nntp/camel-nntp-utils.c (get_XOVER_headers): Fixes for + the summary messageid changes. Hash the messageid and store it. + (get_XOVER_headers): Use camel_folder_summary_info_new() to create + the summary item before adding it. + + * camel-folder-summary.h (CamelMessageInfo): Changed the + messgae-id to be an 8 byte md5 hash, and the references list to be + an array of these. + + * providers/local/camel-mh-summary.c (mh_summary_sync_message): + New function, sync out the message info stuff. Only updates the + X-Ev header if it can get away with it, otherwise writes out a + whole new message. + (mh_summary_sync): Added more functionality. All summary info is + now written to the X-Ev header, etc, and new messages re-written + if required during the sync process. + + * providers/local/camel-local-folder.c + (local_set_message_user_flag): Set the XEVCHANGE flag. + (local_set_message_user_tag): And here too. + + * providers/local/camel-local-summary.h: New flag + CAMEL_MESSAGE_FOLDER_XEVCHANGE to indicate the XEV header has + probably changed size and needs to be rewritten in whole. + + * camel-folder-summary.c (next_uid_string): Want this static, not + const. + (message_info_new): Store the references and message-id values as + 64 bit, binary hashes. + (message_info_load): fix for message-id/references changes. + (message_info_save): Likewise. + (camel_message_info_dup_to): And here. + (camel_message_info_free): And here too. No longer free + message_id, and simple free for references array. + (CAMEL_FOLDER_SUMMARY_VERSION): Bumped file revision. + (camel_folder_summary_init): Init memchunk allocators to empty. + (camel_folder_summary_finalize): Free memchunk allocators if + there. + (message_info_new): Use the chunk allocator to allocate message + info's. + (camel_folder_summary_info_new): New helper to allocate the + message info, and setup the memchunk if required. + (content_info_alloc): Likewise for content info's. + (message_info_load): Use summary_info_new_empty. + (content_info_new): Use content_info_alloc. + (content_info_load): " + (content_info_free): Free the content info as a memchunk. + (message_info_free): Free everything directly and the base as a + memchunk, rather than calling camel_message_info_free(), which + assumes a malloc'd array. + + * providers/local/camel-local-summary.c: Include ctype.h, kill a + warning. + (local_summary_decode_x_evolution): If we get a NULL message info, + then dont try and set anything, just check for validity. + (camel_local_summary_write_headers): New function to write a set + of headers to an fd. + (camel_local_summary_check): Added some statistic generation + stuff for memory profiling. + + * providers/local/camel-mbox-summary.c (header_write): Changed to + use stdoi functions to write out the header to a buffered stream, + instead of using writev, which is apparently slow (and writing + each line separately is slow anyway). + (mbox_summary_sync_full): New implementation. Does things + differently, doesn't use or require the content info stuff. + (summary_rebuild): Dont return an error if we start scanning at + the end of file. + (mbox_summary_sync_full): If we are not writing out new headers, + make sure we copy the From line as we go, and update frompos + appropriately. + (mbox_summary_sync_full): Always copy the From line from the + existing one, rather than trying to make one up ourselves. + (mbox_summary_sync): If we can get by with a quick-sync, then try + it, if that fails, then try a full sync anyway. + (mbox_summary_sync_quick): Quick sync. Only update system flags, + etc. + (mbox_summary_sync_full): Use the proper local summary encode_xev + function. + (header_evolution_decode): Removed, no longer needed. + (header_evolution_encode): Same. + (copy_block): No longer needed, removed. + (header_write): Removed, replaced with + camel_local_summary_write_headers. + (mbox_summary_sync_full): Fixed for header_write change. + + * camel-mime-parser.c (folder_scan_step): Implement the new + optional parser state HSCAN_PRE_FROM, that returns the (currently + unfiltered) input data. + (folder_scan_drop_step): Do the right thing for the PRE_FROM + state. + (camel_mime_parser_scan_from): Update the doco. + (camel_mime_parser_scan_pre_from): Ok, make this behaviour + optional, it simplifies a lot of loops that dont otherwise need to + know about it. + (folder_scan_step): Made the PRE_FROM state optional. + (struct _header_scan_state): Made the bool vars 1 bit. + (folder_pull_part): Free the from_line buffer if it is there. + (folder_scan_skip_line): Added a new arg, can save the skpped data + to a byte_array, as we go. + (folder_scan_step): Fixed calls to skip_line approrpiately. Now + we save the from line as we parse it. + (camel_mime_parser_read): New function to read from the mime + parser buffer directly. Useful if you use the parser to read the + first/some headers, then need to scan the rest of the data, + without needing to use a seek(), or allocate your own buffers. + + * camel-mime-parser.h (struct _header_state): Added a new parser state, + pre-from which returns any data found before a from line during + parsing (all other data can be retrieved by the caller except + this). + 2000-11-17 Jeffrey Stedfast * providers/imap/camel-imap-utils.c (imap_parse_nstring): When @@ -17,6 +131,11 @@ 2000-11-17 Not Zed + * providers/local/camel-local-summary.c (local_summary_add): Clear + the NOXEV/FLAGGED bits, since we do have an xev header. um m, + maybe this is right, this assumes a write is following. Maybe + this should be done in folder::append() instead ... + * camel-stream-buffer.c (camel_stream_buffer_gets): We should always terminate the string. No need to check outptr is in range, its already been checked. diff --git a/camel/camel-folder-summary.c b/camel/camel-folder-summary.c index efb8c1e195..12e276245d 100644 --- a/camel/camel-folder-summary.c +++ b/camel/camel-folder-summary.c @@ -40,6 +40,8 @@ #include #include "hash-table-utils.h" +#include "e-util/md5-utils.h" +#include "e-util/e-memory.h" /* this should probably be conditional on it existing */ #define USE_BSEARCH @@ -51,7 +53,7 @@ extern int strdup_count, malloc_count, free_count; #endif -#define CAMEL_FOLDER_SUMMARY_VERSION (9) +#define CAMEL_FOLDER_SUMMARY_VERSION (10) struct _CamelFolderSummaryPrivate { GHashTable *filter_charset; /* CamelMimeFilterCharset's indexed by source charset */ @@ -91,7 +93,7 @@ static CamelMessageContentInfo * content_info_load(CamelFolderSummary *, FILE *) static int content_info_save(CamelFolderSummary *, FILE *, CamelMessageContentInfo *); static void content_info_free(CamelFolderSummary *, CamelMessageContentInfo *); -const char *next_uid_string(CamelFolderSummary *s); +static char *next_uid_string(CamelFolderSummary *s); static CamelMessageContentInfo * summary_build_content_info(CamelFolderSummary *s, CamelMimeParser *mp); static CamelMessageContentInfo * summary_build_content_info_message(CamelFolderSummary *s, CamelMessageInfo *msginfo, CamelMimePart *object); @@ -139,6 +141,9 @@ camel_folder_summary_init (CamelFolderSummary *s) s->message_info_size = sizeof(CamelMessageInfo); s->content_info_size = sizeof(CamelMessageContentInfo); + s->message_info_chunks = NULL; + s->content_info_chunks = NULL; + s->version = CAMEL_FOLDER_SUMMARY_VERSION; s->flags = 0; s->time = 0; @@ -171,6 +176,11 @@ camel_folder_summary_finalize (CamelObject *obj) g_free(s->summary_path); + if (s->message_info_chunks) + e_memchunk_destroy(s->message_info_chunks); + if (s->content_info_chunks) + e_memchunk_destroy(s->content_info_chunks); + if (p->filter_index) camel_object_unref ((CamelObject *)p->filter_index); if (p->filter_64) @@ -356,8 +366,6 @@ void camel_folder_summary_set_uid(CamelFolderSummary *s, guint32 uid) char * camel_folder_summary_next_uid_string(CamelFolderSummary *s) { - char *(*next_uid_string)(CamelFolderSummary *); - return ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->next_uid_string(s); } @@ -505,7 +513,7 @@ camel_folder_summary_save(CamelFolderSummary *s) * * The @info record should have been generated by calling one of the * info_new_*() functions, as it will be free'd based on the summary - * class. + * class. And MUST NOT be allocated directly using malloc. **/ void camel_folder_summary_add(CamelFolderSummary *s, CamelMessageInfo *info) { @@ -1428,13 +1436,50 @@ camel_folder_summary_format_string(struct _header_raw *h, const char *name) } } +/** + * camel_folder_summary_info_new: + * @s: + * + * Allocate a new camel message info, suitable for adding + * to this summary. + * + * Return value: + **/ +CamelMessageInfo * +camel_folder_summary_info_new(CamelFolderSummary *s) +{ + CamelMessageInfo *mi; + + if (s->message_info_chunks == NULL) + s->message_info_chunks = e_memchunk_new(32, s->message_info_size); + mi = e_memchunk_alloc(s->message_info_chunks); + memset(mi, 0, s->message_info_size); + return mi; +} + +static CamelMessageContentInfo * +content_info_alloc(CamelFolderSummary *s) +{ + CamelMessageContentInfo *ci; + + if (s->content_info_chunks == NULL) + s->content_info_chunks = e_memchunk_new(32, s->content_info_size); + ci = e_memchunk_alloc(s->content_info_chunks); + memset(ci, 0, s->content_info_size); + return ci; +} + static CamelMessageInfo * message_info_new(CamelFolderSummary *s, struct _header_raw *h) { CamelMessageInfo *mi; const char *received; + guchar digest[16]; + struct _header_references *refs, *scan; + char *msgid; + int count; - mi = g_malloc0(s->message_info_size); + mi = camel_folder_summary_info_new(s); mi->subject = camel_folder_summary_format_string(h, "subject"); mi->from = camel_folder_summary_format_address(h, "from"); @@ -1450,12 +1495,34 @@ message_info_new(CamelFolderSummary *s, struct _header_raw *h) mi->date_received = header_decode_date(received + 1, NULL); else mi->date_received = 0; - mi->message_id = header_msgid_decode(header_raw_find(&h, "message-id", NULL)); + + msgid = header_msgid_decode(header_raw_find(&h, "message-id", NULL)); + if (msgid) { + md5_get_digest(msgid, strlen(msgid), digest); + memcpy(mi->message_id.id.hash, digest, sizeof(mi->message_id.id.hash)); + g_free(msgid); + } /* if we have a references, use that, otherwise, see if we have an in-reply-to header, with parsable content, otherwise *shrug* */ - mi->references = header_references_decode(header_raw_find(&h, "references", NULL)); - if (mi->references == NULL) - mi->references = header_references_decode(header_raw_find(&h, "in-reply-to", NULL)); + if ((refs = header_references_decode(header_raw_find(&h, "references", NULL))) != NULL + || (refs = header_references_decode(header_raw_find(&h, "in-reply-to", NULL))) != NULL) { + count = header_references_list_size(&refs); + mi->references = g_malloc(sizeof(*mi->references) + ((count-1) * sizeof(mi->references->references[0]))); + count = 0; + scan = refs; + while (scan) { + /* FIXME: the id might be NULL because of a small bug in camel-mime-utils */ + if (scan->id) { + md5_get_digest(scan->id, strlen(scan->id), digest); + memcpy(mi->references->references[count].id.hash, digest, sizeof(mi->message_id.id.hash)); + count++; + } + scan = scan->next; + } + mi->references->size = count; + header_references_list_clear(&refs); + } + return mi; } @@ -1467,7 +1534,7 @@ message_info_load(CamelFolderSummary *s, FILE *in) guint count; int i; - mi = g_malloc0(s->message_info_size); + mi = camel_folder_summary_info_new(s); io(printf("Loading message info\n")); @@ -1482,13 +1549,17 @@ message_info_load(CamelFolderSummary *s, FILE *in) camel_folder_summary_decode_string(in, &mi->cc); mi->content = NULL; - camel_folder_summary_decode_string(in, &mi->message_id); + camel_folder_summary_decode_fixed_int32(in, &mi->message_id.id.part.hi); + camel_folder_summary_decode_fixed_int32(in, &mi->message_id.id.part.lo); camel_folder_summary_decode_uint32(in, &count); - for (i=0;ireferences, id); + if (count > 0) { + mi->references = g_malloc(sizeof(*mi->references) + ((count-1) * sizeof(mi->references->references[0]))); + mi->references->size = count; + for (i=0;ireferences->references[i].id.part.hi); + camel_folder_summary_decode_fixed_int32(in, &mi->references->references[i].id.part.lo); + } } camel_folder_summary_decode_uint32(in, &count); @@ -1518,7 +1589,7 @@ message_info_save(CamelFolderSummary *s, FILE *out, CamelMessageInfo *mi) guint32 count; CamelFlag *flag; CamelTag *tag; - struct _header_references *refs; + int i; io(printf("Saving message info\n")); @@ -1532,14 +1603,17 @@ message_info_save(CamelFolderSummary *s, FILE *out, CamelMessageInfo *mi) camel_folder_summary_encode_string(out, mi->to); camel_folder_summary_encode_string(out, mi->cc); - camel_folder_summary_encode_string(out, mi->message_id); + camel_folder_summary_encode_fixed_int32(out, mi->message_id.id.part.hi); + camel_folder_summary_encode_fixed_int32(out, mi->message_id.id.part.lo); - count = header_references_list_size(&mi->references); - camel_folder_summary_encode_uint32(out, count); - refs = mi->references; - while (refs) { - camel_folder_summary_encode_string(out, refs->id); - refs = refs->next; + if (mi->references) { + camel_folder_summary_encode_uint32(out, mi->references->size); + for (i=0;ireferences->size;i++) { + camel_folder_summary_encode_fixed_int32(out, mi->references->references[i].id.part.hi); + camel_folder_summary_encode_fixed_int32(out, mi->references->references[i].id.part.lo); + } + } else { + camel_folder_summary_encode_uint32(out, 0); } count = camel_flag_list_size(&mi->user_flags); @@ -1565,7 +1639,15 @@ message_info_save(CamelFolderSummary *s, FILE *out, CamelMessageInfo *mi) static void message_info_free(CamelFolderSummary *s, CamelMessageInfo *mi) { - camel_message_info_free(mi); + g_free(mi->uid); + g_free(mi->subject); + g_free(mi->from); + g_free(mi->to); + g_free(mi->cc); + g_free(mi->references); + camel_flag_list_free(&mi->user_flags); + camel_tag_list_free(&mi->user_tags); + e_memchunk_free(s->message_info_chunks, mi); } static CamelMessageContentInfo * @@ -1573,7 +1655,7 @@ content_info_new(CamelFolderSummary *s, struct _header_raw *h) { CamelMessageContentInfo *ci; - ci = g_malloc0(s->content_info_size); + ci = content_info_alloc(s); ci->id = header_msgid_decode(header_raw_find(&h, "content-id", NULL)); ci->description = header_decode_string(header_raw_find(&h, "content-description", NULL)); @@ -1595,7 +1677,7 @@ content_info_load(CamelFolderSummary *s, FILE *in) io(printf("Loading content info\n")); - ci = g_malloc0(s->content_info_size); + ci = content_info_alloc(s); camel_folder_summary_decode_off_t(in, &ci->pos); camel_folder_summary_decode_off_t(in, &ci->bodypos); @@ -1666,10 +1748,10 @@ content_info_free(CamelFolderSummary *s, CamelMessageContentInfo *ci) g_free(ci->id); g_free(ci->description); g_free(ci->encoding); - g_free(ci); + e_memchunk_free(s->content_info_chunks, ci); } -const char * +static char * next_uid_string(CamelFolderSummary *s) { return g_strdup_printf("%u", camel_folder_summary_next_uid(s)); @@ -2068,10 +2150,18 @@ camel_message_info_dup_to(const CamelMessageInfo *from, CamelMessageInfo *to) to->to = g_strdup(from->to); to->cc = g_strdup(from->cc); to->uid = g_strdup(from->uid); - to->message_id = g_strdup(from->message_id); + memcpy(&to->message_id, &from->message_id, sizeof(from->message_id)); /* Copy structures */ - to->references = header_references_dup(from->references); + if (from->references) { + int len = sizeof(*from->references) + ((from->references->size-1) * sizeof(from->references->references[0])); + + to->references = g_malloc(len); + memcpy(to->references, from->references, len); + } else { + to->references = NULL; + } + flag = from->user_flags; while (flag) { camel_flag_set(&to->user_flags, flag->name, TRUE); @@ -2107,14 +2197,29 @@ camel_message_info_free(CamelMessageInfo *mi) g_free(mi->from); g_free(mi->to); g_free(mi->cc); - g_free(mi->message_id); - header_references_list_clear(&mi->references); + g_free(mi->references); camel_flag_list_free(&mi->user_flags); camel_tag_list_free(&mi->user_tags); /* FIXME: content info? */ g_free(mi); } +#ifdef DOESTRV +const char *camel_message_info_string(CamelMessageInfo *mi, int type) +{ + if (mi->strings == NULL || type >= CAMEL_MESSAGE_INFO_STRING_COUNT) + return ""; + return e_strv_get(mi->strings, type); +} + +void camel_message_info_set_string(CamelMessageInfo *mi, int type, const char *str) +{ + g_assert(mi->strings != NULL); + + mi->strings = e_strv_set(mi->strings, type, str); +} +#endif + #if 0 static void content_info_dump(CamelMessageContentInfo *ci, int depth) diff --git a/camel/camel-folder-summary.h b/camel/camel-folder-summary.h index 65018ba3c8..19f17855bc 100644 --- a/camel/camel-folder-summary.h +++ b/camel/camel-folder-summary.h @@ -48,6 +48,8 @@ typedef struct _CamelMessageContentInfo { char *description; char *encoding; + /* NOTE: The fields below are to be deprecated, and eventually removed */ + /* information about where this object lives in the stream. if pos is -1 these are all invalid */ off_t pos; @@ -78,24 +80,57 @@ typedef struct _CamelTag { char name[1]; /* name allocated as part of the structure */ } CamelTag; +/* a summary messageid is a 64 bit identifier (partial md5 hash) */ +typedef struct _CamelSummaryMessageID { + union { + guint64 id; + unsigned char hash[8]; + struct { + guint32 hi; + guint32 lo; + } part; + } id; +} CamelSummaryMessageID; + +/* summary references is a fixed size array of references */ +typedef struct _CamelSummaryReferences { + int size; + CamelSummaryMessageID references[1]; +} CamelSummaryReferences; + +#ifdef DOESTRV +/* string array indices */ +enum { + CAMEL_MESSAGE_INFO_SUBJECT, + CAMEL_MESSAGE_INFO_FROM, + CAMEL_MESSAGE_INFO_TO, + CAMEL_MESSAGE_INFO_CC, + CAMEL_MESSAGE_INFO_UID, + CAMEL_MESSAGE_INFO_STRING_COUNT, +}; +#endif + /* information about a given object */ typedef struct { /* public fields */ +#ifdef DOESTRV + EStrv *strings; /* all strings packed into a single compact array */ +#else gchar *subject; gchar *from; gchar *to; gchar *cc; gchar *uid; +#endif guint32 flags; guint32 size; time_t date_sent; time_t date_received; - /* Message-ID / References structures */ - char *message_id; /* for this message */ - struct _header_references *references; /* from parent to root */ + CamelSummaryMessageID message_id;/* for this message */ + CamelSummaryReferences *references;/* from parent to root */ struct _CamelFlag *user_flags; struct _CamelTag *user_tags; @@ -123,6 +158,10 @@ struct _CamelFolderSummary { /* sizes of memory objects */ guint32 message_info_size; guint32 content_info_size; + + /* memory allocators (setup automatically) */ + struct _EMemChunk *message_info_chunks; + struct _EMemChunk *content_info_chunks; char *summary_path; gboolean build_content; /* do we try and parse/index the content, or not? */ @@ -185,6 +224,7 @@ CamelMessageInfo *camel_folder_summary_add_from_parser(CamelFolderSummary *, Cam CamelMessageInfo *camel_folder_summary_add_from_message(CamelFolderSummary *, CamelMimeMessage *); /* Just build raw summary items */ +CamelMessageInfo *camel_folder_summary_info_new(CamelFolderSummary *s); CamelMessageInfo *camel_folder_summary_info_new_from_header(CamelFolderSummary *, struct _header_raw *); CamelMessageInfo *camel_folder_summary_info_new_from_parser(CamelFolderSummary *, CamelMimeParser *); CamelMessageInfo *camel_folder_summary_info_new_from_message(CamelFolderSummary *, CamelMimeMessage *); @@ -238,8 +278,27 @@ void camel_tag_set(CamelTag **list, const char *name, const char *value); int camel_tag_list_size(CamelTag **list); void camel_tag_list_free(CamelTag **list); -/* message info utils */ +/* message info utils for working with pseudo-messageinfo structures + NOTE: These cannot be added to a real summary object, but suffice for all + other external interfaces that use message info's */ void camel_message_info_dup_to(const CamelMessageInfo *from, CamelMessageInfo *to); void camel_message_info_free(CamelMessageInfo *mi); +/* accessors */ +#ifdef DOESTRV +const char *camel_message_info_string(CamelMessageInfo *mi, int type); +#define camel_message_info_subject(x) camel_message_info_string(m, CAMEL_MESSAGE_INFO_SUBJECT) +#define camel_message_info_from(x) camel_message_info_string(m, CAMEL_MESSAGE_INFO_FROM) +#define camel_message_info_to(x) camel_message_info_string(m, CAMEL_MESSAGE_INFO_TO) +#define camel_message_info_cc(x) camel_message_info_string(m, CAMEL_MESSAGE_INFO_CC) +#define camel_message_info_uid(x) camel_message_info_string(m, CAMEL_MESSAGE_INFO_UID) + +void camel_message_info_set_string(CamelMessageInfo *mi, int type, const char *str); +#define camel_message_info_set_subject(x, s) camel_message_info_string(m, CAMEL_MESSAGE_INFO_SUBJECT, s) +#define camel_message_info_set_from(x, s) camel_message_info_string(m, CAMEL_MESSAGE_INFO_FROM, s) +#define camel_message_info_set_to(x, s) camel_message_info_string(m, CAMEL_MESSAGE_INFO_TO, s) +#define camel_message_info_set_cc(x, s) camel_message_info_string(m, CAMEL_MESSAGE_INFO_CC, s) +#define camel_message_info_set_uid(x, s) camel_message_info_string(m, CAMEL_MESSAGE_INFO_UID, s) +#endif + #endif /* ! _CAMEL_FOLDER_SUMMARY_H */ diff --git a/camel/camel-lock.c b/camel/camel-lock.c index f14e4ece5d..fdd2c183e7 100644 --- a/camel/camel-lock.c +++ b/camel/camel-lock.c @@ -57,7 +57,7 @@ /* dunno where this fucking thing is got from */ #define _(x) (x) -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ /** * camel_lock_dot: diff --git a/camel/camel-mime-parser.c b/camel/camel-mime-parser.c index 9ee07211ff..b019fa5ee0 100644 --- a/camel/camel-mime-parser.c +++ b/camel/camel-mime-parser.c @@ -225,8 +225,9 @@ struct _header_scan_state { int seek; /* current offset to start of buffer */ int unstep; /* how many states to 'unstep' (repeat the current state) */ - int midline; /* are we mid-line interrupted? */ - int scan_from; /* do we care about From lines? */ + unsigned int midline:1; /* are we mid-line interrupted? */ + unsigned int scan_from:1; /* do we care about From lines? */ + unsigned int scan_pre_from:1; /* do we return pre-from data? */ int start_of_from; /* where from started */ int start_of_headers; /* where headers started from the last scan */ @@ -259,6 +260,8 @@ struct _header_scan_stack { GByteArray *posttext; /* for multipart types, save the post-boundary data here */ int prestage; /* used to determine if it is a pre-boundary or post-boundary data segment */ + GByteArray *from_line; /* the from line */ + char *boundary; /* for multipart/ * boundaries, including leading -- and trailing -- for the final part */ int boundarylen; /* actual length of boundary, including leading -- if there is one */ int boundarylenfinal; /* length of boundary, including trailing -- if there is one */ @@ -279,9 +282,10 @@ static struct _header_scan_state *folder_scan_init(void); static void folder_scan_close(struct _header_scan_state *s); static struct _header_scan_stack *folder_scan_content(struct _header_scan_state *s, int *lastone, char **data, int *length); static struct _header_scan_stack *folder_scan_header(struct _header_scan_state *s, int *lastone); -static int folder_scan_skip_line(struct _header_scan_state *s); +static int folder_scan_skip_line(struct _header_scan_state *s, GByteArray *save); static off_t folder_seek(struct _header_scan_state *s, off_t offset, int whence); static off_t folder_tell(struct _header_scan_state *s); +static int folder_read(struct _header_scan_state *s); #ifdef MEMPOOL static void header_append_mempool(struct _header_scan_state *s, struct _header_scan_stack *h, char *header, int offset); #endif @@ -291,6 +295,7 @@ static void camel_mime_parser_init (CamelMimeParser *obj); static char *states[] = { "HSCAN_INITIAL", + "HSCAN_PRE_FROM", /* pre-from data */ "HSCAN_FROM", /* got 'From' line */ "HSCAN_HEADER", /* toplevel header */ "HSCAN_BODY", /* scanning body of message */ @@ -298,9 +303,9 @@ static char *states[] = { "HSCAN_MESSAGE", /* rfc822/news message */ "HSCAN_PART", /* part of a multipart */ - "", "HSCAN_EOF", /* end of file */ + "HSCAN_PRE_FROM_END", "HSCAN_FROM_END", "HSCAN_HEAER_END", "HSCAN_BODY_END", @@ -532,6 +537,29 @@ camel_mime_parser_postface(CamelMimeParser *m) return NULL; } +/** + * camel_mime_parser_from_line: + * @m: + * + * Get the last scanned "From " line, from a recently scanned from. + * This should only be called in the HSCAN_FROM state. The + * from line will include the closing \n found (if there was one). + * + * The return value will remain valid while in the HSCAN_FROM + * state, or any deeper state. + * + * Return value: The From line, or NULL if called out of context. + **/ +const char * +camel_mime_parser_from_line(CamelMimeParser *m) +{ + struct _header_scan_state *s = _PRIVATE(m); + + if (s->parts) + return byte_array_to_string(s->parts->from_line); + + return NULL; +} /** * camel_mime_parser_init_with_fd: @@ -589,6 +617,11 @@ camel_mime_parser_init_with_stream(CamelMimeParser *m, CamelStream *stream) * If the scanner is scanning from lines, two additional * states HSCAN_FROM and HSCAN_FROM_END will be returned * to the caller during parsing. + * + * This may also be preceeded by an optional + * HSCAN_PRE_FROM state which contains the scanned data + * found before the From line is encountered. See also + * scan_pre_from(). **/ void camel_mime_parser_scan_from(CamelMimeParser *m, int scan_from) @@ -597,6 +630,22 @@ camel_mime_parser_scan_from(CamelMimeParser *m, int scan_from) s->scan_from = scan_from; } +/** + * camel_mime_parser_scan_pre_from: + * @: + * @scan_pre_from: #TRUE if we want to get pre-from data. + * + * Tell the scanner whether we want to know abou the pre-from + * data during a scan. If we do, then we may get an additional + * state HSCAN_PRE_FROM which returns the specified data. + **/ +void +camel_mime_parser_scan_pre_from(CamelMimeParser *m, int scan_pre_from) +{ + struct _header_scan_state *s = _PRIVATE(m); + s->scan_pre_from = scan_pre_from; +} + /** * camel_mime_parser_content_type: * @m: @@ -706,6 +755,52 @@ camel_mime_parser_step(CamelMimeParser *m, char **databuffer, int *datalength) return s->state; } +/** + * camel_mime_parser_read: + * @m: + * @databuffer: + * @len: + * + * Read at most @len bytes from the internal mime parser buffer. + * + * Returns the address of the internal buffer in @databuffer, + * and the length of useful data. + * + * @len may be specified as INT_MAX, in which case you will + * get the full remainder of the buffer at each call. + * + * Note that no parsing of the data read through this function + * occurs, so no state changes occur, but the seek position + * is updated appropriately. + * + * Return value: The number of bytes available, or -1 on error. + **/ +int +camel_mime_parser_read(CamelMimeParser *m, const char **databuffer, int len) +{ + struct _header_scan_state *s = _PRIVATE(m); + int there; + + if (len == 0) + return 0; + + there = MIN(s->inend - s->inptr, len); + if (there > 0) { + *databuffer = s->inptr; + s->inptr += there; + return there; + } + + if (folder_read(s) == -1) + return -1; + + there = MIN(s->inend - s->inptr, len); + *databuffer = s->inptr; + s->inptr += there; + + return there; +} + /** * camel_mime_parser_tell: * @m: @@ -976,6 +1071,8 @@ folder_pull_part(struct _header_scan_state *s) g_byte_array_free(h->pretext, TRUE); if (h->posttext) g_byte_array_free(h->posttext, TRUE); + if (h->from_line) + g_byte_array_free(h->from_line, TRUE); g_free(h); } else { g_warning("Header stack underflow!\n"); @@ -983,7 +1080,7 @@ folder_pull_part(struct _header_scan_state *s) } static int -folder_scan_skip_line(struct _header_scan_state *s) +folder_scan_skip_line(struct _header_scan_state *s, GByteArray *save) { int atleast = s->atleast; register char *inptr, *inend, c; @@ -1000,6 +1097,9 @@ folder_scan_skip_line(struct _header_scan_state *s) && (c = *inptr++)!='\n') ; + if (save) + g_byte_array_append(save, s->inptr, inptr-s->inptr); + s->inptr = inptr; if (c=='\n') { @@ -1390,6 +1490,7 @@ folder_scan_init(void) s->midline = FALSE; s->scan_from = FALSE; + s->scan_pre_from = FALSE; s->filters = NULL; s->filterid = 1; @@ -1483,37 +1584,49 @@ tail_recurse: switch (s->state) { - case HSCAN_INITIAL: #ifdef USE_FROM + case HSCAN_INITIAL: if (s->scan_from) { - /* FIXME: it would be nice not to have to allocate this every pass */ h = g_malloc0(sizeof(*h)); h->boundary = g_strdup("From "); h->boundarylen = strlen(h->boundary); h->boundarylenfinal = h->boundarylen; + h->from_line = g_byte_array_new(); folder_push_part(s, h); - - h = s->parts; - do { - hb = folder_scan_content(s, &state, databuffer, datalength); - } while (hb==h && *datalength>0); - - if (*datalength==0 && hb==h) { - d(printf("found 'From '\n")); - s->start_of_from = folder_tell(s); - folder_scan_skip_line(s); - h->savestate = HSCAN_INITIAL; - s->state = HSCAN_FROM; - } else { - folder_pull_part(s); - s->state = HSCAN_EOF; - } - return; + s->state = HSCAN_PRE_FROM; } else { s->start_of_from = -1; + goto scan_header; } -#endif + case HSCAN_PRE_FROM: + + h = s->parts; + do { + hb = folder_scan_content(s, &state, databuffer, datalength); + if (s->scan_pre_from && *datalength > 0) { + d(printf("got pre-from content %d bytes\n", *datalength)); + return; + } + } while (hb==h && *datalength>0); + + if (*datalength==0 && hb==h) { + d(printf("found 'From '\n")); + s->start_of_from = folder_tell(s); + folder_scan_skip_line(s, h->from_line); + h->savestate = HSCAN_INITIAL; + s->state = HSCAN_FROM; + } else { + folder_pull_part(s); + s->state = HSCAN_EOF; + } + return; +#else + case HSCAN_INITIAL: + case HSCAN_PRE_FROM: +#endif /* !USE_FROM */ + + scan_header: case HSCAN_FROM: s->start_of_headers = folder_tell(s); h = folder_scan_header(s, &state); @@ -1636,7 +1749,7 @@ tail_recurse: h->prestage++; if (*datalength==0 && hb==h) { d(printf("got boundary: %s\n", hb->boundary)); - folder_scan_skip_line(s); + folder_scan_skip_line(s, NULL); if (!state) { s->state = HSCAN_FROM; folder_scan_step(s, databuffer, datalength); @@ -1688,6 +1801,7 @@ folder_scan_drop_step(struct _header_scan_state *s) return; case HSCAN_FROM: + case HSCAN_PRE_FROM: s->state = HSCAN_INITIAL; folder_pull_part(s); return; diff --git a/camel/camel-mime-parser.h b/camel/camel-mime-parser.h index d9a6e8bbb3..477dd4288a 100644 --- a/camel/camel-mime-parser.h +++ b/camel/camel-mime-parser.h @@ -38,6 +38,7 @@ typedef struct _CamelMimeParserClass CamelMimeParserClass; the same as the matching start tag, with a bit difference */ enum _header_state { HSCAN_INITIAL, + HSCAN_PRE_FROM, /* data before a 'From' line */ HSCAN_FROM, /* got 'From' line */ HSCAN_HEADER, /* toplevel header */ HSCAN_BODY, /* scanning body of message */ @@ -49,6 +50,7 @@ enum _header_state { HSCAN_END = 8, /* bit mask for 'end' flags */ HSCAN_EOF = 8, /* end of file */ + HSCAN_PRE_FROM_END, /* pre from end */ HSCAN_FROM_END, /* end of whole from bracket */ HSCAN_HEADER_END, /* dummy value */ HSCAN_BODY_END, /* end of message */ @@ -84,6 +86,8 @@ int camel_mime_parser_fd(CamelMimeParser *m); /* scan 'From' separators? */ void camel_mime_parser_scan_from(CamelMimeParser *, int); +/* Do we want to know about the pre-from data? */ +void camel_mime_parser_scan_pre_from(CamelMimeParser *, int); /* what headers to save, MUST include ^Content-Type: */ int camel_mime_parser_set_header_regex(CamelMimeParser *m, char *matchstr); @@ -94,6 +98,9 @@ void camel_mime_parser_unstep(CamelMimeParser *); void camel_mime_parser_drop_step(CamelMimeParser *m); enum _header_state camel_mime_parser_state(CamelMimeParser *); +/* read through the parser */ +int camel_mime_parser_read(CamelMimeParser *m, const char **databuffer, int len); + /* get content type for the current part/header */ struct _header_content_type *camel_mime_parser_content_type(CamelMimeParser *); @@ -107,6 +114,9 @@ struct _header_raw *camel_mime_parser_headers_raw(CamelMimeParser *); const char *camel_mime_parser_preface(CamelMimeParser *m); const char *camel_mime_parser_postface(CamelMimeParser *m); +/* return the from line content */ +const char *camel_mime_parser_from_line(CamelMimeParser *m); + /* add a processing filter for body contents */ int camel_mime_parser_filter_add(CamelMimeParser *, CamelMimeFilter *); void camel_mime_parser_filter_remove(CamelMimeParser *, int); diff --git a/camel/providers/local/camel-local-folder.c b/camel/providers/local/camel-local-folder.c index bec2d7fb56..16ca2d8b2e 100644 --- a/camel/providers/local/camel-local-folder.c +++ b/camel/providers/local/camel-local-folder.c @@ -42,7 +42,7 @@ #include "camel-mime-filter-from.h" #include "camel-exception.h" -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ static CamelFolderClass *parent_class = NULL; @@ -455,7 +455,7 @@ local_set_message_user_flag(CamelFolder *folder, const char *uid, const char *na g_return_if_fail(info != NULL); camel_flag_set(&info->user_flags, name, value); - info->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED; + info->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED|CAMEL_MESSAGE_FOLDER_XEVCHANGE; camel_folder_summary_touch(CAMEL_FOLDER_SUMMARY(mf->summary)); camel_object_trigger_event(CAMEL_OBJECT(folder), "message_changed", (char *) uid); } @@ -482,7 +482,7 @@ local_set_message_user_tag(CamelFolder *folder, const char *uid, const char *nam g_return_if_fail(info != NULL); camel_tag_set(&info->user_tags, name, value); - info->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED; + info->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED|CAMEL_MESSAGE_FOLDER_XEVCHANGE; camel_folder_summary_touch(CAMEL_FOLDER_SUMMARY(mf->summary)); camel_object_trigger_event(CAMEL_OBJECT(folder), "message_changed", (char *) uid); } diff --git a/camel/providers/local/camel-local-summary.c b/camel/providers/local/camel-local-summary.c index 7455e96cf8..5ba470990b 100644 --- a/camel/providers/local/camel-local-summary.c +++ b/camel/providers/local/camel-local-summary.c @@ -23,6 +23,8 @@ #include "camel-local-summary.h" #include +#include + #include #include #include @@ -31,7 +33,7 @@ #include #define io(x) -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ #define CAMEL_LOCAL_SUMMARY_VERSION (0x100) @@ -166,10 +168,114 @@ camel_local_summary_decode_x_evolution(CamelLocalSummary *cls, const char *xev, return ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->decode_x_evolution(cls, xev, info); } +#define DOSTATS +#ifdef DOSTATS +struct _stat_info { + int mitotal; + int micount; + int citotal; + int cicount; + int msgid; + int msgcount; +}; + +static void +do_stat_ci(CamelLocalSummary *cls, struct _stat_info *info, CamelMessageContentInfo *ci) +{ + info->cicount++; + info->citotal += ((CamelFolderSummary *)cls)->content_info_size /*+ 4 memchunks are 1/4 byte overhead per mi */; + if (ci->id) + info->citotal += strlen(ci->id) + 4; + if (ci->description) + info->citotal += strlen(ci->description) + 4; + if (ci->encoding) + info->citotal += strlen(ci->encoding) + 4; + if (ci->type) { + struct _header_content_type *ct = ci->type; + struct _header_param *param; + + info->citotal += sizeof(*ct) + 4; + if (ct->type) + info->citotal += strlen(ct->type) + 4; + if (ct->subtype) + info->citotal += strlen(ct->subtype) + 4; + param = ct->params; + while (param) { + info->citotal += sizeof(*param) + 4; + if (param->name) + info->citotal += strlen(param->name)+4; + if (param->value) + info->citotal += strlen(param->value)+4; + param = param->next; + } + } + ci = ci->childs; + while (ci) { + do_stat_ci(cls, info, ci); + ci = ci->next; + } +} + +static void +do_stat_mi(CamelLocalSummary *cls, struct _stat_info *info, CamelMessageInfo *mi) +{ + info->micount++; + info->mitotal += ((CamelFolderSummary *)cls)->content_info_size /*+ 4*/; + + if (mi->subject) + info->mitotal += strlen(mi->subject) + 4; + if (mi->to) + info->mitotal += strlen(mi->to) + 4; + if (mi->from) + info->mitotal += strlen(mi->from) + 4; + if (mi->cc) + info->mitotal += strlen(mi->cc) + 4; + if (mi->uid) + info->mitotal += strlen(mi->uid) + 4; + + if (mi->references) { + info->mitotal += (mi->references->size-1) * sizeof(CamelSummaryMessageID) + sizeof(CamelSummaryReferences) + 4; + info->msgid += (mi->references->size) * sizeof(CamelSummaryMessageID); + info->msgcount += mi->references->size; + } + + /* dont have any user flags yet */ + + if (mi->content) { + do_stat_ci(cls, info, mi->content); + } +} + +#endif + int camel_local_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changeinfo, CamelException *ex) { - return ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->check(cls, changeinfo, ex); + int ret; + + ret = ((CamelLocalSummaryClass *)(CAMEL_OBJECT_GET_CLASS(cls)))->check(cls, changeinfo, ex); + +#ifdef DOSTATS + if (ret != -1) { + int i; + CamelFolderSummary *s = (CamelFolderSummary *)cls; + struct _stat_info stats = { 0 }; + + for (i=0;iadd(cls, msg, info, ci, ex); } +/** + * camel_local_summary_write_headers: + * @fd: + * @header: + * @xevline: + * + * Write a bunch of headers to the file @fd. IF xevline is non NULL, then + * an X-Evolution header line is created at the end of all of the headers. + * The headers written are termianted with a blank line. + * + * Return value: -1 on error, otherwise the number of bytes written. + **/ +int +camel_local_summary_write_headers(int fd, struct _header_raw *header, char *xevline) +{ + int outlen = 0, len; + int newfd; + FILE *out; + + /* dum de dum, maybe the whole sync function should just use stdio for output */ + newfd = dup(fd); + if (newfd == -1) + return -1; + + out = fdopen(newfd, "w"); + if (out == NULL) { + close(newfd); + errno = EINVAL; + return -1; + } + + while (header) { + if (strcasecmp(header->name, "X-Evolution")) { + len = fprintf(out, "%s:%s\n", header->name, header->value); + if (len == -1) { + fclose(out); + return -1; + } + outlen += len; + } + header = header->next; + } + + if (xevline) { + len = fprintf(out, "X-Evolution: %s\n\n", xevline); + if (len == -1) { + fclose(out); + return -1; + } + outlen += len; + } + + if (fclose(out) == -1) + return -1; + + return outlen; +} + #if 0 static int summary_header_load(CamelFolderSummary *s, FILE *in) @@ -271,6 +435,7 @@ local_summary_add(CamelLocalSummary *cls, CamelMimeMessage *msg, const CamelMess mi->flags = mi->flags | (info->flags & 0xffff); } + mi->flags &= ~(CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_FLAGGED); xev = camel_local_summary_encode_x_evolution(cls, mi); camel_medium_set_header((CamelMedium *)msg, "X-Evolution", xev); g_free(xev); @@ -351,16 +516,21 @@ local_summary_decode_x_evolution(CamelLocalSummary *cls, const char *xev, CamelM if (header && strlen(header) == strlen("00000000-0000") && sscanf(header, "%08x-%04x", &uid, &flags) == 2) { char uidstr[20]; - sprintf(uidstr, "%u", uid); - g_free(mi->uid); - mi->uid = g_strdup(uidstr); - mi->flags = flags; + if (mi) { + sprintf(uidstr, "%u", uid); + g_free(mi->uid); + mi->uid = g_strdup(uidstr); + mi->flags = flags; + } } else { g_free(header); return -1; } g_free(header); + if (mi == NULL) + return 0; + /* check for additional data */ header = strchr(xev, ';'); if (header) { diff --git a/camel/providers/local/camel-local-summary.h b/camel/providers/local/camel-local-summary.h index f1816e06e5..5349194edf 100644 --- a/camel/providers/local/camel-local-summary.h +++ b/camel/providers/local/camel-local-summary.h @@ -37,6 +37,7 @@ typedef struct _CamelLocalSummaryClass CamelLocalSummaryClass; /* extra summary flags */ enum { CAMEL_MESSAGE_FOLDER_NOXEV = 1<<17, + CAMEL_MESSAGE_FOLDER_XEVCHANGE = 1<<18, }; struct _CamelLocalSummary { @@ -77,5 +78,8 @@ CamelMessageInfo *camel_local_summary_add(CamelLocalSummary *cls, CamelMimeMessa char *camel_local_summary_encode_x_evolution(CamelLocalSummary *cls, const CamelMessageInfo *info); int camel_local_summary_decode_x_evolution(CamelLocalSummary *cls, const char *xev, CamelMessageInfo *info); +/* utility functions - write headers to a file with optional X-Evolution header */ +int camel_local_summary_write_headers(int fd, struct _header_raw *header, char *xevline); + #endif /* ! _CAMEL_LOCAL_SUMMARY_H */ diff --git a/camel/providers/local/camel-mbox-folder.c b/camel/providers/local/camel-mbox-folder.c index cfffbf7c16..d5e76f7cab 100644 --- a/camel/providers/local/camel-mbox-folder.c +++ b/camel/providers/local/camel-mbox-folder.c @@ -42,7 +42,7 @@ #include "camel-mime-filter-from.h" #include "camel-exception.h" -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ static CamelLocalFolderClass *parent_class = NULL; @@ -118,7 +118,7 @@ camel_mbox_folder_new(CamelStore *parent_store, const char *full_name, guint32 f { CamelFolder *folder; - d(printf("Creating mbox folder: %s\n", full_name)); + d(printf("Creating mbox folder: %s in %s\n", full_name, camel_local_store_get_toplevel_dir((CamelLocalStore *)parent_store))); folder = (CamelFolder *)camel_object_new(CAMEL_MBOX_FOLDER_TYPE); folder = (CamelFolder *)camel_local_folder_construct((CamelLocalFolder *)folder, @@ -169,7 +169,9 @@ mbox_append_message(CamelFolder *folder, CamelMimeMessage * message, const Camel char *fromline = NULL; int fd; struct stat st; - +#if 0 + char *xev; +#endif /* If we can't lock, dont do anything */ if (camel_local_folder_lock(lf, CAMEL_LOCK_WRITE, ex) == -1) return; @@ -186,9 +188,6 @@ mbox_append_message(CamelFolder *folder, CamelMimeMessage * message, const Camel if (camel_exception_is_set(ex)) goto fail; - /* and we need to set the frompos explicitly */ - ((CamelMboxMessageInfo *)mi)->frompos = mbs->folder_size?mbs->folder_size+1:0; - d(printf("Appending message: uid is %s\n", mi->uid)); output_stream = camel_stream_fs_new_with_name(lf->folder_path, O_WRONLY|O_APPEND, 0600); @@ -197,6 +196,18 @@ mbox_append_message(CamelFolder *folder, CamelMimeMessage * message, const Camel goto fail; } + /* and we need to set the frompos/XEV explicitly */ + ((CamelMboxMessageInfo *)mi)->frompos = mbs->folder_size?mbs->folder_size+1:0; +#if 0 + xev = camel_local_summary_encode_x_evolution(lf->summary, mi); + if (xev) { + /* the x-ev header should match the 'current' flags, no problem, so store as much */ + camel_medium_set_header((CamelMedium *)message, "X-Evolution", xev); + mi->flags &= ~ CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_FLAGGED; + g_free(xev); + } +#endif + /* we must write this to the non-filtered stream ... prepend a \n if not at the start of the file */ fromline = camel_mbox_summary_build_from(((CamelMimePart *)message)->headers); if (camel_stream_printf(output_stream, mbs->folder_size==0?"%s":"\n%s", fromline) == -1) @@ -330,8 +341,9 @@ retry: || camel_mime_parser_tell_start_from(parser) != info->frompos) { g_warning("Summary doesn't match the folder contents! eek!\n" - " expecting offset %ld got %ld", (long int)info->frompos, - (long int)camel_mime_parser_tell_start_from(parser)); + " expecting offset %ld got %ld, state = %d", (long int)info->frompos, + (long int)camel_mime_parser_tell_start_from(parser), + camel_mime_parser_state(parser)); camel_object_unref((CamelObject *)parser); diff --git a/camel/providers/local/camel-mbox-summary.c b/camel/providers/local/camel-mbox-summary.c index adff872cc4..b4c2d151a5 100644 --- a/camel/providers/local/camel-mbox-summary.c +++ b/camel/providers/local/camel-mbox-summary.c @@ -31,7 +31,7 @@ #include #define io(x) -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ #define CAMEL_MBOX_SUMMARY_VERSION (0x1000) @@ -157,31 +157,6 @@ summary_header_save(CamelFolderSummary *s, FILE *out) return camel_folder_summary_encode_uint32(out, mbs->folder_size); } -static int -header_evolution_decode(const char *in, guint32 *uid, guint32 *flags) -{ - char *header; - - if (in && (header = header_token_decode(in))) { - if (strlen (header) == strlen ("00000000-0000") - && sscanf (header, "%08x-%04x", uid, flags) == 2) { - g_free(header); - return *uid; - } - g_free(header); - } - - return -1; -} - -/* we still use our own version here, as we dont grok the flag stuff yet, during an expunge - anyway */ -static char * -header_evolution_encode(guint32 uid, guint32 flags) -{ - return g_strdup_printf("%08x-%04x", uid, flags & 0xffff); -} - static CamelMessageInfo * message_info_new(CamelFolderSummary *s, struct _header_raw *h) { @@ -250,6 +225,9 @@ summary_rebuild(CamelMboxSummary *mbs, off_t offset, CamelException *ex) int fd; int ok = 0; + /* FIXME: If there is a failure, it shouldn't clear the summary and restart, + it should try and merge the summary info's. This is a bit tricky. */ + fd = open(cls->folder_path, O_RDONLY); if (fd == -1) { printf("%s failed to open: %s", cls->folder_path, strerror(errno)); @@ -275,9 +253,10 @@ summary_rebuild(CamelMboxSummary *mbs, off_t offset, CamelException *ex) camel_mime_parser_unstep(mp); } } else { + d(printf("mime parser state ran out? state is %d\n", camel_mime_parser_state(mp))); camel_object_unref(CAMEL_OBJECT(mp)); - /* end of file - no content? */ - return -1; + /* end of file - no content? no error either */ + return 0; } } @@ -302,6 +281,7 @@ summary_rebuild(CamelMboxSummary *mbs, off_t offset, CamelException *ex) struct stat st; if (stat(cls->folder_path, &st) == 0) { + camel_folder_summary_touch(s); mbs->folder_size = st.st_size; s->time = st.st_mtime; } @@ -312,11 +292,13 @@ summary_rebuild(CamelMboxSummary *mbs, off_t offset, CamelException *ex) /* like summary_rebuild, but also do changeinfo stuff (if supplied) */ static int -summary_update(CamelMboxSummary *mbs, off_t offset, CamelFolderChangeInfo *changeinfo, CamelException *ex) +summary_update(CamelLocalSummary *cls, off_t offset, CamelFolderChangeInfo *changeinfo, CamelException *ex) { int ret, i, count; - CamelFolderSummary *s = (CamelFolderSummary *)mbs; - CamelLocalSummary *cls = (CamelLocalSummary *)mbs; + CamelFolderSummary *s = (CamelFolderSummary *)cls; + CamelMboxSummary *mbs = (CamelMboxSummary *)cls; + + d(printf("Calling summary update, from pos %d\n", (int)offset)); if (changeinfo) { /* we use the diff function of the change_info to build the update list. */ @@ -368,18 +350,18 @@ mbox_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Camel } else if (s->messages->len == 0) { /* if we are empty, then we rebuilt from scratch */ d(printf("Empty summary, rebuilding from start\n")); - ret = summary_update(mbs, 0, changes, ex); + ret = summary_update(cls, 0, changes, ex); } else { /* is the summary uptodate? */ if (st.st_size != mbs->folder_size || st.st_mtime != s->time) { if (mbs->folder_size < st.st_size) { /* this will automatically rescan from 0 if there is a problem */ d(printf("folder grew, attempting to rebuild from %d\n", mbs->folder_size)); - ret = summary_update(mbs, mbs->folder_size, changes, ex); + ret = summary_update(cls, mbs->folder_size, changes, ex); } else { d(printf("folder shrank! rebuilding from start\n")); camel_folder_summary_clear(s); - ret = summary_update(mbs, 0, changes, ex); + ret = summary_update(cls, 0, changes, ex); } } } @@ -401,104 +383,6 @@ mbox_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Camel return ret; } -static int -header_write(int fd, struct _header_raw *header, char *xevline) -{ - struct iovec iv[4]; - int outlen = 0, len; - - iv[1].iov_base = ":"; - iv[1].iov_len = 1; - iv[3].iov_base = "\n"; - iv[3].iov_len = 1; - - while (header) { - if (strcasecmp(header->name, "X-Evolution")) { - iv[0].iov_base = header->name; - iv[0].iov_len = strlen(header->name); - iv[2].iov_base = header->value; - iv[2].iov_len = strlen(header->value); - - do { - len = writev(fd, iv, 4); - } while (len == -1 && errno == EINTR); - - if (len == -1) - return -1; - outlen += len; - } - header = header->next; - } - - iv[0].iov_base = "X-Evolution: "; - iv[0].iov_len = strlen(iv[0].iov_base); - iv[1].iov_base = xevline; - iv[1].iov_len = strlen(xevline); - iv[2].iov_base = "\n\n"; - iv[2].iov_len = 2; - - do { - len = writev(fd, iv, 3); - } while (len == -1 && errno == EINTR); - - if (len == -1) - return -1; - - outlen += 1; - - d(printf("Wrote %d bytes of headers\n", outlen)); - - return outlen; -} - -static int -copy_block(int fromfd, int tofd, off_t start, size_t bytes) -{ - char buffer[4096]; - int written = 0; - - d(printf("writing %d bytes ... \n", bytes)); - - if (lseek(fromfd, start, SEEK_SET) != start) - return -1; - - while (bytes > 0) { - int toread, towrite; - - toread = bytes; - if (bytes > 4096) - toread = 4096; - else - toread = bytes; - do { - towrite = read(fromfd, buffer, toread); - } while (towrite == -1 && errno == EINTR); - - if (towrite == -1) - return -1; - - /* check for 'end of file' */ - if (towrite == 0) { - d(printf("end of file?\n")); - break; - } - - do { - toread = write(tofd, buffer, towrite); - } while (toread == -1 && errno == EINTR); - - if (toread == -1) - return -1; - - written += toread; - bytes -= toread; - } - - d(printf("written %d bytes\n", written)); - - return written; -} - static char *tz_months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" @@ -567,8 +451,9 @@ camel_mbox_summary_build_from(struct _header_raw *header) return ret; } +/* perform a full sync */ static int -mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex) +mbox_summary_sync_full(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex) { CamelMboxSummary *mbs = (CamelMboxSummary *)cls; CamelFolderSummary *s = (CamelFolderSummary *)mbs; @@ -576,41 +461,15 @@ mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInf int i, count; CamelMboxMessageInfo *info; int fd = -1, fdout = -1; - off_t offset = 0; char *tmpname = NULL; char *buffer, *xevnew = NULL; - const char *xev; int len; - guint32 uid, flags; - int quick = TRUE, work = FALSE; - struct stat st; - char *fromline; - - /* make sure we're in sync, after this point we at least have a complete list of id's */ - summary_update(mbs, mbs->folder_size, changeinfo, ex); - - if (camel_exception_is_set(ex)) - return -1; - - /* FIXME: This needs to take the user flags and tags fields into account */ - - /* check if we have any work to do */ - d(printf("Performing sync, %d messages in inbox\n", count)); - for (i = 0; quick && i < count; i++) { - info = (CamelMboxMessageInfo *)camel_folder_summary_index(s, i); - if ((expunge && (info->info.flags & CAMEL_MESSAGE_DELETED)) || - (info->info.flags & CAMEL_MESSAGE_FOLDER_NOXEV)) - quick = FALSE; - else - work |= (info->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED) != 0; - } + const char *fromline; + int lastdel = FALSE; - d(printf("Options: %s %s %s\n", expunge ? "expunge" : "", quick ? "quick" : "", work ? "Work" : "")); + d(printf("performing full summary/sync\n")); - if (quick && !work) - return 0; - - fd = open(cls->folder_path, O_RDWR); + fd = open(cls->folder_path, O_RDONLY); if (fd == -1) { camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Could not open folder to summarise: %s: %s"), @@ -620,46 +479,52 @@ mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInf mp = camel_mime_parser_new(); camel_mime_parser_scan_from(mp, TRUE); + camel_mime_parser_scan_pre_from(mp, TRUE); camel_mime_parser_init_with_fd(mp, fd); - if (!quick) { - tmpname = alloca(strlen (cls->folder_path) + 5); - sprintf(tmpname, "%s.tmp", cls->folder_path); - d(printf("Writing tmp file to %s\n", tmpname)); - retry_out: - fdout = open(tmpname, O_WRONLY|O_CREAT|O_EXCL, 0600); - if (fdout == -1) { - if (errno == EEXIST) - if (unlink(tmpname) != -1) - goto retry_out; - - tmpname = NULL; - camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Cannot open temporary mailbox: %s"), strerror(errno)); - goto error; - } + tmpname = alloca(strlen (cls->folder_path) + 5); + sprintf(tmpname, "%s.tmp", cls->folder_path); + d(printf("Writing tmp file to %s\n", tmpname)); + fdout = open(tmpname, O_WRONLY|O_CREAT|O_TRUNC, 0600); + if (fdout == -1) { + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Cannot open temporary mailbox: %s"), strerror(errno)); + goto error; } + count = camel_folder_summary_count(s); for (i = 0; i < count; i++) { - off_t frompos, bodypos, lastpos; - /* This has to be an int, not an off_t, because that's - * what camel_mime_parser_header returns... FIXME. - */ - int xevoffset; - info = (CamelMboxMessageInfo *)camel_folder_summary_index(s, i); g_assert(info); d(printf("Looking at message %s\n", info->info.uid)); + /* only need to seek past deleted messages, otherwise we should be at the right spot/state already */ + if (lastdel) { + d(printf("seeking to %d\n", (int)info->frompos)); + camel_mime_parser_seek(mp, info->frompos, SEEK_SET); + } + + if (camel_mime_parser_step(mp, &buffer, &len) != HSCAN_FROM) { + g_warning("Expected a From line here, didn't get it"); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Summary and folder mismatch, even after a sync")); + goto error; + } + + if (camel_mime_parser_tell_start_from(mp) != info->frompos) { + g_warning("Didn't get the next message where I expected (%d) got %d instead", + (int)info->frompos, (int)camel_mime_parser_tell_start_from(mp)); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Summary and folder mismatch, even after a sync")); + goto error; + } + + lastdel = FALSE; if (expunge && info->info.flags & CAMEL_MESSAGE_DELETED) { d(printf("Deleting %s\n", info->info.uid)); - g_assert(!quick); - offset -= (info->info.content->endpos - info->frompos); - - /* FIXME: put this in folder_summary::remove()? */ if (cls->index) ibex_unindex(cls->index, info->info.uid); @@ -669,113 +534,54 @@ mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInf count--; i--; info = NULL; - } else if (info->info.flags & (CAMEL_MESSAGE_FOLDER_NOXEV | CAMEL_MESSAGE_FOLDER_FLAGGED)) { - int xevok = FALSE; + lastdel = TRUE; + } else { + /* otherwise, the message is staying, copy its From_ line across */ + if (i>0) { + write(fdout, "\n", 1); + } + info->frompos = lseek(fdout, 0, SEEK_CUR); + fromline = camel_mime_parser_from_line(mp); + write(fdout, fromline, strlen(fromline)); + } + if (info && info->info.flags & (CAMEL_MESSAGE_FOLDER_NOXEV | CAMEL_MESSAGE_FOLDER_FLAGGED)) { d(printf("Updating header for %s flags = %08x\n", info->info.uid, info->info.flags)); - /* find the next message, header parts */ - camel_mime_parser_seek(mp, info->frompos, SEEK_SET); - if (camel_mime_parser_step(mp, &buffer, &len) != HSCAN_FROM) { - g_warning("camel_mime_parser_step failed (1)"); - goto error; - } - - if (camel_mime_parser_tell_start_from (mp) != info->frompos) { - g_warning("Summary/mbox mismatch, aborting sync"); - camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM, - _("Summary mismatch, aborting sync")); - goto error; - } - - if (camel_mime_parser_step (mp, &buffer, &len) == HSCAN_FROM_END) { + if (camel_mime_parser_step(mp, &buffer, &len) == HSCAN_FROM_END) { g_warning("camel_mime_parser_step failed (2)"); goto error; } - /* Check if the X-Evolution header is valid. */ - - /* FIXME: Use camel_local_summary versions here */ - - xev = camel_mime_parser_header(mp, "X-Evolution", &xevoffset); - if (xev && header_evolution_decode (xev, &uid, &flags) != -1) - xevok = TRUE; - - xevnew = header_evolution_encode(strtoul (info->info.uid, NULL, 10), info->info.flags & 0xffff); - if (quick) { - if (!xevok) { - g_warning("The summary told me I had an X-Evolution header, but i dont!"); - camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Summary mismatch, X-Evolution header missing")); - goto error; - } - buffer = g_strdup_printf("X-Evolution: %s", xevnew); - lastpos = lseek(fd, 0, SEEK_CUR); - lseek(fd, xevoffset, SEEK_SET); - do { - len = write(fd, buffer, strlen (buffer)); - } while (len == -1 && errno == EINTR); - lseek(fd, lastpos, SEEK_SET); - g_free(buffer); - if (len == -1) { - goto error; - } - } else { - frompos = lseek(fdout, 0, SEEK_CUR); - fromline = camel_mbox_summary_build_from(camel_mime_parser_headers_raw (mp)); - write(fdout, fromline, strlen(fromline)); - g_free(fromline); - if (header_write(fdout, camel_mime_parser_headers_raw(mp), xevnew) == -1) { - d(printf("Error writing to tmp mailbox\n")); - camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Error writing to temp mailbox: %s"), - strerror(errno)); - goto error; - } - bodypos = lseek(fdout, 0, SEEK_CUR); - d(printf("pos = %d, endpos = %d, bodypos = %d\n", - (int) info->info.content->pos, - (int) info->info.content->endpos, - (int) info->info.content->bodypos)); - if (copy_block(fd, fdout, info->info.content->bodypos, - info->info.content->endpos - info->info.content->bodypos) == -1) { - g_warning("Cannot copy data to output fd"); - camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Cannot copy data to output file: %s"), - strerror (errno)); - goto error; - } - info->frompos = frompos; - offset = bodypos - info->info.content->bodypos; + xevnew = camel_local_summary_encode_x_evolution(cls, (CamelMessageInfo *)info); + if (camel_local_summary_write_headers(fdout, camel_mime_parser_headers_raw(mp), xevnew) == -1) { + d(printf("Error writing to tmp mailbox\n")); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Error writing to temp mailbox: %s"), + strerror(errno)); + goto error; } info->info.flags &= 0xffff; g_free(xevnew); xevnew = NULL; camel_mime_parser_drop_step(mp); - camel_mime_parser_drop_step(mp); - } else { - if (!quick) { - if (copy_block(fd, fdout, info->frompos, - info->info.content->endpos - info->frompos) == -1) { - g_warning("Cannot copy data to output fd"); + } + + camel_mime_parser_drop_step(mp); + if (info) { + d(printf("looking for message content to copy across from %d\n", (int)camel_mime_parser_tell(mp))); + while (camel_mime_parser_step(mp, &buffer, &len) == HSCAN_PRE_FROM) { + d(printf("copying mbox contents to tmp: '%.*s'\n", len, buffer)); + if (write(fdout, buffer, len) != len) { camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Cannot copy data to output file: %s"), - strerror(errno)); + _("Writing to tmp mailbox failed: %s: %s"), + cls->folder_path, strerror(errno)); goto error; } - /* update from pos here? */ - info->frompos += offset; - } else { - d(printf("Nothing to do for this message\n")); } - } - if (!quick && info != NULL && offset != 0) { - d(printf("offsetting content: %d\n", (int)offset)); - camel_folder_summary_offset_content(info->info.content, offset); - d(printf("pos = %d, endpos = %d, bodypos = %d\n", - (int) info->info.content->pos, - (int) info->info.content->endpos, - (int) info->info.content->bodypos)); + d(printf("we are now at %d, from = %d\n", (int)camel_mime_parser_tell(mp), + (int)camel_mime_parser_tell_start_from(mp))); + camel_mime_parser_unstep(mp); } } @@ -786,45 +592,31 @@ mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInf camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Could not close source folder %s: %s"), cls->folder_path, strerror(errno)); + fd = -1; goto error; } - if (!quick) { - if (close(fdout) == -1) { - g_warning("Cannot close tmp folder: %s", strerror(errno)); - camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Could not close temp folder: %s"), - strerror(errno)); - goto error; - } - - if (rename(tmpname, cls->folder_path) == -1) { - g_warning("Cannot rename folder: %s", strerror(errno)); - camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Could not rename folder: %s"), - strerror(errno)); - goto error; - } - tmpname = NULL; - - /* TODO: move up? */ - if (cls->index) - ibex_save(cls->index); + if (close(fdout) == -1) { + g_warning("Cannot close tmp folder: %s", strerror(errno)); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Could not close temp folder: %s"), + strerror(errno)); + fdout = -1; + goto error; } - if (stat(cls->folder_path, &st) == -1) { + /* this should probably either use unlink/link/unlink, or recopy over + the original mailbox, for various locking reasons/etc */ + if (rename(tmpname, cls->folder_path) == -1) { + g_warning("Cannot rename folder: %s", strerror(errno)); camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, - _("Unknown error: %s"), + _("Could not rename folder: %s"), strerror(errno)); goto error; } + tmpname = NULL; - camel_folder_summary_touch(s); - s->time = st.st_mtime; - mbs->folder_size = st.st_size; - camel_folder_summary_save(s); - - camel_object_unref(CAMEL_OBJECT(mp)); + camel_object_unref((CamelObject *)mp); return 0; error: @@ -839,12 +631,186 @@ mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInf if (tmpname) unlink(tmpname); if (mp) - camel_object_unref(CAMEL_OBJECT(mp)); + camel_object_unref((CamelObject *)mp); return -1; } +/* perform a quick sync - only system flags have changed */ +static int +mbox_summary_sync_quick(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex) +{ + CamelMboxSummary *mbs = (CamelMboxSummary *)cls; + CamelFolderSummary *s = (CamelFolderSummary *)mbs; + CamelMimeParser *mp = NULL; + int i, count; + CamelMboxMessageInfo *info; + int fd = -1; + char *xevnew; + const char *xev; + int len; + off_t lastpos; + + d(printf("Performing quick summary sync\n")); + + fd = open(cls->folder_path, O_RDWR); + if (fd == -1) { + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Could not open folder to summarise: %s: %s"), + cls->folder_path, strerror(errno)); + return -1; + } + + mp = camel_mime_parser_new(); + camel_mime_parser_scan_from(mp, TRUE); + camel_mime_parser_scan_pre_from(mp, TRUE); + camel_mime_parser_init_with_fd(mp, fd); + + count = camel_folder_summary_count(s); + for (i = 0; i < count; i++) { + int xevoffset; + + info = (CamelMboxMessageInfo *)camel_folder_summary_index(s, i); + + g_assert(info); + + d(printf("Checking message %s %08x\n", info->info.uid, info->info.flags)); + + if ((info->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED) == 0) + continue; + + d(printf("Updating message %s\n", info->info.uid)); + + camel_mime_parser_seek(mp, info->frompos, SEEK_SET); + + if (camel_mime_parser_step(mp, 0, 0) != HSCAN_FROM) { + g_warning("Expected a From line here, didn't get it"); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Summary and folder mismatch, even after a sync")); + goto error; + } + + if (camel_mime_parser_tell_start_from(mp) != info->frompos) { + g_warning("Didn't get the next message where I expected (%d) got %d instead", + (int)info->frompos, (int)camel_mime_parser_tell_start_from(mp)); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Summary and folder mismatch, even after a sync")); + goto error; + } + + if (camel_mime_parser_step(mp, 0, 0) == HSCAN_FROM_END) { + g_warning("camel_mime_parser_step failed (2)"); + goto error; + } + + xev = camel_mime_parser_header(mp, "X-Evolution", &xevoffset); + if (xev == NULL || camel_local_summary_decode_x_evolution(cls, xev, NULL) == -1) { + g_warning("We're supposed to have a valid x-ev header, but we dont"); + goto error; + } + xevnew = camel_local_summary_encode_x_evolution(cls, (CamelMessageInfo *)info); + /* the raw header contains a leading ' ', so count that too */ + if (strlen(xev)-1 != strlen(xevnew)) { + g_free(xevnew); + g_warning("Hmm, the xev headers shouldn't have changed size, but they did"); + goto error; + } + + lastpos = lseek(fd, 0, SEEK_CUR); + lseek(fd, xevoffset+strlen("X-Evolution: "), SEEK_SET); + do { + len = write(fd, xevnew, strlen(xevnew)); + } while (len == -1 && errno == EINTR); + lseek(fd, lastpos, SEEK_SET); + + camel_mime_parser_drop_step(mp); + camel_mime_parser_drop_step(mp); + + info->info.flags &= 0xffff; + } + + d(printf("Closing folders\n")); + + if (close(fd) == -1) { + g_warning("Cannot close source folder: %s", strerror(errno)); + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, + _("Could not close source folder %s: %s"), + cls->folder_path, strerror(errno)); + fd = -1; + goto error; + } + + camel_object_unref((CamelObject *)mp); + return 0; + error: + if (fd != -1) + close(fd); - - + if (mp) + camel_object_unref((CamelObject *)mp); + + return -1; +} + +static int +mbox_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo *changeinfo, CamelException *ex) +{ + struct stat st; + CamelMboxSummary *mbs = (CamelMboxSummary *)cls; + CamelFolderSummary *s = (CamelFolderSummary *)cls; + int i, count; + int quick = TRUE, work=FALSE; + int ret; + + /* first, sync ourselves up, just to make sure */ + summary_update(cls, mbs->folder_size, changeinfo, ex); + if (camel_exception_is_set(ex)) + return -1; + + count = camel_folder_summary_count(s); + if (count == 0) + return 0; + + /* check what work we have to do, if any */ + for (i=0;quick && iflags & CAMEL_MESSAGE_DELETED)) || + (info->flags & (CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_XEVCHANGE))) + quick = FALSE; + else + work |= (info->flags & CAMEL_MESSAGE_FOLDER_FLAGGED) != 0; + } + + /* yuck i hate this logic, but its to simplify the 'all ok, update summary' and failover cases */ + ret = -1; + if (quick) { + if (work) { + ret = mbox_summary_sync_quick(cls, expunge, changeinfo, ex); + if (ret == -1) { + g_warning("failed a quick-sync, trying a full sync"); + camel_exception_clear(ex); + } + } else { + ret = 0; + } + } + + if (ret == -1) + ret = mbox_summary_sync_full(cls, expunge, changeinfo, ex); + if (ret == -1) + return -1; + + if (stat(cls->folder_path, &st) == -1) { + camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Unknown error: %s"), strerror(errno)); + return -1; + } + + camel_folder_summary_touch(s); + s->time = st.st_mtime; + mbs->folder_size = st.st_size; + camel_folder_summary_save(s); + + return 0; +} diff --git a/camel/providers/local/camel-mh-folder.c b/camel/providers/local/camel-mh-folder.c index 8d6ce69c3c..c0a9f3f2af 100644 --- a/camel/providers/local/camel-mh-folder.c +++ b/camel/providers/local/camel-mh-folder.c @@ -40,7 +40,7 @@ #include "camel-mime-message.h" #include "camel-exception.h" -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ static CamelFolderClass *parent_class = NULL; diff --git a/camel/providers/local/camel-mh-summary.c b/camel/providers/local/camel-mh-summary.c index b6b31664b4..2f3c829e8d 100644 --- a/camel/providers/local/camel-mh-summary.c +++ b/camel/providers/local/camel-mh-summary.c @@ -33,7 +33,7 @@ #include -#define d(x) (printf("%s(%d): ", __FILE__, __LINE__),(x)) +#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/ #define CAMEL_MH_SUMMARY_VERSION (0x2000) @@ -279,10 +279,71 @@ mh_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changeinfo, Came return 0; } +static int +mh_summary_sync_message(CamelLocalSummary *cls, CamelMessageInfo *info, CamelException *ex) +{ + CamelMimeParser *mp; + const char *xev, *buffer; + int xevoffset; + int fd, outfd, len, outlen, ret=0; + char *name, *tmpname, *xevnew; + + name = g_strdup_printf("%s/%s", cls->folder_path, info->uid); + fd = open(name, O_RDWR); + if (fd == -1) + return -1; + mp = camel_mime_parser_new(); + camel_mime_parser_init_with_fd(mp, fd); + if (camel_mime_parser_step(mp, 0, 0) != HSCAN_EOF) { + xev = camel_mime_parser_header(mp, "X-Evolution", &xevoffset); + xevnew = camel_local_summary_encode_x_evolution(cls, info); + if (xev == NULL + || camel_local_summary_decode_x_evolution(cls, xev, NULL) == -1 + || strlen(xev)+1 != strlen(xevnew)) { + + /* need to write a new copy/unlink old */ + tmpname = g_strdup_printf("%s/.tmp.%d.%s", cls->folder_path, getpid(), info->uid); + outfd = open(tmpname, O_CREAT|O_WRONLY|O_TRUNC, 0600); + if (outfd != -1) { + outlen = 0; + if ( (len = camel_local_summary_write_headers(outfd, camel_mime_parser_headers_raw(mp), xevnew)) == 0) { + while (outlen != -1 && (len = camel_mime_parser_read(mp, &buffer, 10240)) > 0) { + do { + outlen = write(fd, buffer, len); + } while (outlen == -1 && errno == EINTR); + } + } + if (close(outfd) == -1 + || len == -1 + || outlen == -1 + || rename(tmpname, name) == -1) { + unlink(tmpname); + ret = -1; + } + } else { + g_warning("sync can't create tmp file: %s", strerror(errno)); + } + g_free(tmpname); + } else { + /* else, we can just update the flags field */ + lseek(fd, xevoffset+strlen("X-Evolution: "), SEEK_SET); + do { + len = write(fd, xevnew, strlen(xevnew)); + } while (len == -1 && errno == EINTR); + if (len == -1) + ret = -1; + } + + g_free(xevnew); + } -/* sync the summary with the ondisk files. - It doesnt store the state in the file, the summary only, == MUCH faster */ + camel_object_unref((CamelObject *)mp); + g_free(name); + return ret; +} + +/* sync the summary file with the ondisk files */ static int mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo *changes, CamelException *ex) { @@ -292,16 +353,16 @@ mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo d(printf("summary_sync(expunge=%s)\n", expunge?"true":"false")); - if (cls->index) { - ibex_save(cls->index); - } - if (!expunge) - return 0; + /* we could probably get away without this ... but why not use it, esp if we're going to + be doing any significant io already */ + if (camel_local_summary_check(cls, changes, ex) == -1) + return -1; count = camel_folder_summary_count((CamelFolderSummary *)cls); for (i=count-1;i>=0;i--) { info = camel_folder_summary_index((CamelFolderSummary *)cls, i); - if (info && info->flags & CAMEL_MESSAGE_DELETED) { + g_assert(info); + if (expunge && (info->flags & CAMEL_MESSAGE_DELETED)) { name = g_strdup_printf("%s/%s", cls->folder_path, info->uid); d(printf("deleting %s\n", name)); if (unlink(name) == 0 || errno==ENOENT) { @@ -309,12 +370,19 @@ mh_summary_sync(CamelLocalSummary *cls, gboolean expunge, CamelFolderChangeInfo /* FIXME: put this in folder_summary::remove()? */ if (cls->index) ibex_unindex(cls->index, info->uid); - + camel_folder_change_info_remove_uid(changes, info->uid); camel_folder_summary_remove((CamelFolderSummary *)cls, info); } + g_free(name); + } else if (info->flags & (CAMEL_MESSAGE_FOLDER_NOXEV|CAMEL_MESSAGE_FOLDER_FLAGGED)) { + if (mh_summary_sync_message(cls, info, ex) != -1) { + info->flags &= 0xffff; + } else { + g_warning("Problem occured when trying to expunge, ignored"); + } } } + return 0; } - diff --git a/camel/providers/nntp/camel-nntp-utils.c b/camel/providers/nntp/camel-nntp-utils.c index e8ee9bb112..90c09a494e 100644 --- a/camel/providers/nntp/camel-nntp-utils.c +++ b/camel/providers/nntp/camel-nntp-utils.c @@ -30,6 +30,8 @@ #include "camel-stream-mem.h" #include "camel-exception.h" +#include "e-util/md5-utils.h" + #include #include @@ -39,6 +41,7 @@ get_XOVER_headers(CamelNNTPStore *nntp_store, CamelFolder *folder, { int status; CamelNNTPFolder *nntp_folder = CAMEL_NNTP_FOLDER (folder); + char digest[16]; status = camel_nntp_command (nntp_store, ex, NULL, "XOVER %d-%d", @@ -61,7 +64,7 @@ get_XOVER_headers(CamelNNTPStore *nntp_store, CamelFolder *folder, g_print ("done\n"); } else { - CamelMessageInfo *new_info = g_new0(CamelMessageInfo, 1); + CamelMessageInfo *new_info = camel_folder_summary_info_new(nntp_folder->summary); char **split_line = g_strsplit (line, "\t", 7); char *subject, *from, *date, *message_id, *bytes; @@ -95,7 +98,8 @@ get_XOVER_headers(CamelNNTPStore *nntp_store, CamelFolder *folder, #endif new_info->size = atoi(bytes); new_info->uid = g_strdup_printf ("%s,%s", split_line[0], message_id); - new_info->message_id = g_strdup(message_id); + md5_get_digest(message_id, strlen(message_id), digest); + memcpy(new_info->message_id.id.hash, digest, sizeof(new_info->message_id.id.hash)); if (camel_nntp_newsrc_article_is_read (nntp_store->newsrc, folder->name, -- cgit v1.2.3