[dovecot-cvs] dovecot/src/lib-index/mbox mbox-append.c,1.39,1.40 mbox-index.c,1.65,1.66 mbox-index.h,1.24,1.25 mbox-rebuild.c,1.22,1.23 mbox-rewrite.c,1.52,1.53 mbox-sync-full.c,1.11,1.12 mbox-sync.c,1.28,1.29
cras at procontrol.fi
cras at procontrol.fi
Thu Mar 6 21:23:47 EET 2003
Update of /home/cvs/dovecot/src/lib-index/mbox
In directory danu:/tmp/cvs-serv13584/mbox
Modified Files:
mbox-append.c mbox-index.c mbox-index.h mbox-rebuild.c
mbox-rewrite.c mbox-sync-full.c mbox-sync.c
Log Message:
UIDs are now saved into mbox file. added a few rewriting optimizations so
that we don't always have to rewrite the whole file when updating messages
at the beginning of file.
Index: mbox-append.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-append.c,v
retrieving revision 1.39
retrieving revision 1.40
diff -u -d -r1.39 -r1.40
--- mbox-append.c 5 Mar 2003 01:41:37 -0000 1.39
+++ mbox-append.c 6 Mar 2003 19:23:45 -0000 1.40
@@ -19,7 +19,7 @@
const unsigned char *data;
unsigned char md5_digest[16];
size_t size, pos;
- int failed;
+ int ret;
/* get the From-line */
pos = 0;
@@ -41,7 +41,7 @@
"From-line not found where expected",
index->mailbox_path);
index->set_flags |= MAIL_INDEX_FLAG_FSCK;
- return FALSE;
+ return -1;
}
/* parse the From-line */
@@ -60,7 +60,7 @@
/* add message to index */
rec = index->append_begin(index);
if (rec == NULL)
- return FALSE;
+ return -1;
update = index->update_begin(index, rec);
@@ -89,31 +89,75 @@
i_stream_seek(input, input->v_limit);
i_stream_set_read_limit(input, 0);
- /* save MD5 */
- md5_final(&ctx.md5, md5_digest);
- index->update_field_raw(update, DATA_FIELD_MD5,
- md5_digest, sizeof(md5_digest));
+ ret = 1;
+ if (index->header->messages_count == 0 &&
+ ctx.uid_validity != index->header->messages_count) {
+ /* UID validity is different */
+ if (ctx.uid_validity == 0) {
+ /* we have to write it to mbox */
+ if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) {
+ /* try again */
+ ret = 0;
+ } else {
+ index->header->flags |=
+ MAIL_INDEX_FLAG_DIRTY_MESSAGES;
+ rec->index_flags |= INDEX_MAIL_FLAG_DIRTY;
+ }
+ } else {
+ /* change it in index */
+ index->header->uid_validity = ctx.uid_validity;
+ index->header->next_uid = 1;
+ index->header->last_nonrecent_uid = 0;
+ index->inconsistent = TRUE;
+ }
+ }
- if (!index->update_end(update)) {
+ if (ctx.uid >= index->header->next_uid) {
+ /* X-UID header looks ok */
+ if (ret != 0)
+ index->header->next_uid = ctx.uid;
+ } else if (!index->mailbox_readonly) {
+ /* Write X-UID for it */
+ if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) {
+ /* try again */
+ ret = 0;
+ } else {
+ index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES;
+ rec->index_flags |= INDEX_MAIL_FLAG_DIRTY;
+ }
+ } else {
+ /* save MD5 */
+ md5_final(&ctx.md5, md5_digest);
+ index->update_field_raw(update, DATA_FIELD_MD5,
+ md5_digest, sizeof(md5_digest));
+ }
+
+ if (ret <= 0) {
+ index->update_abort(update);
index->append_abort(index, rec);
- failed = TRUE;
} else {
- /* save message flags */
- rec->msg_flags = ctx.flags;
- mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags);
- failed = FALSE;
+ if (!index->update_end(update)) {
+ index->append_abort(index, rec);
+ ret = -1;
+ } else {
+ /* save message flags */
+ rec->msg_flags = ctx.flags;
+ mail_index_mark_flag_changes(index, rec, 0,
+ rec->msg_flags);
+ ret = 1;
- if (!index->append_end(index, rec))
- failed = TRUE;
+ if (!index->append_end(index, rec))
+ ret = -1;
+ }
}
mbox_header_free_context(&ctx);
-
- return !failed;
+ return ret;
}
int mbox_index_append(struct mail_index *index, struct istream *input)
{
+ uoff_t offset;
int ret;
if (input->v_offset == input->v_size) {
@@ -124,7 +168,8 @@
if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
- for (;;) {
+ do {
+ offset = input->v_offset;
if (input->start_offset + input->v_offset != 0) {
/* we're at the [\r]\n before the From-line,
skip it */
@@ -139,16 +184,26 @@
}
}
- if (input->v_offset == input->v_size)
+ if (input->v_offset == input->v_size) {
+ ret = 1;
break;
+ }
t_push();
ret = mbox_index_append_next(index, input);
t_pop();
- if (!ret)
- return FALSE;
+ if (ret == 0) {
+ /* we want to rescan this message with exclusive
+ locking */
+ i_stream_seek(input, offset);
+ }
+ } while (ret > 0);
+
+ if (index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE) {
+ /* Write missing X-IMAPbase and new/changed X-UID headers */
+ return mbox_index_rewrite(index);
}
- return TRUE;
+ return ret >= 0;
}
Index: mbox-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-index.c,v
retrieving revision 1.65
retrieving revision 1.66
diff -u -d -r1.65 -r1.66
--- mbox-index.c 5 Mar 2003 01:41:37 -0000 1.65
+++ mbox-index.c 6 Mar 2003 19:23:45 -0000 1.66
@@ -9,6 +9,7 @@
#include "mail-index-data.h"
#include "mail-custom-flags.h"
+#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
@@ -191,31 +192,32 @@
return flags;
}
-static int mbox_parse_imapbase(const unsigned char *value, size_t len,
- struct mbox_header_context *ctx)
+static void mbox_parse_imapbase(const unsigned char *value, size_t len,
+ struct mbox_header_context *ctx)
{
- const char **flag;
+ const char **flag, *str;
+ char *end;
buffer_t *buf;
size_t pos, start;
enum mail_flags flags;
unsigned int count;
- int ret, spaces;
+ int ret;
- /* skip <uid validity> and <last uid> fields */
- spaces = 0;
- for (pos = 0; pos < len; pos++) {
- if (value[pos] == ' ' && (pos == 0 || value[pos-1] != ' ')) {
- if (++spaces == 2)
- break;
- }
- }
+ t_push();
- while (pos < len && value[pos] == ' ') pos++;
+ /* <uid validity> <last uid> */
+ str = t_strndup(value, len);
+ ctx->uid_validity = strtoul(str, &end, 10);
+ ctx->uid_last = strtoul(end, &end, 10);
+ pos = end - str;
- if (pos == len)
- return TRUE;
+ while (pos < len && value[pos] == ' ')
+ pos++;
- t_push();
+ if (pos == len) {
+ t_pop();
+ return;
+ }
/* we're at the 3rd field now, which begins the list of custom flags */
buf = buffer_create_dynamic(data_stack_pool,
@@ -243,8 +245,6 @@
flag, count);
t_pop();
-
- return ret > 0;
}
void mbox_header_cb(struct message_part *part __attr_unused__,
@@ -348,6 +348,14 @@
ID's but don't blindly trust this header alone as
it could just as easily come from the remote. */
fixed = memcasecmp(name, "X-Delivery-ID:", 13) == 0;
+ } else if (name_len == 5 &&
+ memcasecmp(name, "X-UID", 5) == 0) {
+ ctx->uid = 0;
+ for (i = 0; i < value_len; i++) {
+ if (value[i] < '0' || value[i] > '9')
+ break;
+ ctx->uid = ctx->uid * 10 + (value[i]-'0');
+ }
} else if (name_len == 8 &&
memcasecmp(name, "X-Status", 8) == 0) {
/* update message flags */
@@ -359,8 +367,7 @@
ctx->custom_flags);
} else if (name_len == 10 &&
memcasecmp(name, "X-IMAPbase", 10) == 0) {
- /* update list of custom message flags */
- (void)mbox_parse_imapbase(value, value_len, ctx);
+ mbox_parse_imapbase(value, value_len, ctx);
}
break;
}
@@ -799,6 +806,18 @@
return TRUE;
}
+static int mbox_index_append_end(struct mail_index *index,
+ struct mail_index_record *rec)
+{
+ if (!mail_index_append_end(index, rec))
+ return FALSE;
+
+ /* update last_uid in X-IMAPbase */
+ index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES |
+ MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS;
+ return TRUE;
+}
+
struct mail_index mbox_index = {
mail_index_open,
mbox_index_free,
@@ -820,10 +839,11 @@
mbox_index_expunge,
mbox_index_update_flags,
mail_index_append_begin,
- mail_index_append_end,
+ mbox_index_append_end,
mail_index_append_abort,
mail_index_update_begin,
mail_index_update_end,
+ mail_index_update_abort,
mail_index_update_field,
mail_index_update_field_raw,
mail_index_get_last_error,
Index: mbox-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-index.h,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- mbox-index.h 11 Jan 2003 19:55:56 -0000 1.24
+++ mbox-index.h 6 Mar 2003 19:23:45 -0000 1.25
@@ -11,6 +11,8 @@
struct md5_context md5;
int received;
+ unsigned int uid_validity, uid_last, uid;
+
struct istream *input;
uoff_t content_length;
int set_read_limit;
Index: mbox-rebuild.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-rebuild.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- mbox-rebuild.c 5 Jan 2003 13:09:52 -0000 1.22
+++ mbox-rebuild.c 6 Mar 2003 19:23:45 -0000 1.23
@@ -1,22 +1,17 @@
/* Copyright (C) 2002 Timo Sirainen */
#include "lib.h"
-#include "istream.h"
#include "mbox-index.h"
-#include "mbox-lock.h"
#include "mail-index-data.h"
#include "mail-index-util.h"
#include <unistd.h>
-#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
int mbox_index_rebuild(struct mail_index *index)
{
- struct istream *input;
struct stat st;
- int failed;
i_assert(index->lock_type != MAIL_LOCK_SHARED);
@@ -34,7 +29,8 @@
/* update indexid, which also means that our state has completely
changed */
index->indexid = index->header->indexid;
- index->inconsistent = TRUE;
+ if (index->opened)
+ index->inconsistent = TRUE;
if (msync(index->mmap_base,
sizeof(struct mail_index_header), MS_SYNC) < 0)
@@ -44,16 +40,7 @@
if (!mail_index_data_reset(index->data))
return FALSE;
- input = mbox_get_stream(index, 0, MAIL_LOCK_SHARED);
- if (input == NULL)
- return FALSE;
-
- mbox_skip_empty_lines(input);
- failed = !mbox_index_append(index, input);
-
- i_stream_unref(input);
-
- if (failed)
+ if (!mbox_sync_full(index))
return FALSE;
/* update sync stamp */
@@ -64,5 +51,6 @@
/* rebuild is complete - remove the flag */
index->header->flags &= ~(MAIL_INDEX_FLAG_REBUILD|MAIL_INDEX_FLAG_FSCK);
+ index->set_flags &= ~MAIL_INDEX_FLAG_REBUILD;
return TRUE;
}
Index: mbox-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-rewrite.c,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- mbox-rewrite.c 19 Feb 2003 23:35:21 -0000 1.52
+++ mbox-rewrite.c 6 Mar 2003 19:23:45 -0000 1.53
@@ -4,6 +4,7 @@
#include "ioloop.h"
#include "istream.h"
#include "ostream.h"
+#include "file-set-size.h"
#include "str.h"
#include "write-full.h"
#include "mbox-index.h"
@@ -15,13 +16,14 @@
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
+#include <sys/stat.h>
struct mbox_rewrite_context {
struct ostream *output;
int failed;
uoff_t content_length;
- unsigned int seq;
+ unsigned int seq, uid;
unsigned int msg_flags;
const char **custom_flags;
@@ -30,6 +32,7 @@
unsigned int ximapbase_found:1;
unsigned int xkeywords_found:1;
+ unsigned int xuid_found:1;
unsigned int status_found:1;
unsigned int xstatus_found:1;
unsigned int content_length_found:1;
@@ -105,6 +108,18 @@
return TRUE;
}
+static int mbox_write_xuid(struct mbox_rewrite_context *ctx)
+{
+ const char *str;
+
+ str = t_strdup_printf("X-UID: %u\n", ctx->uid);
+
+ if (o_stream_send_str(ctx->output, str) < 0)
+ return FALSE;
+
+ return TRUE;
+}
+
static int mbox_write_xkeywords(struct mbox_rewrite_context *ctx,
const char *x_keywords)
{
@@ -250,7 +265,6 @@
{
struct mbox_rewrite_context *ctx = context;
const char *str;
- char *end;
if (ctx->failed)
return;
@@ -269,20 +283,12 @@
(void)mbox_write_xkeywords(ctx, str);
} else if (name_len == 10 && memcasecmp(name, "X-IMAPbase", 10) == 0) {
if (ctx->seq == 1) {
- /* temporarily copy the value to make sure we
- don't overflow it */
- const char *str;
-
- t_push();
- str = t_strndup(value, value_len);
- ctx->uid_validity = strtoul(str, &end, 10);
- while (*end == ' ') end++;
- ctx->uid_last = strtoul(end, &end, 10);
- t_pop();
-
ctx->ximapbase_found = TRUE;
(void)mbox_write_ximapbase(ctx);
}
+ } else if (name_len == 5 && memcasecmp(name, "X-UID", 5) == 0) {
+ ctx->xuid_found = TRUE;
+ (void)mbox_write_xuid(ctx);
} else if (name_len == 14 &&
memcasecmp(name, "Content-Length", 14) == 0) {
ctx->content_length_found = TRUE;
@@ -331,10 +337,12 @@
/* parse the header, write the fields we don't want to change */
memset(&ctx, 0, sizeof(ctx));
ctx.output = output;
- ctx.seq = seq;
ctx.content_length = body_size;
+ ctx.seq = seq;
+ ctx.uid = rec->uid;
ctx.msg_flags = rec->msg_flags;
- ctx.uid_validity = index->header->uid_validity-1;
+ ctx.uid_validity = index->header->uid_validity;
+ ctx.uid_last = index->header->next_uid-1;
ctx.custom_flags = mail_custom_flags_list_get(index->custom_flags);
i_stream_set_read_limit(input, input->v_offset + hdr_size);
@@ -349,12 +357,14 @@
(void)mbox_write_ximapbase(&ctx);
}
- if (!ctx.xkeywords_found)
- (void)mbox_write_xkeywords(&ctx, NULL);
if (!ctx.status_found)
(void)mbox_write_status(&ctx, NULL);
if (!ctx.xstatus_found)
(void)mbox_write_xstatus(&ctx, NULL);
+ if (!ctx.xkeywords_found)
+ (void)mbox_write_xkeywords(&ctx, NULL);
+ if (!ctx.xuid_found)
+ (void)mbox_write_xuid(&ctx);
if (!ctx.content_length_found)
(void)mbox_write_content_length(&ctx);
@@ -366,31 +376,50 @@
return TRUE;
}
-static int fd_copy(int in_fd, int out_fd, uoff_t out_offset)
+static int fd_copy(struct mail_index *index, int in_fd, int out_fd,
+ uoff_t out_offset, uoff_t size)
{
struct istream *input;
struct ostream *output;
+ struct stat st;
int ret;
i_assert(out_offset <= OFF_T_MAX);
- if (lseek(out_fd, (off_t)out_offset, SEEK_SET) < 0)
+ /* first grow the file to wanted size, to make sure we don't run out
+ of disk space */
+ if (fstat(out_fd, &st) < 0) {
+ mbox_set_syscall_error(index, "fstat()");
return -1;
+ }
+
+ if ((uoff_t)st.st_size < out_offset + size) {
+ if (file_set_size(out_fd, (off_t)(out_offset + size)) < 0) {
+ mbox_set_syscall_error(index, "file_set_size()");
+ (void)ftruncate(out_fd, st.st_size);
+ return -1;
+ }
+ }
+
+ if (lseek(out_fd, (off_t)out_offset, SEEK_SET) < 0) {
+ mbox_set_syscall_error(index, "lseek()");
+ (void)ftruncate(out_fd, st.st_size);
+ return -1;
+ }
t_push();
input = i_stream_create_mmap(in_fd, data_stack_pool,
1024*256, 0, 0, FALSE);
+ i_stream_set_read_limit(input, size);
+
output = o_stream_create_file(out_fd, data_stack_pool, 1024, 0, FALSE);
o_stream_set_blocking(output, 60000, NULL, NULL);
ret = o_stream_send_istream(output, input);
- if (ret < 0)
+ if (ret < 0) {
errno = output->stream_errno;
- else {
- /* we may have shrinked the file */
- i_assert(out_offset + input->v_size <= OFF_T_MAX);
- ret = ftruncate(out_fd, (off_t) (out_offset + input->v_size));
+ mbox_set_syscall_error(index, "o_stream_send_istream()");
}
o_stream_unref(output);
@@ -400,6 +429,47 @@
return ret;
}
+static int dirty_flush(struct mail_index *index, uoff_t dirty_offset,
+ struct ostream *output, int output_fd)
+{
+ if (output->offset == 0)
+ return TRUE;
+
+ if (o_stream_flush(output) < 0) {
+ mbox_set_syscall_error(index, "o_stream_flush()");
+ return FALSE;
+ }
+
+ /* POSSIBLE DATA LOSS HERE. We're writing to the mbox file,
+ so if we get killed here before finished, we'll lose some
+ bytes. I can't really think of any way to fix this,
+ rename() is problematic too especially because of file
+ locking issues (new mail could be lost).
+
+ Usually we're moving the data by just a few bytes, so
+ the data loss should never be more than those few bytes..
+ If we moved more, we could have written the file from end
+ to beginning in blocks (it'd be a bit slow to do it in
+ blocks of ~1-10 bytes which is the usual case, so we don't
+ bother).
+
+ Also, we might as well be shrinking the file, in which
+ case we can't lose data. */
+ if (fd_copy(index, output_fd, index->mbox_fd,
+ dirty_offset, output->offset) < 0)
+ return FALSE;
+
+ /* All ok. Just make sure the timestamps of index and
+ mbox differ, so index will be updated at next sync */
+ index->file_sync_stamp = ioloop_time-61;
+
+ if (o_stream_seek(output, 0) < 0) {
+ mbox_set_syscall_error(index, "o_stream_seek()");
+ return FALSE;
+ }
+ return TRUE;
+}
+
#define INDEX_DIRTY_FLAGS \
(MAIL_INDEX_FLAG_DIRTY_MESSAGES | MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS)
@@ -415,18 +485,25 @@
uoff_t offset, hdr_size, body_size, dirty_offset;
const char *path;
unsigned int seq;
- int tmp_fd, failed, dirty_found, rewrite;
+ int tmp_fd, failed, dirty_found, rewrite, no_locking;
- i_assert(index->lock_type == MAIL_LOCK_UNLOCK);
+ i_assert(index->lock_type == MAIL_LOCK_UNLOCK ||
+ (index->lock_type == MAIL_LOCK_EXCLUSIVE &&
+ index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE));
- if (!index->set_lock(index, MAIL_LOCK_SHARED))
- return FALSE;
+ no_locking = index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE;
+ if (!no_locking) {
+ if (!index->set_lock(index, MAIL_LOCK_SHARED))
+ return FALSE;
+ }
rewrite = (index->header->flags & INDEX_DIRTY_FLAGS) &&
index->header->messages_count > 0;
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- return FALSE;
+ if (!no_locking) {
+ if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
+ return FALSE;
+ }
if (!rewrite) {
/* no need to rewrite */
@@ -436,11 +513,14 @@
tmp_fd = -1; input = NULL;
failed = TRUE; rewrite = FALSE;
do {
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- break;
+ if (!no_locking) {
+ if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
+ break;
- if (!index->sync_and_lock(index, MAIL_LOCK_EXCLUSIVE, NULL))
- break;
+ if (!index->sync_and_lock(index, MAIL_LOCK_EXCLUSIVE,
+ NULL))
+ break;
+ }
input = mbox_get_stream(index, 0, MAIL_LOCK_EXCLUSIVE);
if (input == NULL)
@@ -461,8 +541,10 @@
} while (0);
if (!rewrite) {
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- failed = TRUE;
+ if (!no_locking) {
+ if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
+ failed = TRUE;
+ }
if (input != NULL)
i_stream_unref(input);
return !failed;
@@ -532,11 +614,22 @@
break;
}
- /* write body */
- offset += body_size;
- if (!mbox_write(index, input, output, offset)) {
- failed = TRUE;
- break;
+ if (dirty_found &&
+ offset - dirty_offset == output->offset) {
+ /* no need to write more, flush */
+ if (!dirty_flush(index, dirty_offset,
+ output, tmp_fd)) {
+ failed = TRUE;
+ break;
+ }
+ dirty_found = FALSE;
+ } else {
+ /* write body */
+ offset += body_size;
+ if (!mbox_write(index, input, output, offset)) {
+ failed = TRUE;
+ break;
+ }
}
}
@@ -544,14 +637,8 @@
rec = index->next(index, rec);
}
- if (!dirty_found) {
- index_set_error(index, "Expected dirty messages not found "
- "from mbox file %s", index->mailbox_path);
- failed = TRUE;
- }
-
- if (!failed) {
- /* always end with a \n */
+ if (!failed && dirty_found) {
+ /* end with \n */
(void)o_stream_send(output, "\n", 1);
}
@@ -561,38 +648,32 @@
failed = TRUE;
}
- i_stream_unref(input);
- o_stream_unref(output);
-
- if (!failed) {
- /* POSSIBLE DATA LOSS HERE. We're writing to the mbox file,
- so if we get killed here before finished, we'll lose some
- bytes. I can't really think of any way to fix this,
- rename() is problematic too especially because of file
- locking issues (new mail could be lost).
-
- Usually we're moving the data by just a few bytes, so
- the data loss should never be more than those few bytes..
- If we moved more, we could have written the file from end
- to beginning in blocks (it'd be a bit slow to do it in
- blocks of ~1-10 bytes which is the usual case, so we don't
- bother).
+ if (!failed && dirty_found) {
+ uoff_t dirty_size = output->offset;
- Also, we might as well be shrinking the file, in which
- case we can't lose data. */
- if (fd_copy(tmp_fd, index->mbox_fd, dirty_offset) == 0) {
- /* All ok. Just make sure the timestamps of index and
- mbox differ, so index will be updated at next sync */
- index->file_sync_stamp = ioloop_time-61;
- reset_dirty_flags(index);
- } else {
- mbox_set_syscall_error(index, "fd_copy()");
+ if (!dirty_flush(index, dirty_offset, output, tmp_fd))
failed = TRUE;
+ else {
+ /* we may have shrinked the file */
+ i_assert(dirty_offset + dirty_size <= OFF_T_MAX);
+ if (ftruncate(index->mbox_fd,
+ (off_t)(dirty_offset + dirty_size)) < 0) {
+ mbox_set_syscall_error(index, "ftruncate()");
+ failed = TRUE;
+ }
}
}
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- failed = TRUE;
+ if (!failed)
+ reset_dirty_flags(index);
+
+ i_stream_unref(input);
+ o_stream_unref(output);
+
+ if (!no_locking) {
+ if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
+ failed = TRUE;
+ }
(void)unlink(path);
Index: mbox-sync-full.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-sync-full.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mbox-sync-full.c 11 Jan 2003 23:13:36 -0000 1.11
+++ mbox-sync-full.c 6 Mar 2003 19:23:45 -0000 1.12
@@ -11,6 +11,7 @@
#include <unistd.h>
#include <fcntl.h>
+#include <sys/stat.h>
static void skip_line(struct istream *input)
{
@@ -29,17 +30,20 @@
}
}
-static int verify_header_md5sum(struct mail_index *index,
- struct mail_index_record *rec,
- unsigned char current_digest[16])
+static int verify_header(struct mail_index *index,
+ struct mail_index_record *rec,
+ unsigned int uid, unsigned char current_digest[16])
{
const unsigned char *old_digest;
size_t size;
/* MD5 sums must match */
old_digest = index->lookup_field_raw(index, rec, DATA_FIELD_MD5, &size);
- return old_digest != NULL && size >= 16 &&
- memcmp(old_digest, current_digest, 16) == 0;
+ if (old_digest == NULL)
+ return uid == rec->uid;
+
+ return size >= 16 && memcmp(old_digest, current_digest, 16) == 0 &&
+ (uid == 0 || uid == rec->uid);
}
static int mail_update_header_size(struct mail_index *index,
@@ -95,6 +99,26 @@
return TRUE;
}
+static int mbox_check_uidvalidity(struct mail_index *index,
+ unsigned int uid_validity)
+{
+ if (uid_validity == index->header->uid_validity)
+ return TRUE;
+
+ index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES |
+ MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS;
+
+ if (uid_validity == 0) {
+ /* X-IMAPbase header isn't written yet */
+ } else {
+ /* UID validity has changed - rebuild whole index */
+ index->set_flags |= MAIL_INDEX_FLAG_REBUILD;
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
static int match_next_record(struct mail_index *index,
struct mail_index_record *rec,
unsigned int seq, struct istream *input,
@@ -139,13 +163,27 @@
mbox_header_cb, &ctx);
md5_final(&ctx.md5, current_digest);
+ if (seq == 1) {
+ if (!mbox_check_uidvalidity(index,
+ ctx.uid_validity)) {
+ /* uidvalidity changed, abort */
+ break;
+ }
+
+ if (ctx.uid_last >= index->header->next_uid) {
+ /* last_uid larger than ours */
+ index->header->next_uid =
+ ctx.uid_last+1;
+ }
+ }
+
mbox_header_free_context(&ctx);
i_stream_set_read_limit(input, 0);
body_offset = input->v_offset;
}
- if (verify_header_md5sum(index, rec, current_digest) &&
+ if (verify_header(index, rec, ctx.uid, current_digest) &&
mbox_verify_end_of_body(input, body_offset + body_size)) {
/* valid message */
update = index->update_begin(index, rec);
@@ -211,7 +249,7 @@
/* first make sure we start with a "From " line. If file is too
small, we'll just treat it as empty mbox file. */
if (i_stream_read_data(input, &data, &size, 5) > 0 &&
- strncmp((const char *) data, "From ", 5) != 0) {
+ memcmp(data, "From ", 5) != 0) {
index_set_error(index, "File isn't in mbox format: %s",
index->mailbox_path);
return FALSE;
@@ -269,11 +307,12 @@
}
if (!dirty && (index->header->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES)) {
- /* no flags were dirty anymore, no need to rewrite */
+ /* no flags are dirty anymore, no need to rewrite */
index->header->flags &= ~MAIL_INDEX_FLAG_DIRTY_MESSAGES;
}
- if (input->v_offset == input->v_size)
+ if (input->v_offset == input->v_size ||
+ (index->set_flags & MAIL_INDEX_FLAG_REBUILD))
return TRUE;
else
return mbox_index_append(index, input);
@@ -282,6 +321,8 @@
int mbox_sync_full(struct mail_index *index)
{
struct istream *input;
+ struct stat orig_st, st;
+ uoff_t continue_offset;
int failed;
i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
@@ -290,8 +331,43 @@
if (input == NULL)
return FALSE;
- failed = !mbox_sync_from_stream(index, input);
- i_stream_unref(input);
+ if (fstat(index->mbox_fd, &orig_st) < 0) {
+ mbox_set_syscall_error(index, "fstat()");
+ continue_offset = (uoff_t)-1;
+ failed = TRUE;
+ } else {
+ failed = !mbox_sync_from_stream(index, input);
+ continue_offset = failed || input->v_offset == input->v_size ||
+ (index->set_flags & MAIL_INDEX_FLAG_REBUILD) ?
+ (uoff_t)-1 : input->v_offset;
+ i_stream_unref(input);
+ }
+
+ if (continue_offset != (uoff_t)-1) {
+ /* mbox_index_append() stopped, which means that it wants
+ write access to mbox. if mbox hasn't changed after
+ unlock+lock, we should be able to safely continue where we
+ were left off last time. otherwise do full resync. */
+ if (!mbox_unlock(index))
+ return FALSE;
+
+ input = mbox_get_stream(index, 0, MAIL_LOCK_EXCLUSIVE);
+ if (input == NULL)
+ return FALSE;
+
+ if (fstat(index->mbox_fd, &st) < 0) {
+ mbox_set_syscall_error(index, "fstat()");
+ failed = TRUE;
+ } else if (st.st_mtime == orig_st.st_mtime &&
+ st.st_size == orig_st.st_size) {
+ i_stream_seek(input, continue_offset);
+ failed = !mbox_index_append(index, input);
+ } else {
+ failed = !mbox_sync_from_stream(index, input);
+ }
+
+ i_stream_unref(input);
+ }
return !failed;
}
Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-sync.c,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- mbox-sync.c 5 Jan 2003 13:09:52 -0000 1.28
+++ mbox-sync.c 6 Mar 2003 19:23:45 -0000 1.29
@@ -134,6 +134,12 @@
if (!mbox_lock_and_sync_full(index, data_lock_type))
return FALSE;
+ if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) != 0) {
+ /* uidvalidity probably changed, rebuild */
+ if (!index->rebuild(index))
+ return FALSE;
+ }
+
index->mbox_size = filesize;
}
More information about the dovecot-cvs
mailing list