[dovecot-cvs] dovecot/src/lib-storage/index/mbox mbox-mail.c, 1.2,
1.3 mbox-storage.c, 1.76, 1.77 mbox-sync-parse.c, 1.9,
1.10 mbox-sync-private.h, 1.8, 1.9 mbox-sync-rewrite.c, 1.9,
1.10 mbox-sync-update.c, 1.7, 1.8 mbox-sync.c, 1.16, 1.17
cras at procontrol.fi
cras at procontrol.fi
Mon Jun 14 07:30:35 EEST 2004
Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv21799/mbox
Modified Files:
mbox-mail.c mbox-storage.c mbox-sync-parse.c
mbox-sync-private.h mbox-sync-rewrite.c mbox-sync-update.c
mbox-sync.c
Log Message:
Save mbox offsets to index file using extra_records. Some other fixes.
Index: mbox-mail.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-mail.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mbox-mail.c 6 May 2004 03:05:42 -0000 1.2
+++ mbox-mail.c 14 Jun 2004 04:30:32 -0000 1.3
@@ -13,21 +13,28 @@
static int mbox_mail_seek(struct index_mail *mail)
{
- i_assert(mail->mail.seq <= mail->ibox->mbox_data_count);
+ struct index_mailbox *ibox = mail->ibox;
+ const void *data;
+
+ if (mail_index_lookup_extra(ibox->view, mail->mail.seq,
+ ibox->mbox_extra_idx, &data) < 0) {
+ mail_storage_set_index_error(ibox);
+ return -1;
+ }
// FIXME: lock the file. sync if needed.
- if (mbox_file_open_stream(mail->ibox) < 0)
+ if (mbox_file_open_stream(ibox) < 0)
return -1;
- istream_raw_mbox_seek(mail->ibox->mbox_stream,
- mail->ibox->mbox_data[mail->mail.seq-1] >> 1);
+ istream_raw_mbox_seek(ibox->mbox_stream, *((const uint64_t *)data));
return 0;
}
static const struct mail_full_flags *mbox_mail_get_flags(struct mail *_mail)
{
- struct index_mail *mail = (struct index_mail *)_mail;
+ return index_mail_get_flags(_mail);
+ /*FIXME:struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
i_assert(_mail->seq <= mail->ibox->mbox_data_count);
@@ -36,7 +43,7 @@
if ((mail->ibox->mbox_data[_mail->seq-1] & 1) != 0)
data->flags.flags |= MAIL_RECENT;
- return &data->flags;
+ return &data->flags;*/
}
static time_t mbox_mail_get_received_date(struct mail *_mail)
Index: mbox-storage.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-storage.c,v
retrieving revision 1.76
retrieving revision 1.77
diff -u -d -r1.76 -r1.77
--- mbox-storage.c 9 May 2004 23:32:44 -0000 1.76
+++ mbox-storage.c 14 Jun 2004 04:30:32 -0000 1.77
@@ -397,6 +397,7 @@
struct index_mailbox *ibox;
struct mail_index *index;
const char *path, *index_dir;
+ uint32_t mbox_extra_idx;
if (strcasecmp(name, "INBOX") == 0) {
/* name = "INBOX"
@@ -413,6 +414,8 @@
}
index = index_storage_alloc(index_dir, path, MBOX_INDEX_PREFIX);
+ mbox_extra_idx = mail_index_register_record_extra(index, "mbox",
+ sizeof(uint64_t));
ibox = index_storage_mailbox_init(storage, &mbox_mailbox,
index, name, flags);
if (ibox == NULL)
@@ -421,6 +424,7 @@
ibox->path = i_strdup(path);
ibox->mbox_fd = -1;
ibox->mbox_lock_type = F_UNLCK;
+ ibox->mbox_extra_idx = mbox_extra_idx;
ibox->get_recent_count = mbox_get_recent_count;
ibox->mail_interface = &mbox_mail;
@@ -775,10 +779,6 @@
static int mbox_storage_close(struct mailbox *box)
{
- struct index_mailbox *ibox = (struct index_mailbox *)box;
-
- if (ibox->mbox_data_buf != NULL)
- buffer_free(ibox->mbox_data_buf);
index_storage_mailbox_free(box);
return 0;
}
Index: mbox-sync-parse.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-parse.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mbox-sync-parse.c 13 Jun 2004 07:12:24 -0000 1.9
+++ mbox-sync-parse.c 14 Jun 2004 04:30:32 -0000 1.10
@@ -124,6 +124,17 @@
return TRUE;
}
+static int parse_x_imap(struct mbox_sync_mail_context *ctx,
+ struct message_header_line *hdr)
+{
+ if (!parse_x_imap_base(ctx, hdr))
+ return FALSE;
+
+ /* this is the UW-IMAP style "FOLDER INTERNAL DATA" message. skip it. */
+ ctx->pseudo = TRUE;
+ return TRUE;
+}
+
static int parse_x_keywords(struct mbox_sync_mail_context *ctx,
struct message_header_line *hdr)
{
@@ -226,6 +237,7 @@
static struct header_func header_funcs[] = {
{ "Content-Length", parse_content_length },
{ "Status", parse_status },
+ { "X-IMAP", parse_x_imap },
{ "X-IMAPbase", parse_x_imap_base },
{ "X-Keywords", parse_x_keywords },
{ "X-Status", parse_x_status },
Index: mbox-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mbox-sync-private.h 13 Jun 2004 07:12:24 -0000 1.8
+++ mbox-sync-private.h 14 Jun 2004 04:30:32 -0000 1.9
@@ -30,6 +30,7 @@
uint8_t flags;
keywords_mask_t keywords;
+ uoff_t from_offset;
uoff_t offset; /* if space <= 0, points to beginning */
off_t space;
uoff_t body_size;
@@ -52,6 +53,7 @@
unsigned int have_eoh:1;
unsigned int need_rewrite:1;
unsigned int seen_imapbase:1;
+ unsigned int pseudo:1;
unsigned int updated:1;
};
Index: mbox-sync-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mbox-sync-rewrite.c 13 Jun 2004 20:39:17 -0000 1.9
+++ mbox-sync-rewrite.c 14 Jun 2004 04:30:32 -0000 1.10
@@ -196,17 +196,6 @@
return 1;
}
-static void mbox_sync_fix_from_offset(struct mbox_sync_context *sync_ctx,
- uint32_t idx, off_t diff)
-{
- uoff_t *offset_p;
-
- offset_p = buffer_get_space_unsafe(sync_ctx->ibox->mbox_data_buf,
- idx * sizeof(*offset_p),
- sizeof(*offset_p));
- *offset_p = (*offset_p & 1) | (((*offset_p >> 1) + diff) << 1);
-}
-
static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mails,
uint32_t seq, uint32_t idx,
@@ -245,9 +234,6 @@
i_assert(mail_ctx.mail.space == mails[idx].space);
sync_ctx->prev_msg_uid = old_prev_msg_uid;
- /* we're moving next message - update it's from_offset */
- mbox_sync_fix_from_offset(sync_ctx, idx+1, mails[idx+1].space);
-
if (mail_ctx.mail.space <= 0)
mbox_sync_headers_add_space(&mail_ctx, extra_per_mail);
else if (mail_ctx.mail.space <= extra_per_mail) {
@@ -267,6 +253,7 @@
// FIXME: error handling
return -1;
}
+ mails[idx+1].from_offset += mails[idx+1].space;
*end_offset = offset + mails[idx+1].space - str_len(mail_ctx.header);
@@ -334,7 +321,7 @@
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mails = buffer_get_modifyable_data(mails_buf, &size);
- size /= sizeof(*mails);
+ i_assert(size / sizeof(*mails) == last_seq - first_seq + 1);
/* if there's expunges in mails[], we would get more correct balancing
by counting only them here. however, that might make us overwrite
@@ -386,8 +373,7 @@
break;
}
- mbox_sync_fix_from_offset(sync_ctx, idx+1,
- mails[idx+1].space);
+ mails[idx+1].from_offset += mails[idx+1].space;
mails[idx].space += mails[idx+1].space;
if (mails[idx].uid != 0)
@@ -408,8 +394,7 @@
// FIXME: error handling
ret = -1;
}
- mbox_sync_fix_from_offset(sync_ctx, 1,
- (off_t)end_offset - offset);
+ mails[1].from_offset -= offset - end_offset;
idx++;
start_offset += offset - end_offset;
Index: mbox-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-update.c,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- mbox-sync-update.c 13 Jun 2004 07:12:24 -0000 1.7
+++ mbox-sync-update.c 14 Jun 2004 04:30:32 -0000 1.8
@@ -90,7 +90,7 @@
if (ctx->mail.uid == ctx->sync_ctx->first_uid &&
ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1) {
ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
- str_printfa(ctx->header, "X-IMAPbase: %u %u",
+ str_printfa(ctx->header, "X-IMAPbase: %u %010u",
ctx->sync_ctx->base_uid_validity,
ctx->sync_ctx->next_uid-1);
//FIXME:keywords_append(ctx, all_keywords);
Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- mbox-sync.c 13 Jun 2004 21:50:42 -0000 1.16
+++ mbox-sync.c 14 Jun 2004 04:30:32 -0000 1.17
@@ -106,8 +106,6 @@
mbox_sync_next_mail(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx, uint32_t seq)
{
- uoff_t from_offset;
-
memset(mail_ctx, 0, sizeof(*mail_ctx));
mail_ctx->sync_ctx = sync_ctx;
mail_ctx->seq = seq;
@@ -132,14 +130,12 @@
mail_ctx->content_length);
/* save the offset permanently with recent flag state */
- from_offset = (mail_ctx->from_offset - sync_ctx->expunged_space) << 1;
+ mail_ctx->mail.from_offset = mail_ctx->from_offset;
if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
/* need to add 'O' flag to Status-header */
mail_ctx->need_rewrite = TRUE;
- from_offset |= 1;
+ // FIXME: save it somewhere
}
- buffer_append(sync_ctx->ibox->mbox_data_buf, &from_offset,
- sizeof(from_offset));
}
static void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
@@ -202,10 +198,27 @@
return 0;
}
+static void
+update_from_offsets(struct index_mailbox *ibox,
+ struct mail_index_transaction *t, buffer_t *mails_buf,
+ uint32_t seq1, uint32_t seq2)
+{
+ uint32_t extra_idx = ibox->mbox_extra_idx;
+ const struct mbox_sync_mail *mails;
+
+ mails = buffer_get_modifyable_data(mails_buf, NULL);
+
+ for (; seq1 <= seq2; seq1++, mails++) {
+ uint64_t offset = mails->from_offset;
+ mail_index_update_extra_rec(t, seq1, extra_idx, &offset);
+ }
+}
+
static int mbox_sync_do(struct index_mailbox *ibox,
struct mail_index_sync_ctx *index_sync_ctx,
struct mail_index_view *sync_view,
- buffer_t *syncs, struct mail_index_sync_rec *sync_rec)
+ buffer_t *syncs, struct mail_index_sync_rec *sync_rec,
+ int index_synced)
{
/* a horrible function. needs some serious cleanups. */
struct mbox_sync_context sync_ctx;
@@ -219,18 +232,13 @@
off_t space_diff, move_diff;
uoff_t offset, extra_space, trailer_size;
buffer_t *mails;
- size_t size;
struct stat st;
- int sync_expunge, ret = 0;
+ int sync_expunge, update_from_offset, ret = 0;
- t = mail_index_transaction_begin(sync_view, FALSE);
+ if (mail_index_get_header(sync_view, &hdr) < 0)
+ return -1;
- if (ibox->mbox_data_buf == NULL) {
- ibox->mbox_data_buf =
- buffer_create_dynamic(default_pool, 512, (size_t)-1);
- } else {
- buffer_set_used_size(ibox->mbox_data_buf, 0);
- }
+ t = mail_index_transaction_begin(sync_view, FALSE);
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
@@ -279,10 +287,6 @@
break;
if (seq == 1 && sync_ctx.base_uid_validity == 0) {
- if (mail_index_get_header(sync_view, &hdr) < 0) {
- ret = -1;
- break;
- }
sync_ctx.base_uid_validity =
hdr->uid_validity == 0 ? (uint32_t)ioloop_time :
hdr->uid_validity;
@@ -327,6 +331,7 @@
move_diff);
if (ret > 0) {
+ mail_ctx.mail.from_offset += move_diff;
mail_ctx.mail.offset += move_diff;
ret = mbox_write_from_line(&mail_ctx,
move_diff);
@@ -393,6 +398,7 @@
if (sync_expunge) {
if (rec != NULL)
mail_index_expunge(t, idx_seq);
+ update_from_offset = FALSE;
} else if (rec != NULL) {
/* see if flags changed */
keywords_mask_t old_keywords;
@@ -417,6 +423,8 @@
mail_ctx.mail.keywords);
}
+ update_from_offset = sync_ctx.expunged_space > 0;
+
/* we used this record */
rec = NULL;
} else {
@@ -427,6 +435,34 @@
mail_index_update_flags(t, idx_seq, MODIFY_REPLACE,
new_flags,
mail_ctx.mail.keywords);
+ update_from_offset = TRUE;
+ }
+
+ if (update_from_offset) {
+ /* from_offset needs updating */
+ uint64_t offset;
+
+ offset = mail_ctx.mail.from_offset;
+ mail_index_update_extra_rec(t, idx_seq,
+ ibox->mbox_extra_idx, &offset);
+ } else if (need_space_seq == 0 && !sync_expunge) {
+ /* see if from_offset needs updating */
+ const void *data;
+ uint64_t offset;
+
+ if (mail_index_lookup_extra(sync_view, idx_seq,
+ ibox->mbox_extra_idx,
+ &data) < 0) {
+ ret = -1;
+ break;
+ }
+
+ offset = *((const uint64_t *)data);
+ if (offset != mail_ctx.mail.from_offset) {
+ offset = mail_ctx.mail.from_offset;
+ mail_index_update_extra_rec(t, idx_seq,
+ ibox->mbox_extra_idx, &offset);
+ }
}
istream_raw_mbox_next(input, mail_ctx.mail.body_size);
@@ -474,6 +510,10 @@
ret = -1;
break;
}
+
+ update_from_offsets(ibox, t, mails,
+ need_space_seq, seq);
+
/* mail_ctx may contain wrong data after
rewrite, so make sure we don't try to access
it */
@@ -513,6 +553,9 @@
if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
seq, extra_space) < 0)
ret = -1;
+
+ update_from_offsets(ibox, t, mails,
+ need_space_seq, seq);
}
}
@@ -562,9 +605,6 @@
st.st_size = 0;
}
- if (mail_index_get_header(sync_view, &hdr) < 0)
- ret = -1;
-
if (sync_ctx.base_uid_validity != hdr->uid_validity) {
mail_index_update_header(t,
offsetof(struct mail_index_header, uid_validity),
@@ -615,9 +655,6 @@
mail_storage_set_index_error(ibox);
}
- ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size);
- ibox->mbox_data_count = size / sizeof(*ibox->mbox_data);
-
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(mails);
@@ -652,10 +689,13 @@
uoff_t offset;
struct mail_index_sync_rec sync_rec;
buffer_t *syncs;
- int ret, lock_type;
+ int ret, lock_type, index_synced;
- if ((ret = mbox_sync_has_changed(ibox)) <= 0)
- return ret;
+ if ((ret = mbox_sync_has_changed(ibox)) < 0)
+ return -1;
+ if (ret == 0 && !last_commit)
+ return 0;
+ index_synced = ret > 0;
if (last_commit) {
seq = ibox->commit_log_file_seq;
@@ -681,7 +721,7 @@
if (mbox_lock(ibox, lock_type, &lock_id) > 0 &&
mbox_file_open_stream(ibox) == 0) {
ret = mbox_sync_do(ibox, index_sync_ctx, sync_view,
- syncs, &sync_rec);
+ syncs, &sync_rec, index_synced);
if (ret == -2) {
/* read lock -> write lock. do it again. */
(void)mbox_unlock(ibox, lock_id);
@@ -692,7 +732,8 @@
ret = -1;
else {
ret = mbox_sync_do(ibox, index_sync_ctx,
- sync_view, syncs, &sync_rec);
+ sync_view, syncs, &sync_rec,
+ FALSE);
}
}
} else {
More information about the dovecot-cvs
mailing list