[dovecot-cvs] dovecot/src/lib-storage/index/maildir maildir-sync.c,
1.5, 1.6 maildir-uidlist.c, 1.5, 1.6 maildir-uidlist.h, 1.2, 1.3
cras at procontrol.fi
cras at procontrol.fi
Sun May 2 21:07:27 EEST 2004
Update of /home/cvs/dovecot/src/lib-storage/index/maildir
In directory talvi:/tmp/cvs-serv2133/lib-storage/index/maildir
Modified Files:
maildir-sync.c maildir-uidlist.c maildir-uidlist.h
Log Message:
Syncing optimizations.
Index: maildir-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-sync.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- maildir-sync.c 1 May 2004 18:30:53 -0000 1.5
+++ maildir-sync.c 2 May 2004 18:07:25 -0000 1.6
@@ -30,6 +30,7 @@
struct maildir_sync_context {
struct index_mailbox *ibox;
const char *new_dir, *cur_dir;
+ int partial;
struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
};
@@ -124,7 +125,7 @@
break;
}
}
- if (mail_index_sync_end(sync_ctx) < 0)
+ if (mail_index_sync_end(sync_ctx, 0, 0) < 0)
ret = -1;
}
@@ -261,6 +262,7 @@
static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
int *new_changed_r, int *cur_changed_r)
{
+ const struct mail_index_header *hdr;
struct index_mailbox *ibox = ctx->ibox;
struct stat st;
time_t new_mtime, cur_mtime;
@@ -281,19 +283,27 @@
}
cur_mtime = st.st_mtime;
+ if (ibox->last_cur_mtime == 0) {
+ /* first sync in this session, get cur stamp from index */
+ if (mail_index_get_header(ibox->view, &hdr) == 0)
+ ibox->last_cur_mtime = hdr->sync_stamp;
+ }
+
if (new_mtime != ibox->last_new_mtime ||
new_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS) {
*new_changed_r = TRUE;
ibox->last_new_mtime = new_mtime;
}
+
if (cur_mtime != ibox->last_cur_mtime ||
- (cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS &&
+ (ibox->last_cur_dirty &&
ioloop_time - ibox->last_sync > MAILDIR_SYNC_SECS)) {
/* cur/ changed, or delayed cur/ check */
*cur_changed_r = TRUE;
ibox->last_cur_mtime = cur_mtime;
}
ibox->last_sync = ioloop_time;
+ ibox->last_cur_dirty = cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS;
return 0;
}
@@ -308,10 +318,12 @@
struct mail_index_view *view;
const struct mail_index_header *hdr;
const struct mail_index_record *rec;
- uint32_t seq, uid, uflags;
+ uint32_t seq, uid;
+ enum maildir_uidlist_rec_flag uflags;
const char *filename;
enum mail_flags flags;
custom_flags_mask_t custom_flags;
+ uint32_t sync_stamp;
int ret;
if (mail_index_sync_begin(ibox->index, &sync_ctx, &view,
@@ -332,6 +344,11 @@
__again:
seq++;
+ if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+ /* partial syncing */
+ continue;
+ }
+
if (seq > hdr->messages_count) {
mail_index_append(trans, uid, &seq);
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
@@ -372,6 +389,12 @@
}
maildir_uidlist_iter_deinit(iter);
+ if (!ctx->partial) {
+ /* expunge the rest */
+ for (seq++; seq <= hdr->messages_count; seq++)
+ mail_index_expunge(trans, seq);
+ }
+
if (ret < 0)
mail_index_transaction_rollback(trans);
else {
@@ -393,7 +416,9 @@
break;
}
}
- if (mail_index_sync_end(sync_ctx) < 0)
+
+ sync_stamp = ibox->last_cur_dirty ? 0 : ibox->last_cur_mtime;
+ if (mail_index_sync_end(sync_ctx, sync_stamp, 0) < 0)
ret = -1;
if (ret == 0) {
@@ -416,13 +441,16 @@
if (!new_changed && !cur_changed)
return 0;
- // FIXME: don't sync cur/ directory if not needed
- ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
+ ctx->partial = !cur_changed;
+ ctx->uidlist_sync_ctx =
+ maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
- if (maildir_scan_dir(ctx, FALSE) < 0)
- return -1;
+ if (cur_changed) {
+ if (maildir_scan_dir(ctx, FALSE) < 0)
+ return -1;
+ }
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
@@ -436,7 +464,8 @@
{
int ret;
- ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
+ ctx->uidlist_sync_ctx =
+ maildir_uidlist_sync_init(ctx->ibox->uidlist, FALSE);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
Index: maildir-uidlist.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-uidlist.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- maildir-uidlist.c 1 May 2004 18:30:53 -0000 1.5
+++ maildir-uidlist.c 2 May 2004 18:07:25 -0000 1.6
@@ -51,6 +51,9 @@
struct hash_table *files;
buffer_t *new_record_buf;
+ unsigned int partial_new_pos;
+
+ unsigned int partial:1;
unsigned int new_files:1;
unsigned int synced:1;
unsigned int failed:1;
@@ -204,6 +207,7 @@
}
rec = buffer_append_space_unsafe(uidlist->record_buf, sizeof(*rec));
+ memset(rec, 0, sizeof(*rec));
rec->uid = uid;
rec->flags = flags;
rec->filename = p_strdup(uidlist->filename_pool, line);
@@ -308,7 +312,11 @@
unsigned int idx, left_idx, right_idx;
size_t size;
- i_assert(uidlist->last_mtime != 0);
+ if (uidlist->last_mtime == 0) {
+ /* first time we need to read uidlist */
+ if (maildir_uidlist_update(uidlist) < 0)
+ return NULL;
+ }
rec = buffer_get_data(uidlist->record_buf, &size);
size /= sizeof(*rec);
@@ -378,7 +386,7 @@
rec = buffer_get_data(uidlist->record_buf, &size);
size /= sizeof(*rec);
- maildir_uidlist_lookup(uidlist, uidlist->first_recent_uid, &idx);
+ maildir_uidlist_lookup_rec(uidlist, uidlist->first_recent_uid, &idx);
for (count = 0; idx < size; idx++) {
if ((rec[idx].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0)
count++;
@@ -393,7 +401,8 @@
struct maildir_uidlist_iter_ctx *iter;
struct utimbuf ut;
string_t *str;
- uint32_t uid, flags;
+ uint32_t uid;
+ enum maildir_uidlist_rec_flag flags;
const char *filename, *flags_str;
int ret = 0;
@@ -485,26 +494,115 @@
return ret;
}
+static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist,
+ int nonsynced)
+{
+ struct maildir_uidlist_rec *rec;
+ size_t i, size;
+
+ rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
+ size /= sizeof(*rec);
+
+ if (nonsynced) {
+ for (i = 0; i < size; i++)
+ rec[i].flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+ } else {
+ for (i = 0; i < size; i++)
+ rec[i].flags &= ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+ }
+}
+
struct maildir_uidlist_sync_ctx *
-maildir_uidlist_sync_init(struct maildir_uidlist *uidlist)
+maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial)
{
struct maildir_uidlist_sync_ctx *ctx;
ctx = i_new(struct maildir_uidlist_sync_ctx, 1);
ctx->uidlist = uidlist;
+ ctx->partial = partial;
+
+ if (partial) {
+ /* initially mark all nonsynced */
+ maildir_uidlist_mark_all(uidlist, TRUE);
+ return ctx;
+ }
+
ctx->filename_pool =
pool_alloconly_create("maildir_uidlist_sync", 16384);
ctx->new_record_buf =
buffer_create_dynamic(default_pool, 512, (size_t)-1);
ctx->files = hash_create(default_pool, ctx->filename_pool, 4096,
maildir_hash, maildir_cmp);
+ return ctx;
+}
- if (uidlist->last_mtime == 0) {
- /* uidlist not read yet, do it */
- if (maildir_uidlist_update(uidlist) < 0)
+static int maildir_uidlist_sync_uidlist(struct maildir_uidlist_sync_ctx *ctx)
+{
+ int ret;
+
+ if (ctx->uidlist->last_mtime == 0) {
+ /* first time reading the uidlist,
+ no locking yet */
+ if (maildir_uidlist_update(ctx->uidlist) < 0) {
ctx->failed = TRUE;
+ return -1;
+ }
+ return 0;
}
- return ctx;
+
+ /* lock and update uidlist to see if it's just been added */
+ ret = maildir_uidlist_try_lock(ctx->uidlist);
+ if (ret <= 0) {
+ if (ret == 0)
+ return 1; // FIXME: does it work right?
+ ctx->failed = TRUE;
+ return -1;
+ }
+ if (maildir_uidlist_update(ctx->uidlist) < 0) {
+ ctx->failed = TRUE;
+ return -1;
+ }
+
+ ctx->synced = TRUE;
+ return 1;
+}
+
+static int
+maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,
+ const char *filename,
+ enum maildir_uidlist_rec_flag flags)
+{
+ struct maildir_uidlist *uidlist = ctx->uidlist;
+ struct maildir_uidlist_rec *rec;
+ int ret;
+
+ /* we'll update uidlist directly */
+ rec = hash_lookup(uidlist->files, filename);
+ if (rec == NULL && !ctx->synced) {
+ ret = maildir_uidlist_sync_uidlist(ctx);
+ if (ret < 0)
+ return -1;
+ if (ret == 0) {
+ return maildir_uidlist_sync_next_partial(ctx, filename,
+ flags);
+ }
+ rec = hash_lookup(uidlist->files, filename);
+ }
+
+ if (rec == NULL) {
+ ctx->new_files = TRUE;
+ ctx->partial_new_pos =
+ buffer_get_used_size(uidlist->record_buf) /
+ sizeof(*rec);
+ rec = buffer_append_space_unsafe(uidlist->record_buf,
+ sizeof(*rec));
+ memset(rec, 0, sizeof(*rec));
+ }
+
+ rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+ rec->filename = p_strdup(uidlist->filename_pool, filename);
+ hash_insert(uidlist->files, rec->filename, rec);
+ return 1;
}
int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
@@ -512,12 +610,14 @@
enum maildir_uidlist_rec_flag flags)
{
struct maildir_uidlist_rec *rec;
- char *fname;
int ret;
if (ctx->failed)
return -1;
+ if (ctx->partial)
+ return maildir_uidlist_sync_next_partial(ctx, filename, flags);
+
rec = hash_lookup(ctx->files, filename);
if (rec != NULL) {
if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
@@ -531,21 +631,13 @@
} else {
rec = hash_lookup(ctx->uidlist->files, filename);
if (rec == NULL && !ctx->synced) {
- /* lock and update uidlist to see if it's just
- been added */
- ret = maildir_uidlist_try_lock(ctx->uidlist);
- if (ret <= 0) {
- if (ret == 0)
- return 1; // FIXME: does it work right?
- ctx->failed = TRUE;
- return -1;
- }
- if (maildir_uidlist_update(ctx->uidlist) < 0) {
- ctx->failed = TRUE;
+ ret = maildir_uidlist_sync_uidlist(ctx);
+ if (ret < 0)
return -1;
+ if (ret == 0) {
+ return maildir_uidlist_sync_next(ctx, filename,
+ flags);
}
-
- ctx->synced = TRUE;
rec = hash_lookup(ctx->uidlist->files, filename);
}
@@ -558,11 +650,8 @@
}
rec->flags |= flags;
-
- fname = p_strdup(ctx->filename_pool, filename);
- if (rec->filename == NULL)
- rec->filename = fname;
- hash_insert(ctx->files, fname, rec);
+ rec->filename = p_strdup(ctx->filename_pool, filename);
+ hash_insert(ctx->files, rec->filename, rec);
return 1;
}
@@ -587,6 +676,31 @@
return t1 < t2 ? -1 : t1 > t2 ? 1 : 0;
}
+static void maildir_uidlist_assign_uids(struct maildir_uidlist *uidlist,
+ unsigned int first_new_pos)
+{
+ struct maildir_uidlist_rec *rec;
+ unsigned int dest;
+ size_t size;
+
+ rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
+ size /= sizeof(*rec);
+
+ /* sort new files and assign UIDs for them */
+ qsort(rec + first_new_pos, size - first_new_pos,
+ sizeof(*rec), maildir_time_cmp);
+ for (dest = first_new_pos; dest < size; dest++) {
+ i_assert(rec[dest].uid == 0);
+ rec[dest].uid = uidlist->next_uid++;
+ rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
+
+ if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) {
+ maildir_uidlist_mark_recent(uidlist,
+ rec[dest].uid);
+ }
+ }
+}
+
static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx)
{
struct maildir_uidlist *uidlist = ctx->uidlist;
@@ -620,21 +734,7 @@
buffer_append_buf(uidlist->record_buf, ctx->new_record_buf,
0, (size_t)-1);
-
- rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
- size /= sizeof(*rec);
-
- /* sort new files and assign UIDs for them */
- qsort(rec + dest, size - dest, sizeof(*rec), maildir_time_cmp);
- for (; dest < size; dest++) {
- rec[dest].uid = uidlist->next_uid++;
- rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
-
- if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) {
- maildir_uidlist_mark_recent(ctx->uidlist,
- rec[dest].uid);
- }
- }
+ maildir_uidlist_assign_uids(uidlist, dest);
hash_destroy(uidlist->files);
uidlist->files = ctx->files;
@@ -653,7 +753,16 @@
if (ctx->failed)
ret = -1;
else {
- maildir_uidlist_swap(ctx);
+ if (!ctx->partial)
+ maildir_uidlist_swap(ctx);
+ else {
+ if (ctx->new_files) {
+ maildir_uidlist_assign_uids(ctx->uidlist,
+ ctx->partial_new_pos);
+ }
+ maildir_uidlist_mark_all(ctx->uidlist, FALSE);
+ }
+
if (!ctx->new_files)
ret = 0;
else
@@ -667,7 +776,8 @@
hash_destroy(ctx->files);
if (ctx->filename_pool != NULL)
pool_unref(ctx->filename_pool);
- buffer_free(ctx->new_record_buf);
+ if (ctx->new_record_buf != NULL)
+ buffer_free(ctx->new_record_buf);
i_free(ctx);
return ret;
}
@@ -686,7 +796,8 @@
}
int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx,
- uint32_t *uid_r, uint32_t *flags_r,
+ uint32_t *uid_r,
+ enum maildir_uidlist_rec_flag *flags_r,
const char **filename_r)
{
if (ctx->next == ctx->end)
Index: maildir-uidlist.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-uidlist.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- maildir-uidlist.h 1 May 2004 18:30:53 -0000 1.2
+++ maildir-uidlist.h 2 May 2004 18:07:25 -0000 1.3
@@ -6,7 +6,8 @@
enum maildir_uidlist_rec_flag {
MAILDIR_UIDLIST_REC_FLAG_NEW_DIR = 0x01,
MAILDIR_UIDLIST_REC_FLAG_MOVED = 0x02,
- MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04
+ MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04,
+ MAILDIR_UIDLIST_REC_FLAG_NONSYNCED = 0x08
};
int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist);
@@ -29,7 +30,7 @@
/* Sync uidlist with what's actually on maildir. */
struct maildir_uidlist_sync_ctx *
-maildir_uidlist_sync_init(struct maildir_uidlist *uidlist);
+maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial);
int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
const char *filename,
enum maildir_uidlist_rec_flag flags);
@@ -39,7 +40,8 @@
struct maildir_uidlist_iter_ctx *
maildir_uidlist_iter_init(struct maildir_uidlist *uidlist);
int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx,
- uint32_t *uid_r, uint32_t *flags_r,
+ uint32_t *uid_r,
+ enum maildir_uidlist_rec_flag *flags_r,
const char **filename_r);
void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx *ctx);
More information about the dovecot-cvs
mailing list